@dxos/echo-pipeline 0.1.53-next.63b0547 → 0.1.53-next.e9dc2bc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-SCMKAWNW.mjs → chunk-OJGL5427.mjs} +101 -67
- package/dist/lib/browser/chunk-OJGL5427.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +2 -2
- package/dist/lib/browser/testing/index.mjs.map +2 -2
- package/dist/lib/node/index.cjs +102 -68
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +108 -74
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/dbhost/data-service-host.d.ts +3 -0
- package/dist/types/src/dbhost/data-service-host.d.ts.map +1 -1
- package/dist/types/src/dbhost/data-service.d.ts +2 -2
- package/dist/types/src/dbhost/data-service.d.ts.map +1 -1
- package/dist/types/src/dbhost/snapshot-manager.d.ts +2 -0
- package/dist/types/src/dbhost/snapshot-manager.d.ts.map +1 -1
- package/dist/types/src/dbhost/snapshot-store.d.ts +2 -0
- package/dist/types/src/dbhost/snapshot-store.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +4 -0
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +0 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +3 -3
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +0 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/package.json +30 -30
- package/src/dbhost/data-service-host.ts +14 -1
- package/src/dbhost/data-service.ts +6 -2
- package/src/dbhost/snapshot-manager.ts +4 -4
- package/src/dbhost/snapshot-store.ts +3 -2
- package/src/metadata/metadata-store.ts +15 -2
- package/src/pipeline/pipeline.ts +13 -1
- package/src/space/data-pipeline.ts +25 -19
- package/src/space/replication.browser.test.ts +2 -2
- package/src/space/space.test.ts +46 -5
- package/src/space/space.ts +3 -12
- package/src/testing/util.ts +1 -1
- package/src/tests/database.test.ts +1 -1
- package/dist/lib/browser/chunk-SCMKAWNW.mjs.map +0 -7
|
@@ -57,6 +57,7 @@ var import_util9 = require("@dxos/util");
|
|
|
57
57
|
// packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
|
|
58
58
|
var import_tiny_invariant = __toESM(require("tiny-invariant"));
|
|
59
59
|
var import_codec_protobuf = require("@dxos/codec-protobuf");
|
|
60
|
+
var import_context = require("@dxos/context");
|
|
60
61
|
var import_echo_db = require("@dxos/echo-db");
|
|
61
62
|
var import_log = require("@dxos/log");
|
|
62
63
|
var import_util = require("@dxos/util");
|
|
@@ -66,13 +67,20 @@ var DataServiceHost = class {
|
|
|
66
67
|
this._itemManager = _itemManager;
|
|
67
68
|
this._itemDemuxer = _itemDemuxer;
|
|
68
69
|
this._writeStream = _writeStream;
|
|
70
|
+
this._ctx = new import_context.Context();
|
|
69
71
|
this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
|
|
70
72
|
}
|
|
73
|
+
async open() {
|
|
74
|
+
}
|
|
75
|
+
async close() {
|
|
76
|
+
await this._ctx.dispose();
|
|
77
|
+
}
|
|
71
78
|
/**
|
|
72
79
|
* Real-time subscription to data objects in a space.
|
|
73
80
|
*/
|
|
74
81
|
subscribe() {
|
|
75
|
-
return new import_codec_protobuf.Stream(({ next, ctx }) => {
|
|
82
|
+
return new import_codec_protobuf.Stream(({ next, close, ctx }) => {
|
|
83
|
+
ctx.onDispose(this._ctx.onDispose(close));
|
|
76
84
|
const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
|
|
77
85
|
next({
|
|
78
86
|
batch: {
|
|
@@ -88,7 +96,7 @@ var DataServiceHost = class {
|
|
|
88
96
|
meta
|
|
89
97
|
}, {
|
|
90
98
|
F: __dxlog_file,
|
|
91
|
-
L:
|
|
99
|
+
L: 63,
|
|
92
100
|
S: this,
|
|
93
101
|
C: (f, a) => f(...a)
|
|
94
102
|
});
|
|
@@ -115,13 +123,14 @@ var DataServiceHost = class {
|
|
|
115
123
|
}
|
|
116
124
|
async write(request) {
|
|
117
125
|
var _a, _b;
|
|
126
|
+
(0, import_tiny_invariant.default)(!this._ctx.disposed, "Cannot write to closed DataServiceHost");
|
|
118
127
|
(0, import_tiny_invariant.default)(this._writeStream, "Cannot write mutations in readonly mode");
|
|
119
128
|
(0, import_log.log)("write", {
|
|
120
129
|
clientTag: request.clientTag,
|
|
121
130
|
objectCount: (_b = (_a = request.batch.objects) == null ? void 0 : _a.length) != null ? _b : 0
|
|
122
131
|
}, {
|
|
123
132
|
F: __dxlog_file,
|
|
124
|
-
L:
|
|
133
|
+
L: 92,
|
|
125
134
|
S: this,
|
|
126
135
|
C: (f, a) => f(...a)
|
|
127
136
|
});
|
|
@@ -135,7 +144,7 @@ var DataServiceHost = class {
|
|
|
135
144
|
seq: receipt2.seq
|
|
136
145
|
}, {
|
|
137
146
|
F: __dxlog_file,
|
|
138
|
-
L:
|
|
147
|
+
L: 101,
|
|
139
148
|
S: this,
|
|
140
149
|
C: (f, a) => f(...a)
|
|
141
150
|
});
|
|
@@ -202,7 +211,7 @@ var DatabaseHost = class {
|
|
|
202
211
|
|
|
203
212
|
// packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
|
|
204
213
|
var import_async = require("@dxos/async");
|
|
205
|
-
var
|
|
214
|
+
var import_context2 = require("@dxos/context");
|
|
206
215
|
var import_debug = require("@dxos/debug");
|
|
207
216
|
var import_keys = require("@dxos/keys");
|
|
208
217
|
var import_protocols = require("@dxos/protocols");
|
|
@@ -217,6 +226,7 @@ function _ts_decorate(decorators, target, key, desc) {
|
|
|
217
226
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
218
227
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
219
228
|
}
|
|
229
|
+
var SpaceSnapshot = import_protocols.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
|
|
220
230
|
var SnapshotManager = class SnapshotManager2 {
|
|
221
231
|
// prettier-ignore
|
|
222
232
|
constructor(_snapshotStore, _blobStore, _blobSync) {
|
|
@@ -226,7 +236,7 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
226
236
|
}
|
|
227
237
|
async _getBlob(blobId) {
|
|
228
238
|
const blob = await this._blobStore.get(blobId);
|
|
229
|
-
return
|
|
239
|
+
return SpaceSnapshot.decode(blob);
|
|
230
240
|
}
|
|
231
241
|
async load(ctx, id) {
|
|
232
242
|
const blobId = import_keys.PublicKey.fromHex(id).asUint8Array();
|
|
@@ -234,7 +244,7 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
234
244
|
if (blobMeta && blobMeta.state === import_blob.BlobMeta.State.FULLY_PRESENT) {
|
|
235
245
|
return this._getBlob(blobId);
|
|
236
246
|
}
|
|
237
|
-
const fallbackStore = await (0,
|
|
247
|
+
const fallbackStore = await (0, import_context2.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
|
|
238
248
|
if (fallbackStore) {
|
|
239
249
|
return fallbackStore;
|
|
240
250
|
}
|
|
@@ -242,7 +252,7 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
242
252
|
return this._getBlob(blobId);
|
|
243
253
|
}
|
|
244
254
|
async store(snapshot) {
|
|
245
|
-
const { id } = await this._blobStore.set(
|
|
255
|
+
const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
|
|
246
256
|
await this._blobSync.notifyBlobAdded(id);
|
|
247
257
|
return import_keys.PublicKey.from(id).toHex();
|
|
248
258
|
}
|
|
@@ -257,13 +267,14 @@ SnapshotManager = _ts_decorate([
|
|
|
257
267
|
// packages/core/echo/echo-pipeline/src/dbhost/snapshot-store.ts
|
|
258
268
|
var import_crypto = require("@dxos/crypto");
|
|
259
269
|
var import_protocols2 = require("@dxos/protocols");
|
|
270
|
+
var SpaceSnapshot2 = import_protocols2.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
|
|
260
271
|
var SnapshotStore = class {
|
|
261
272
|
// prettier-ignore
|
|
262
273
|
constructor(_directory) {
|
|
263
274
|
this._directory = _directory;
|
|
264
275
|
}
|
|
265
276
|
async saveSnapshot(snapshot) {
|
|
266
|
-
const encoded =
|
|
277
|
+
const encoded = SpaceSnapshot2.encode(snapshot);
|
|
267
278
|
const key = await import_crypto.subtleCrypto.digest("SHA-256", encoded);
|
|
268
279
|
const keyString = Buffer.from(key).toString("hex");
|
|
269
280
|
const file = await this._directory.getOrCreateFile(keyString);
|
|
@@ -282,7 +293,7 @@ var SnapshotStore = class {
|
|
|
282
293
|
return void 0;
|
|
283
294
|
}
|
|
284
295
|
const buffer = await file.read(0, size);
|
|
285
|
-
return
|
|
296
|
+
return SpaceSnapshot2.decode(buffer);
|
|
286
297
|
} finally {
|
|
287
298
|
await file.close();
|
|
288
299
|
}
|
|
@@ -313,7 +324,7 @@ var DataServiceSubscriptions = class {
|
|
|
313
324
|
clear() {
|
|
314
325
|
this._spaces.clear();
|
|
315
326
|
}
|
|
316
|
-
registerSpace(spaceKey, host) {
|
|
327
|
+
async registerSpace(spaceKey, host) {
|
|
317
328
|
(0, import_log2.log)("Registering space", {
|
|
318
329
|
spaceKey
|
|
319
330
|
}, {
|
|
@@ -322,17 +333,21 @@ var DataServiceSubscriptions = class {
|
|
|
322
333
|
S: this,
|
|
323
334
|
C: (f, a) => f(...a)
|
|
324
335
|
});
|
|
336
|
+
(0, import_tiny_invariant2.default)(!this._spaces.has(spaceKey));
|
|
337
|
+
await host.open();
|
|
325
338
|
this._spaces.set(spaceKey, host);
|
|
326
339
|
}
|
|
327
|
-
unregisterSpace(spaceKey) {
|
|
340
|
+
async unregisterSpace(spaceKey) {
|
|
328
341
|
(0, import_log2.log)("Unregistering space", {
|
|
329
342
|
spaceKey
|
|
330
343
|
}, {
|
|
331
344
|
F: __dxlog_file2,
|
|
332
|
-
L:
|
|
345
|
+
L: 38,
|
|
333
346
|
S: this,
|
|
334
347
|
C: (f, a) => f(...a)
|
|
335
348
|
});
|
|
349
|
+
const host = this._spaces.get(spaceKey);
|
|
350
|
+
await (host == null ? void 0 : host.close());
|
|
336
351
|
this._spaces.delete(spaceKey);
|
|
337
352
|
}
|
|
338
353
|
getDataService(spaceKey) {
|
|
@@ -365,6 +380,7 @@ var import_async2 = require("@dxos/async");
|
|
|
365
380
|
var import_errors = require("@dxos/errors");
|
|
366
381
|
var import_log3 = require("@dxos/log");
|
|
367
382
|
var import_protocols3 = require("@dxos/protocols");
|
|
383
|
+
var import_services = require("@dxos/protocols/proto/dxos/client/services");
|
|
368
384
|
var import_util3 = require("@dxos/util");
|
|
369
385
|
function _ts_decorate2(decorators, target, key, desc) {
|
|
370
386
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
@@ -383,6 +399,7 @@ var emptyEchoMetadata = () => ({
|
|
|
383
399
|
created: new Date(),
|
|
384
400
|
updated: new Date()
|
|
385
401
|
});
|
|
402
|
+
var EchoMetadata = import_protocols3.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
386
403
|
var MetadataStore = class {
|
|
387
404
|
// prettier-ignore
|
|
388
405
|
constructor(_directory) {
|
|
@@ -409,6 +426,7 @@ var MetadataStore = class {
|
|
|
409
426
|
* Loads metadata from persistent storage.
|
|
410
427
|
*/
|
|
411
428
|
async load() {
|
|
429
|
+
var _a;
|
|
412
430
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
413
431
|
try {
|
|
414
432
|
const { size: fileLength } = await file.stat();
|
|
@@ -422,7 +440,7 @@ var MetadataStore = class {
|
|
|
422
440
|
checksum
|
|
423
441
|
}, {
|
|
424
442
|
F: __dxlog_file3,
|
|
425
|
-
L:
|
|
443
|
+
L: 72,
|
|
426
444
|
S: this,
|
|
427
445
|
C: (f, a) => f(...a)
|
|
428
446
|
});
|
|
@@ -434,13 +452,17 @@ var MetadataStore = class {
|
|
|
434
452
|
if (calculatedChecksum !== checksum) {
|
|
435
453
|
throw new import_errors.DataCorruptionError("Metadata checksum is invalid.");
|
|
436
454
|
}
|
|
437
|
-
this._metadata =
|
|
455
|
+
this._metadata = EchoMetadata.decode(data);
|
|
456
|
+
(_a = this._metadata.spaces) == null ? void 0 : _a.forEach((space) => {
|
|
457
|
+
var _a2;
|
|
458
|
+
(_a2 = space.state) != null ? _a2 : space.state = import_services.SpaceState.ACTIVE;
|
|
459
|
+
});
|
|
438
460
|
} catch (err) {
|
|
439
461
|
import_log3.log.error("failed to load metadata", {
|
|
440
462
|
err
|
|
441
463
|
}, {
|
|
442
464
|
F: __dxlog_file3,
|
|
443
|
-
L:
|
|
465
|
+
L: 92,
|
|
444
466
|
S: this,
|
|
445
467
|
C: (f, a) => f(...a)
|
|
446
468
|
});
|
|
@@ -460,7 +482,7 @@ var MetadataStore = class {
|
|
|
460
482
|
this.update.emit(data);
|
|
461
483
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
462
484
|
try {
|
|
463
|
-
const encoded = (0, import_util3.arrayToBuffer)(
|
|
485
|
+
const encoded = (0, import_util3.arrayToBuffer)(EchoMetadata.encode(data));
|
|
464
486
|
const checksum = import_crc_32.default.buf(encoded);
|
|
465
487
|
const result = Buffer.alloc(8 + encoded.length);
|
|
466
488
|
result.writeInt32LE(encoded.length, 0);
|
|
@@ -472,7 +494,7 @@ var MetadataStore = class {
|
|
|
472
494
|
checksum
|
|
473
495
|
}, {
|
|
474
496
|
F: __dxlog_file3,
|
|
475
|
-
L:
|
|
497
|
+
L: 124,
|
|
476
498
|
S: this,
|
|
477
499
|
C: (f, a) => f(...a)
|
|
478
500
|
});
|
|
@@ -495,7 +517,7 @@ var MetadataStore = class {
|
|
|
495
517
|
async clear() {
|
|
496
518
|
(0, import_log3.log)("clearing all metadata", void 0, {
|
|
497
519
|
F: __dxlog_file3,
|
|
498
|
-
L:
|
|
520
|
+
L: 145,
|
|
499
521
|
S: this,
|
|
500
522
|
C: (f, a) => f(...a)
|
|
501
523
|
});
|
|
@@ -534,6 +556,10 @@ var MetadataStore = class {
|
|
|
534
556
|
space.dataFeedKey = dataFeedKey;
|
|
535
557
|
await this._save();
|
|
536
558
|
}
|
|
559
|
+
async setSpaceState(spaceKey, state) {
|
|
560
|
+
this._getSpace(spaceKey).state = state;
|
|
561
|
+
await this._save();
|
|
562
|
+
}
|
|
537
563
|
};
|
|
538
564
|
_ts_decorate2([
|
|
539
565
|
import_async2.synchronized
|
|
@@ -546,7 +572,7 @@ var fromBytesInt32 = (buf) => buf.readInt32LE(0);
|
|
|
546
572
|
// packages/core/echo/echo-pipeline/src/space/auth.ts
|
|
547
573
|
var import_tiny_invariant4 = __toESM(require("tiny-invariant"));
|
|
548
574
|
var import_async3 = require("@dxos/async");
|
|
549
|
-
var
|
|
575
|
+
var import_context3 = require("@dxos/context");
|
|
550
576
|
var import_crypto2 = require("@dxos/crypto");
|
|
551
577
|
var import_log4 = require("@dxos/log");
|
|
552
578
|
var import_protocols4 = require("@dxos/protocols");
|
|
@@ -564,7 +590,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
564
590
|
timeout: 60 * 1e3
|
|
565
591
|
});
|
|
566
592
|
this._authParams = _authParams;
|
|
567
|
-
this._ctx = new
|
|
593
|
+
this._ctx = new import_context3.Context({
|
|
568
594
|
onError: (err) => {
|
|
569
595
|
import_log4.log.catch(err, void 0, {
|
|
570
596
|
F: __dxlog_file4,
|
|
@@ -648,7 +674,7 @@ var import_util5 = require("@dxos/util");
|
|
|
648
674
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
649
675
|
var import_tiny_invariant7 = __toESM(require("tiny-invariant"));
|
|
650
676
|
var import_async5 = require("@dxos/async");
|
|
651
|
-
var
|
|
677
|
+
var import_context4 = require("@dxos/context");
|
|
652
678
|
var import_debug4 = require("@dxos/debug");
|
|
653
679
|
var import_feed_store = require("@dxos/feed-store");
|
|
654
680
|
var import_keys3 = require("@dxos/keys");
|
|
@@ -805,6 +831,7 @@ var PipelineState = class {
|
|
|
805
831
|
constructor(_feeds, _timeframeClock) {
|
|
806
832
|
this._feeds = _feeds;
|
|
807
833
|
this._timeframeClock = _timeframeClock;
|
|
834
|
+
this._ctx = new import_context4.Context();
|
|
808
835
|
this.timeframeUpdate = this._timeframeClock.update;
|
|
809
836
|
this.stalled = new import_async5.Event();
|
|
810
837
|
this._startTimeframe = new import_timeframe2.Timeframe();
|
|
@@ -848,7 +875,7 @@ var PipelineState = class {
|
|
|
848
875
|
*
|
|
849
876
|
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
850
877
|
*/
|
|
851
|
-
async waitUntilReachedTargetTimeframe({ ctx = new
|
|
878
|
+
async waitUntilReachedTargetTimeframe({ ctx = new import_context4.Context(), timeout, breakOnStall = true } = {}) {
|
|
852
879
|
var _a;
|
|
853
880
|
(0, import_log7.log)("waitUntilReachedTargetTimeframe", {
|
|
854
881
|
timeout,
|
|
@@ -856,7 +883,7 @@ var PipelineState = class {
|
|
|
856
883
|
target: this.targetTimeframe
|
|
857
884
|
}, {
|
|
858
885
|
F: __dxlog_file7,
|
|
859
|
-
L:
|
|
886
|
+
L: 126,
|
|
860
887
|
S: this,
|
|
861
888
|
C: (f, a) => f(...a)
|
|
862
889
|
});
|
|
@@ -871,7 +898,8 @@ var PipelineState = class {
|
|
|
871
898
|
let done = false;
|
|
872
899
|
if (timeout) {
|
|
873
900
|
return Promise.race([
|
|
874
|
-
(0,
|
|
901
|
+
(0, import_context4.rejectOnDispose)(ctx),
|
|
902
|
+
(0, import_context4.rejectOnDispose)(this._ctx),
|
|
875
903
|
this._reachedTargetPromise.then(() => {
|
|
876
904
|
done = true;
|
|
877
905
|
}),
|
|
@@ -886,7 +914,7 @@ var PipelineState = class {
|
|
|
886
914
|
dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
|
|
887
915
|
}, {
|
|
888
916
|
F: __dxlog_file7,
|
|
889
|
-
L:
|
|
917
|
+
L: 153,
|
|
890
918
|
S: this,
|
|
891
919
|
C: (f, a) => f(...a)
|
|
892
920
|
});
|
|
@@ -944,7 +972,7 @@ var Pipeline = class {
|
|
|
944
972
|
async start() {
|
|
945
973
|
(0, import_log7.log)("starting...", {}, {
|
|
946
974
|
F: __dxlog_file7,
|
|
947
|
-
L:
|
|
975
|
+
L: 273,
|
|
948
976
|
S: this,
|
|
949
977
|
C: (f, a) => f(...a)
|
|
950
978
|
});
|
|
@@ -952,7 +980,7 @@ var Pipeline = class {
|
|
|
952
980
|
await this._feedSetIterator.open();
|
|
953
981
|
(0, import_log7.log)("started", void 0, {
|
|
954
982
|
F: __dxlog_file7,
|
|
955
|
-
L:
|
|
983
|
+
L: 276,
|
|
956
984
|
S: this,
|
|
957
985
|
C: (f, a) => f(...a)
|
|
958
986
|
});
|
|
@@ -962,16 +990,19 @@ var Pipeline = class {
|
|
|
962
990
|
var _a;
|
|
963
991
|
(0, import_log7.log)("stopping...", {}, {
|
|
964
992
|
F: __dxlog_file7,
|
|
965
|
-
L:
|
|
993
|
+
L: 282,
|
|
966
994
|
S: this,
|
|
967
995
|
C: (f, a) => f(...a)
|
|
968
996
|
});
|
|
969
997
|
this._isStopping = true;
|
|
970
998
|
await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
|
|
971
999
|
await this._processingTrigger.wait();
|
|
1000
|
+
await this._state._ctx.dispose();
|
|
1001
|
+
this._state._ctx = new import_context4.Context();
|
|
1002
|
+
this._state._reachedTargetPromise = void 0;
|
|
972
1003
|
(0, import_log7.log)("stopped", void 0, {
|
|
973
1004
|
F: __dxlog_file7,
|
|
974
|
-
L:
|
|
1005
|
+
L: 289,
|
|
975
1006
|
S: this,
|
|
976
1007
|
C: (f, a) => f(...a)
|
|
977
1008
|
});
|
|
@@ -1048,7 +1079,7 @@ var Pipeline = class {
|
|
|
1048
1079
|
feed.undownload({
|
|
1049
1080
|
callback: () => (0, import_log7.log)("Undownloaded", void 0, {
|
|
1050
1081
|
F: __dxlog_file7,
|
|
1051
|
-
L:
|
|
1082
|
+
L: 383,
|
|
1052
1083
|
S: this,
|
|
1053
1084
|
C: (f, a) => f(...a)
|
|
1054
1085
|
})
|
|
@@ -1061,7 +1092,7 @@ var Pipeline = class {
|
|
|
1061
1092
|
err
|
|
1062
1093
|
}, {
|
|
1063
1094
|
F: __dxlog_file7,
|
|
1064
|
-
L:
|
|
1095
|
+
L: 386,
|
|
1065
1096
|
S: this,
|
|
1066
1097
|
C: (f, a) => f(...a)
|
|
1067
1098
|
});
|
|
@@ -1075,7 +1106,7 @@ var Pipeline = class {
|
|
|
1075
1106
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
1076
1107
|
import_log7.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
1077
1108
|
F: __dxlog_file7,
|
|
1078
|
-
L:
|
|
1109
|
+
L: 396,
|
|
1079
1110
|
S: this,
|
|
1080
1111
|
C: (f, a) => f(...a)
|
|
1081
1112
|
});
|
|
@@ -1244,7 +1275,7 @@ var ControlPipeline = class {
|
|
|
1244
1275
|
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1245
1276
|
var import_tiny_invariant8 = __toESM(require("tiny-invariant"));
|
|
1246
1277
|
var import_async6 = require("@dxos/async");
|
|
1247
|
-
var
|
|
1278
|
+
var import_context5 = require("@dxos/context");
|
|
1248
1279
|
var import_credentials3 = require("@dxos/credentials");
|
|
1249
1280
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
1250
1281
|
var import_errors2 = require("@dxos/errors");
|
|
@@ -1267,11 +1298,15 @@ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
|
1267
1298
|
var DataPipeline = class DataPipeline2 {
|
|
1268
1299
|
constructor(_params) {
|
|
1269
1300
|
this._params = _params;
|
|
1270
|
-
this._ctx = new
|
|
1301
|
+
this._ctx = new import_context5.Context();
|
|
1302
|
+
this._pipeline = void 0;
|
|
1303
|
+
this._targetTimeframe = void 0;
|
|
1271
1304
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
1272
1305
|
this._isOpen = false;
|
|
1273
1306
|
this._lastTimeframeSaveTime = 0;
|
|
1274
1307
|
this._lastSnapshotSaveTime = 0;
|
|
1308
|
+
this.currentEpoch = void 0;
|
|
1309
|
+
this.appliedEpoch = void 0;
|
|
1275
1310
|
this._lastProcessedEpoch = -1;
|
|
1276
1311
|
this.onNewEpoch = new import_async6.Event();
|
|
1277
1312
|
}
|
|
@@ -1290,18 +1325,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1290
1325
|
this._targetTimeframe = timeframe;
|
|
1291
1326
|
(_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
|
|
1292
1327
|
}
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
await this._processEpochInSeparateTask(credential);
|
|
1302
|
-
}
|
|
1303
|
-
}
|
|
1304
|
-
};
|
|
1328
|
+
async processCredential(credential) {
|
|
1329
|
+
if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
|
|
1330
|
+
return;
|
|
1331
|
+
}
|
|
1332
|
+
this.currentEpoch = credential;
|
|
1333
|
+
if (this._isOpen) {
|
|
1334
|
+
await this._processEpochInSeparateTask(credential);
|
|
1335
|
+
}
|
|
1305
1336
|
}
|
|
1306
1337
|
async open() {
|
|
1307
1338
|
if (this._isOpen) {
|
|
@@ -1338,7 +1369,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1338
1369
|
}
|
|
1339
1370
|
(0, import_log9.log)("close", void 0, {
|
|
1340
1371
|
F: __dxlog_file9,
|
|
1341
|
-
L:
|
|
1372
|
+
L: 171,
|
|
1342
1373
|
S: this,
|
|
1343
1374
|
C: (f, a) => f(...a)
|
|
1344
1375
|
});
|
|
@@ -1353,13 +1384,21 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1353
1384
|
} catch (err) {
|
|
1354
1385
|
import_log9.log.catch(err, void 0, {
|
|
1355
1386
|
F: __dxlog_file9,
|
|
1356
|
-
L:
|
|
1387
|
+
L: 184,
|
|
1357
1388
|
S: this,
|
|
1358
1389
|
C: (f, a) => f(...a)
|
|
1359
1390
|
});
|
|
1360
1391
|
}
|
|
1361
1392
|
await ((_b = this.databaseHost) == null ? void 0 : _b.close());
|
|
1362
1393
|
await ((_c = this.itemManager) == null ? void 0 : _c.destroy());
|
|
1394
|
+
this._ctx = new import_context5.Context();
|
|
1395
|
+
this._pipeline = void 0;
|
|
1396
|
+
this._targetTimeframe = void 0;
|
|
1397
|
+
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
1398
|
+
this.currentEpoch = void 0;
|
|
1399
|
+
this.appliedEpoch = void 0;
|
|
1400
|
+
this._lastProcessedEpoch = -1;
|
|
1401
|
+
this._epochCtx = void 0;
|
|
1363
1402
|
}
|
|
1364
1403
|
async _consumePipeline() {
|
|
1365
1404
|
if (this.currentEpoch) {
|
|
@@ -1375,7 +1414,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1375
1414
|
seq
|
|
1376
1415
|
}, {
|
|
1377
1416
|
F: __dxlog_file9,
|
|
1378
|
-
L:
|
|
1417
|
+
L: 210,
|
|
1379
1418
|
S: this,
|
|
1380
1419
|
C: (f, a) => f(...a)
|
|
1381
1420
|
});
|
|
@@ -1387,7 +1426,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1387
1426
|
feedKey
|
|
1388
1427
|
}, {
|
|
1389
1428
|
F: __dxlog_file9,
|
|
1390
|
-
L:
|
|
1429
|
+
L: 216,
|
|
1391
1430
|
S: this,
|
|
1392
1431
|
C: (f, a) => f(...a)
|
|
1393
1432
|
});
|
|
@@ -1408,7 +1447,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1408
1447
|
spaceKey: this._params.spaceKey.toHex()
|
|
1409
1448
|
}, {
|
|
1410
1449
|
F: __dxlog_file9,
|
|
1411
|
-
L:
|
|
1450
|
+
L: 230,
|
|
1412
1451
|
S: this,
|
|
1413
1452
|
C: (f, a) => f(...a)
|
|
1414
1453
|
});
|
|
@@ -1417,7 +1456,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1417
1456
|
} catch (err) {
|
|
1418
1457
|
import_log9.log.catch(err, void 0, {
|
|
1419
1458
|
F: __dxlog_file9,
|
|
1420
|
-
L:
|
|
1459
|
+
L: 240,
|
|
1421
1460
|
S: this,
|
|
1422
1461
|
C: (f, a) => f(...a)
|
|
1423
1462
|
});
|
|
@@ -1452,7 +1491,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1452
1491
|
} catch (err) {
|
|
1453
1492
|
import_log9.log.warn("Failed to cache properties", err, {
|
|
1454
1493
|
F: __dxlog_file9,
|
|
1455
|
-
L:
|
|
1494
|
+
L: 275,
|
|
1456
1495
|
S: this,
|
|
1457
1496
|
C: (f, a) => f(...a)
|
|
1458
1497
|
});
|
|
@@ -1474,19 +1513,19 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1474
1513
|
return;
|
|
1475
1514
|
}
|
|
1476
1515
|
await ((_a = this._epochCtx) == null ? void 0 : _a.dispose());
|
|
1477
|
-
const ctx = new
|
|
1516
|
+
const ctx = new import_context5.Context({
|
|
1478
1517
|
onError: (err) => {
|
|
1479
1518
|
if (err instanceof import_errors2.CancelledError) {
|
|
1480
1519
|
(0, import_log9.log)("Epoch processing cancelled.", void 0, {
|
|
1481
1520
|
F: __dxlog_file9,
|
|
1482
|
-
L:
|
|
1521
|
+
L: 307,
|
|
1483
1522
|
S: this,
|
|
1484
1523
|
C: (f, a) => f(...a)
|
|
1485
1524
|
});
|
|
1486
1525
|
} else {
|
|
1487
1526
|
import_log9.log.catch(err, void 0, {
|
|
1488
1527
|
F: __dxlog_file9,
|
|
1489
|
-
L:
|
|
1528
|
+
L: 309,
|
|
1490
1529
|
S: this,
|
|
1491
1530
|
C: (f, a) => f(...a)
|
|
1492
1531
|
});
|
|
@@ -1502,7 +1541,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1502
1541
|
epoch
|
|
1503
1542
|
}, {
|
|
1504
1543
|
F: __dxlog_file9,
|
|
1505
|
-
L:
|
|
1544
|
+
L: 319,
|
|
1506
1545
|
S: this,
|
|
1507
1546
|
C: (f, a) => f(...a)
|
|
1508
1547
|
});
|
|
@@ -1519,7 +1558,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1519
1558
|
epoch
|
|
1520
1559
|
}, {
|
|
1521
1560
|
F: __dxlog_file9,
|
|
1522
|
-
L:
|
|
1561
|
+
L: 333,
|
|
1523
1562
|
S: this,
|
|
1524
1563
|
C: (f, a) => f(...a)
|
|
1525
1564
|
});
|
|
@@ -1529,7 +1568,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1529
1568
|
}
|
|
1530
1569
|
(0, import_log9.log)("restarting pipeline for epoch", void 0, {
|
|
1531
1570
|
F: __dxlog_file9,
|
|
1532
|
-
L:
|
|
1571
|
+
L: 341,
|
|
1533
1572
|
S: this,
|
|
1534
1573
|
C: (f, a) => f(...a)
|
|
1535
1574
|
});
|
|
@@ -1593,8 +1632,6 @@ var Space = class Space2 {
|
|
|
1593
1632
|
this._addFeedLock = new import_async7.Lock();
|
|
1594
1633
|
this.onCredentialProcessed = new import_util6.Callback();
|
|
1595
1634
|
this.stateUpdate = new import_async7.Event();
|
|
1596
|
-
// Processes epoch credentials.
|
|
1597
|
-
this._dataPipelineCredentialConsumer = void 0;
|
|
1598
1635
|
this._isOpen = false;
|
|
1599
1636
|
(0, import_tiny_invariant9.default)(params.spaceKey && params.feedProvider);
|
|
1600
1637
|
this._key = params.spaceKey;
|
|
@@ -1632,7 +1669,7 @@ var Space = class Space2 {
|
|
|
1632
1669
|
credential
|
|
1633
1670
|
}, {
|
|
1634
1671
|
F: __dxlog_file10,
|
|
1635
|
-
L:
|
|
1672
|
+
L: 107,
|
|
1636
1673
|
S: this,
|
|
1637
1674
|
C: (f, a) => f(...a)
|
|
1638
1675
|
});
|
|
@@ -1724,7 +1761,7 @@ var Space = class Space2 {
|
|
|
1724
1761
|
async open() {
|
|
1725
1762
|
(0, import_log10.log)("opening...", void 0, {
|
|
1726
1763
|
F: __dxlog_file10,
|
|
1727
|
-
L:
|
|
1764
|
+
L: 209,
|
|
1728
1765
|
S: this,
|
|
1729
1766
|
C: (f, a) => f(...a)
|
|
1730
1767
|
});
|
|
@@ -1733,38 +1770,35 @@ var Space = class Space2 {
|
|
|
1733
1770
|
}
|
|
1734
1771
|
await this._controlPipeline.start();
|
|
1735
1772
|
await this.protocol.start();
|
|
1736
|
-
|
|
1737
|
-
await this._dataPipelineCredentialConsumer.open();
|
|
1773
|
+
await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
|
|
1738
1774
|
this._isOpen = true;
|
|
1739
1775
|
(0, import_log10.log)("opened", void 0, {
|
|
1740
1776
|
F: __dxlog_file10,
|
|
1741
|
-
L:
|
|
1777
|
+
L: 220,
|
|
1742
1778
|
S: this,
|
|
1743
1779
|
C: (f, a) => f(...a)
|
|
1744
1780
|
});
|
|
1745
1781
|
}
|
|
1746
1782
|
async close() {
|
|
1747
|
-
var _a;
|
|
1748
1783
|
(0, import_log10.log)("closing...", {
|
|
1749
1784
|
key: this._key
|
|
1750
1785
|
}, {
|
|
1751
1786
|
F: __dxlog_file10,
|
|
1752
|
-
L:
|
|
1787
|
+
L: 225,
|
|
1753
1788
|
S: this,
|
|
1754
1789
|
C: (f, a) => f(...a)
|
|
1755
1790
|
});
|
|
1756
1791
|
if (!this._isOpen) {
|
|
1757
1792
|
return;
|
|
1758
1793
|
}
|
|
1759
|
-
await
|
|
1760
|
-
this._dataPipelineCredentialConsumer = void 0;
|
|
1794
|
+
await this._controlPipeline.spaceState.removeCredentialProcessor(this._dataPipeline);
|
|
1761
1795
|
await this._dataPipeline.close();
|
|
1762
1796
|
await this.protocol.stop();
|
|
1763
1797
|
await this._controlPipeline.stop();
|
|
1764
1798
|
this._isOpen = false;
|
|
1765
1799
|
(0, import_log10.log)("closed", void 0, {
|
|
1766
1800
|
F: __dxlog_file10,
|
|
1767
|
-
L:
|
|
1801
|
+
L: 238,
|
|
1768
1802
|
S: this,
|
|
1769
1803
|
C: (f, a) => f(...a)
|
|
1770
1804
|
});
|
|
@@ -1772,7 +1806,7 @@ var Space = class Space2 {
|
|
|
1772
1806
|
async initializeDataPipeline() {
|
|
1773
1807
|
(0, import_log10.log)("initializeDataPipeline", void 0, {
|
|
1774
1808
|
F: __dxlog_file10,
|
|
1775
|
-
L:
|
|
1809
|
+
L: 243,
|
|
1776
1810
|
S: this,
|
|
1777
1811
|
C: (f, a) => f(...a)
|
|
1778
1812
|
});
|
|
@@ -2318,7 +2352,7 @@ var createRemoteDatabaseFromDataServiceHost = async (modelFactory, dataServiceHo
|
|
|
2318
2352
|
const dataServiceSubscriptions = new DataServiceSubscriptions();
|
|
2319
2353
|
const dataService = new DataServiceImpl(dataServiceSubscriptions);
|
|
2320
2354
|
const spaceKey = import_keys8.PublicKey.random();
|
|
2321
|
-
dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
|
|
2355
|
+
await dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
|
|
2322
2356
|
const itemManager = new import_echo_db4.ItemManager(modelFactory);
|
|
2323
2357
|
const backend = new import_echo_db4.DatabaseProxy(dataService, itemManager, spaceKey);
|
|
2324
2358
|
await backend.open(new import_model_factory2.ModelFactory().registerModel(import_document_model2.DocumentModel));
|