@dxos/echo-pipeline 0.1.31-next.ea8f0bc → 0.1.32-next.77d513e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/lib/browser/{chunk-L44NG4N6.mjs → chunk-EM5HSJ7J.mjs} +192 -122
  2. package/dist/lib/browser/chunk-EM5HSJ7J.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +3 -5
  4. package/dist/lib/browser/index.mjs.map +1 -1
  5. package/dist/lib/browser/meta.json +1 -1
  6. package/dist/lib/browser/testing/index.mjs +34 -17
  7. package/dist/lib/browser/testing/index.mjs.map +3 -3
  8. package/dist/lib/node/index.cjs +189 -120
  9. package/dist/lib/node/index.cjs.map +4 -4
  10. package/dist/lib/node/meta.json +1 -1
  11. package/dist/lib/node/testing/index.cjs +175 -119
  12. package/dist/lib/node/testing/index.cjs.map +4 -4
  13. package/dist/types/src/dbhost/data-service-host.d.ts.map +1 -1
  14. package/dist/types/src/dbhost/database-backend.d.ts +2 -2
  15. package/dist/types/src/dbhost/database-backend.d.ts.map +1 -1
  16. package/dist/types/src/metadata/metadata-store.d.ts +1 -2
  17. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  18. package/dist/types/src/pipeline/pipeline.d.ts +10 -1
  19. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  20. package/dist/types/src/space/{data-pipeline-controller.d.ts → data-pipeline.d.ts} +10 -18
  21. package/dist/types/src/space/data-pipeline.d.ts.map +1 -0
  22. package/dist/types/src/space/genesis.d.ts.map +1 -1
  23. package/dist/types/src/space/index.d.ts +1 -1
  24. package/dist/types/src/space/index.d.ts.map +1 -1
  25. package/dist/types/src/space/space.d.ts +6 -4
  26. package/dist/types/src/space/space.d.ts.map +1 -1
  27. package/dist/types/src/testing/test-agent-builder.d.ts +6 -5
  28. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  29. package/dist/types/src/testing/util.d.ts +2 -2
  30. package/dist/types/src/testing/util.d.ts.map +1 -1
  31. package/package.json +29 -29
  32. package/src/dbhost/data-service-host.ts +19 -25
  33. package/src/dbhost/database-backend.ts +1 -1
  34. package/src/metadata/metadata-store.ts +1 -2
  35. package/src/pipeline/pipeline.test.ts +12 -4
  36. package/src/pipeline/pipeline.ts +18 -3
  37. package/src/space/{data-pipeline-controller.ts → data-pipeline.ts} +62 -39
  38. package/src/space/genesis.ts +47 -7
  39. package/src/space/index.ts +1 -1
  40. package/src/space/space-protocol.browser.test.ts +8 -6
  41. package/src/space/space-protocol.test.ts +8 -6
  42. package/src/space/space.ts +22 -23
  43. package/src/testing/database-test-rig.ts +1 -1
  44. package/src/testing/test-agent-builder.ts +23 -11
  45. package/src/testing/util.ts +13 -12
  46. package/src/tests/database-unit.test.ts +25 -0
  47. package/src/tests/database.test.ts +5 -5
  48. package/dist/lib/browser/chunk-L44NG4N6.mjs.map +0 -7
  49. package/dist/types/src/space/data-pipeline-controller.d.ts.map +0 -1
@@ -33,7 +33,7 @@ __export(src_exports, {
33
33
  AuthExtension: () => AuthExtension,
34
34
  AuthStatus: () => AuthStatus,
35
35
  DBError: () => DBError,
36
- DataPipelineControllerImpl: () => DataPipelineControllerImpl,
36
+ DataPipeline: () => DataPipeline,
37
37
  DataServiceHost: () => DataServiceHost,
38
38
  DataServiceImpl: () => DataServiceImpl,
39
39
  DataServiceSubscriptions: () => DataServiceSubscriptions,
@@ -45,7 +45,6 @@ __export(src_exports, {
45
45
  MOCK_AUTH_PROVIDER: () => MOCK_AUTH_PROVIDER,
46
46
  MOCK_AUTH_VERIFIER: () => MOCK_AUTH_VERIFIER,
47
47
  MetadataStore: () => MetadataStore,
48
- NoopDataPipelineController: () => NoopDataPipelineController,
49
48
  Pipeline: () => Pipeline,
50
49
  STORAGE_VERSION: () => STORAGE_VERSION,
51
50
  SnapshotManager: () => SnapshotManager,
@@ -170,7 +169,7 @@ var MetadataStore = class {
170
169
  size: dataSize
171
170
  }, {
172
171
  file: "metadata-store.ts",
173
- line: 68,
172
+ line: 67,
174
173
  scope: this,
175
174
  callSite: (f, a) => f(...a)
176
175
  });
@@ -186,7 +185,7 @@ var MetadataStore = class {
186
185
  err
187
186
  }, {
188
187
  file: "metadata-store.ts",
189
- line: 80,
188
+ line: 79,
190
189
  scope: this,
191
190
  callSite: (f, a) => f(...a)
192
191
  });
@@ -211,7 +210,7 @@ var MetadataStore = class {
211
210
  size: encoded.length
212
211
  }, {
213
212
  file: "metadata-store.ts",
214
- line: 103,
213
+ line: 102,
215
214
  scope: this,
216
215
  callSite: (f, a) => f(...a)
217
216
  });
@@ -226,7 +225,7 @@ var MetadataStore = class {
226
225
  async clear() {
227
226
  (0, import_log.log)("clearing all metadata", {}, {
228
227
  file: "metadata-store.ts",
229
- line: 116,
228
+ line: 115,
230
229
  scope: this,
231
230
  callSite: (f, a) => f(...a)
232
231
  });
@@ -282,6 +281,7 @@ var fromBytesInt32 = (buf) => buf.readInt32LE(0);
282
281
  // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
283
282
  var import_node_assert4 = __toESM(require("node:assert"));
284
283
  var import_async3 = require("@dxos/async");
284
+ var import_context = require("@dxos/context");
285
285
  var import_feed_store = require("@dxos/feed-store");
286
286
  var import_keys = require("@dxos/keys");
287
287
  var import_log3 = require("@dxos/log");
@@ -416,7 +416,7 @@ var PipelineState = class {
416
416
  return this._timeframeClock.timeframe;
417
417
  }
418
418
  get targetTimeframe() {
419
- return this._targetTimeframe ? import_timeframe2.Timeframe.merge(this.endTimeframe, this._targetTimeframe) : this.endTimeframe;
419
+ return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
420
420
  }
421
421
  async waitUntilTimeframe(target) {
422
422
  await this._timeframeClock.waitUntilReached(target);
@@ -431,7 +431,7 @@ var PipelineState = class {
431
431
  *
432
432
  * @param timeout Timeout in milliseconds to specify the maximum wait time.
433
433
  */
434
- async waitUntilReachedTargetTimeframe({ timeout } = {}) {
434
+ async waitUntilReachedTargetTimeframe({ ctx = new import_context.Context(), timeout, breakOnStall = true } = {}) {
435
435
  var _a;
436
436
  (0, import_log3.log)("waitUntilReachedTargetTimeframe", {
437
437
  timeout,
@@ -439,7 +439,7 @@ var PipelineState = class {
439
439
  target: this.targetTimeframe
440
440
  }, {
441
441
  file: "pipeline.ts",
442
- line: 82,
442
+ line: 96,
443
443
  scope: this,
444
444
  callSite: (f, a) => f(...a)
445
445
  });
@@ -447,11 +447,14 @@ var PipelineState = class {
447
447
  this._timeframeClock.update.waitForCondition(() => {
448
448
  return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
449
449
  }),
450
- this._iterator.stalled.discardParameter().waitForCount(1)
450
+ ...breakOnStall ? [
451
+ this._iterator.stalled.discardParameter().waitForCount(1)
452
+ ] : []
451
453
  ]);
452
454
  let done = false;
453
455
  if (timeout) {
454
456
  return Promise.race([
457
+ (0, import_context.rejectOnDispose)(ctx),
455
458
  this._reachedTargetPromise.then(() => {
456
459
  done = true;
457
460
  }),
@@ -466,7 +469,7 @@ var PipelineState = class {
466
469
  dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
467
470
  }, {
468
471
  file: "pipeline.ts",
469
- line: 107,
472
+ line: 122,
470
473
  scope: this,
471
474
  callSite: (f, a) => f(...a)
472
475
  });
@@ -492,7 +495,7 @@ var Pipeline = class {
492
495
  this._feedSetIterator.stalled.on((iterator) => {
493
496
  import_log3.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
494
497
  file: "pipeline.ts",
495
- line: 183,
498
+ line: 198,
496
499
  scope: this,
497
500
  callSite: (f, a) => f(...a)
498
501
  });
@@ -522,14 +525,14 @@ var Pipeline = class {
522
525
  async start() {
523
526
  (0, import_log3.log)("starting...", {}, {
524
527
  file: "pipeline.ts",
525
- line: 218,
528
+ line: 233,
526
529
  scope: this,
527
530
  callSite: (f, a) => f(...a)
528
531
  });
529
532
  await this._feedSetIterator.open();
530
533
  (0, import_log3.log)("started", {}, {
531
534
  file: "pipeline.ts",
532
- line: 220,
535
+ line: 235,
533
536
  scope: this,
534
537
  callSite: (f, a) => f(...a)
535
538
  });
@@ -537,7 +540,7 @@ var Pipeline = class {
537
540
  async stop() {
538
541
  (0, import_log3.log)("stopping...", {}, {
539
542
  file: "pipeline.ts",
540
- line: 224,
543
+ line: 239,
541
544
  scope: this,
542
545
  callSite: (f, a) => f(...a)
543
546
  });
@@ -545,7 +548,7 @@ var Pipeline = class {
545
548
  await this._processingTrigger.wait();
546
549
  (0, import_log3.log)("stopped", {}, {
547
550
  file: "pipeline.ts",
548
- line: 227,
551
+ line: 242,
549
552
  scope: this,
550
553
  callSite: (f, a) => f(...a)
551
554
  });
@@ -570,7 +573,7 @@ var Pipeline = class {
570
573
  // packages/core/echo/echo-pipeline/src/space/auth.ts
571
574
  var import_node_assert5 = __toESM(require("node:assert"));
572
575
  var import_async4 = require("@dxos/async");
573
- var import_context = require("@dxos/context");
576
+ var import_context2 = require("@dxos/context");
574
577
  var import_crypto = require("@dxos/crypto");
575
578
  var import_log4 = require("@dxos/log");
576
579
  var import_protocols3 = require("@dxos/protocols");
@@ -588,7 +591,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
588
591
  // Long timeout because auth can wait for sync in certain cases.
589
592
  });
590
593
  this._authParams = _authParams;
591
- this._ctx = new import_context.Context({
594
+ this._ctx = new import_context2.Context({
592
595
  onError: (err) => {
593
596
  import_log4.log.catch(err, {}, {
594
597
  file: "auth.ts",
@@ -817,7 +820,7 @@ var Space = class Space2 {
817
820
  credential
818
821
  }, {
819
822
  file: "space.ts",
820
- line: 74,
823
+ line: 79,
821
824
  scope: this,
822
825
  callSite: (f, a) => f(...a)
823
826
  });
@@ -880,7 +883,7 @@ var Space = class Space2 {
880
883
  async open() {
881
884
  (0, import_log6.log)("opening...", {}, {
882
885
  file: "space.ts",
883
- line: 145,
886
+ line: 149,
884
887
  scope: this,
885
888
  callSite: (f, a) => f(...a)
886
889
  });
@@ -892,18 +895,17 @@ var Space = class Space2 {
892
895
  this._isOpen = true;
893
896
  (0, import_log6.log)("opened", {}, {
894
897
  file: "space.ts",
895
- line: 155,
898
+ line: 159,
896
899
  scope: this,
897
900
  callSite: (f, a) => f(...a)
898
901
  });
899
902
  }
900
903
  async close() {
901
- var _a;
902
904
  (0, import_log6.log)("closing...", {
903
905
  key: this._key
904
906
  }, {
905
907
  file: "space.ts",
906
- line: 160,
908
+ line: 164,
907
909
  scope: this,
908
910
  callSite: (f, a) => f(...a)
909
911
  });
@@ -911,37 +913,30 @@ var Space = class Space2 {
911
913
  return;
912
914
  }
913
915
  await this.protocol.stop();
914
- await ((_a = this._dataPipelineController) == null ? void 0 : _a.close());
915
916
  await this._controlPipeline.stop();
916
917
  this._isOpen = false;
917
918
  (0, import_log6.log)("closed", {}, {
918
919
  file: "space.ts",
919
- line: 171,
920
+ line: 174,
920
921
  scope: this,
921
922
  callSite: (f, a) => f(...a)
922
923
  });
923
924
  }
924
- async initDataPipeline(controller) {
925
+ // TODO(dmaretskyi): Make reusable.
926
+ async createDataPipeline({ start }) {
925
927
  (0, import_node_assert6.default)(this._isOpen, "Space must be open to initialize data pipeline.");
926
- (0, import_node_assert6.default)(!this._dataPipelineController, "Data pipeline already initialized.");
927
928
  (0, import_node_assert6.default)(!this._dataPipeline, "Data pipeline already initialized.");
928
- this._dataPipelineController = controller;
929
- await this._dataPipelineController.open({
930
- openPipeline: async (start) => {
931
- (0, import_node_assert6.default)(!this._dataPipeline, "Data pipeline already initialized.");
932
- this._dataPipeline = new Pipeline(start);
933
- if (this._dataFeed) {
934
- this._dataPipeline.setWriteFeed(this._dataFeed);
935
- }
936
- for (const feed of this._controlPipeline.spaceState.feeds.values()) {
937
- if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
938
- await this._dataPipeline.addFeed(await this._feedProvider(feed.key));
939
- }
940
- }
941
- await this._dataPipeline.start();
942
- return this._dataPipeline;
929
+ const pipeline = new Pipeline(start);
930
+ if (this._dataFeed) {
931
+ pipeline.setWriteFeed(this._dataFeed);
932
+ }
933
+ this._dataPipeline = pipeline;
934
+ for (const feed of this._controlPipeline.spaceState.feeds.values()) {
935
+ if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
936
+ await pipeline.addFeed(await this._feedProvider(feed.key));
943
937
  }
944
- });
938
+ }
939
+ return pipeline;
945
940
  }
946
941
  };
947
942
  __decorate3([
@@ -1220,11 +1215,49 @@ var import_credentials4 = require("@dxos/credentials");
1220
1215
  var import_debug4 = require("@dxos/debug");
1221
1216
  var import_credentials5 = require("@dxos/protocols/proto/dxos/halo/credentials");
1222
1217
  var spaceGenesis = async (keyring, signingContext, space) => {
1223
- var _a, _b;
1224
- const generator = new import_credentials4.CredentialGenerator(keyring, signingContext.identityKey, signingContext.deviceKey);
1218
+ var _a, _b, _c;
1225
1219
  const credentials = [
1226
- ...await generator.createSpaceGenesis(space.key, (_a = space.controlFeedKey) != null ? _a : (0, import_debug4.failUndefined)(), signingContext.profile),
1227
- await generator.createFeedAdmission(space.key, (_b = space.dataFeedKey) != null ? _b : (0, import_debug4.failUndefined)(), import_credentials5.AdmittedFeed.Designation.DATA)
1220
+ await (0, import_credentials4.createCredential)({
1221
+ signer: keyring,
1222
+ issuer: space.key,
1223
+ subject: space.key,
1224
+ assertion: {
1225
+ "@type": "dxos.halo.credentials.SpaceGenesis",
1226
+ spaceKey: space.key
1227
+ }
1228
+ }),
1229
+ await (0, import_credentials4.createCredential)({
1230
+ signer: keyring,
1231
+ issuer: space.key,
1232
+ subject: signingContext.identityKey,
1233
+ assertion: {
1234
+ "@type": "dxos.halo.credentials.SpaceMember",
1235
+ spaceKey: space.key,
1236
+ role: import_credentials5.SpaceMember.Role.ADMIN,
1237
+ profile: signingContext.profile,
1238
+ genesisFeedKey: (_a = space.controlFeedKey) != null ? _a : (0, import_debug4.failUndefined)()
1239
+ }
1240
+ }),
1241
+ await signingContext.credentialSigner.createCredential({
1242
+ subject: (_b = space.controlFeedKey) != null ? _b : (0, import_debug4.failUndefined)(),
1243
+ assertion: {
1244
+ "@type": "dxos.halo.credentials.AdmittedFeed",
1245
+ spaceKey: space.key,
1246
+ identityKey: signingContext.identityKey,
1247
+ deviceKey: signingContext.deviceKey,
1248
+ designation: import_credentials5.AdmittedFeed.Designation.CONTROL
1249
+ }
1250
+ }),
1251
+ await signingContext.credentialSigner.createCredential({
1252
+ subject: (_c = space.dataFeedKey) != null ? _c : (0, import_debug4.failUndefined)(),
1253
+ assertion: {
1254
+ "@type": "dxos.halo.credentials.AdmittedFeed",
1255
+ spaceKey: space.key,
1256
+ identityKey: signingContext.identityKey,
1257
+ deviceKey: signingContext.deviceKey,
1258
+ designation: import_credentials5.AdmittedFeed.Designation.DATA
1259
+ }
1260
+ })
1228
1261
  ];
1229
1262
  for (const credential of credentials) {
1230
1263
  await space.controlPipeline.writer.write({
@@ -1236,10 +1269,10 @@ var spaceGenesis = async (keyring, signingContext, space) => {
1236
1269
  return credentials;
1237
1270
  };
1238
1271
 
1239
- // packages/core/echo/echo-pipeline/src/space/data-pipeline-controller.ts
1272
+ // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
1240
1273
  var import_node_assert9 = __toESM(require("node:assert"));
1241
1274
  var import_async8 = require("@dxos/async");
1242
- var import_context2 = require("@dxos/context");
1275
+ var import_context3 = require("@dxos/context");
1243
1276
  var import_debug6 = require("@dxos/debug");
1244
1277
  var import_echo_db3 = require("@dxos/echo-db");
1245
1278
  var import_log11 = require("@dxos/log");
@@ -1269,56 +1302,53 @@ var DataServiceHost = class {
1269
1302
  objects
1270
1303
  }
1271
1304
  });
1272
- this._itemDemuxer.mutation.on(ctx, (mutation) => {
1305
+ this._itemDemuxer.mutation.on(ctx, (message) => {
1273
1306
  var _a;
1307
+ const { batch, meta } = message;
1274
1308
  (0, import_log9.log)("Object update", {
1275
- mutation
1309
+ batch,
1310
+ meta
1276
1311
  }, {
1277
1312
  file: "data-service-host.ts",
1278
- line: 49,
1313
+ line: 50,
1279
1314
  scope: this,
1280
1315
  callSite: (f, a) => f(...a)
1281
1316
  });
1282
1317
  const clientTag = this._clientTagMap.get([
1283
- mutation.meta.feedKey,
1284
- mutation.meta.seq
1318
+ message.meta.feedKey,
1319
+ message.meta.seq
1285
1320
  ]);
1286
- const batch = {
1287
- objects: [
1288
- {
1289
- ...mutation.data,
1290
- mutations: (_a = mutation.data.mutations) == null ? void 0 : _a.map((m, mutationIdx) => ({
1291
- ...m,
1292
- meta: mutation.meta
1293
- })),
1294
- meta: mutation.meta
1295
- }
1296
- ]
1297
- };
1321
+ (_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
1322
+ (0, import_echo_db.setMetadataOnObject)(object, meta);
1323
+ });
1298
1324
  if (clientTag) {
1299
1325
  (0, import_echo_db.tagMutationsInBatch)(batch, clientTag);
1300
1326
  }
1301
1327
  next({
1302
1328
  clientTag,
1303
- feedKey: mutation.meta.feedKey,
1304
- seq: mutation.meta.seq,
1329
+ feedKey: message.meta.feedKey,
1330
+ seq: message.meta.seq,
1305
1331
  batch
1306
1332
  });
1307
1333
  });
1308
1334
  });
1309
1335
  }
1310
1336
  async write(request) {
1311
- var _a, _b;
1337
+ var _a;
1312
1338
  (0, import_node_assert7.default)(this._writeStream, "Cannot write mutations in readonly mode");
1313
- (0, import_node_assert7.default)(((_a = request.batch.objects) == null ? void 0 : _a.length) === 1, "Only single object mutations are supported");
1314
1339
  const receipt = await this._writeStream.write({
1315
- object: {
1316
- ...request.batch.objects[0],
1317
- mutations: (_b = request.batch.objects[0].mutations) == null ? void 0 : _b.map((m) => ({
1318
- ...m,
1319
- meta: void 0
1320
- })),
1321
- meta: void 0
1340
+ batch: {
1341
+ objects: (_a = request.batch.objects) == null ? void 0 : _a.map((object) => {
1342
+ var _a2;
1343
+ return {
1344
+ ...object,
1345
+ mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((m) => ({
1346
+ ...m,
1347
+ meta: void 0
1348
+ })),
1349
+ meta: void 0
1350
+ };
1351
+ })
1322
1352
  }
1323
1353
  });
1324
1354
  if (request.clientTag) {
@@ -1521,7 +1551,7 @@ var DataServiceImpl = class {
1521
1551
  }
1522
1552
  };
1523
1553
 
1524
- // packages/core/echo/echo-pipeline/src/space/data-pipeline-controller.ts
1554
+ // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
1525
1555
  var __decorate7 = function(decorators, target, key, desc) {
1526
1556
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1527
1557
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1532,22 +1562,21 @@ var __decorate7 = function(decorators, target, key, desc) {
1532
1562
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1533
1563
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1534
1564
  };
1535
- var NoopDataPipelineController = class {
1536
- async open(context) {
1537
- }
1538
- async close() {
1539
- }
1540
- };
1541
1565
  var MESSAGES_PER_SNAPSHOT = 10;
1542
1566
  var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
1543
1567
  var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1544
- var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1568
+ var DISABLE_SNAPSHOT_CACHE = false;
1569
+ var DataPipeline = class DataPipeline2 {
1545
1570
  constructor(_params) {
1546
1571
  this._params = _params;
1547
- this._ctx = new import_context2.Context();
1572
+ this._ctx = new import_context3.Context();
1548
1573
  this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
1574
+ this._isOpen = false;
1549
1575
  this.onTimeframeReached = new import_async8.Event();
1550
1576
  }
1577
+ get isOpen() {
1578
+ return this._isOpen;
1579
+ }
1551
1580
  get pipelineState() {
1552
1581
  var _a;
1553
1582
  return (_a = this._pipeline) == null ? void 0 : _a.state;
@@ -1560,14 +1589,25 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1560
1589
  var _a;
1561
1590
  return snapshotTimeframeToStartingTimeframe((_a = this.snapshotTimeframe) != null ? _a : new import_timeframe3.Timeframe());
1562
1591
  }
1592
+ setTargetTimeframe(timeframe) {
1593
+ var _a;
1594
+ this._targetTimeframe = timeframe;
1595
+ (_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
1596
+ }
1563
1597
  async open(spaceContext) {
1564
1598
  var _a, _b, _c, _d;
1599
+ if (this._isOpen) {
1600
+ return;
1601
+ }
1565
1602
  this._spaceContext = spaceContext;
1566
1603
  if (this._params.snapshotId) {
1567
1604
  this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
1568
1605
  this._lastAutomaticSnapshotTimeframe = (_b = (_a = this._snapshot) == null ? void 0 : _a.timeframe) != null ? _b : new import_timeframe3.Timeframe();
1569
1606
  }
1570
1607
  this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
1608
+ if (this._targetTimeframe) {
1609
+ this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
1610
+ }
1571
1611
  const feedWriter = createMappedFeedWriter((data) => ({
1572
1612
  data
1573
1613
  }), (_c = this._pipeline.writer) != null ? _c : (0, import_debug6.failUndefined)());
@@ -1578,30 +1618,33 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1578
1618
  await this._consumePipeline();
1579
1619
  });
1580
1620
  this._createPeriodicSnapshots();
1621
+ this._isOpen = true;
1581
1622
  }
1582
1623
  _createPeriodicSnapshots() {
1583
1624
  this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
1584
1625
  await this._saveLatestTimeframe();
1585
1626
  });
1586
- this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
1587
- var _a, _b;
1588
- const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
1589
- if (!latestTimeframe) {
1590
- return;
1591
- }
1592
- if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
1593
- const snapshot = await this._saveSnapshot();
1594
- this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
1595
- (0, import_log11.log)("save", {
1596
- snapshot
1597
- }, {
1598
- file: "data-pipeline-controller.ts",
1599
- line: 149,
1600
- scope: this,
1601
- callSite: (f, a) => f(...a)
1602
- });
1603
- }
1604
- });
1627
+ if (!DISABLE_SNAPSHOT_CACHE) {
1628
+ this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
1629
+ var _a, _b;
1630
+ const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
1631
+ if (!latestTimeframe) {
1632
+ return;
1633
+ }
1634
+ if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
1635
+ const snapshot = await this._saveSnapshot();
1636
+ this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
1637
+ (0, import_log11.log)("save", {
1638
+ snapshot
1639
+ }, {
1640
+ file: "data-pipeline.ts",
1641
+ line: 161,
1642
+ scope: this,
1643
+ callSite: (f, a) => f(...a)
1644
+ });
1645
+ }
1646
+ });
1647
+ }
1605
1648
  }
1606
1649
  async _saveSnapshot() {
1607
1650
  const snapshot = await this.createSnapshot();
@@ -1610,21 +1653,41 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1610
1653
  return snapshot;
1611
1654
  }
1612
1655
  async _saveLatestTimeframe() {
1613
- var _a;
1656
+ var _a, _b;
1614
1657
  const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
1658
+ (0, import_log11.log)("save latest timeframe", {
1659
+ latestTimeframe,
1660
+ spaceKey: this._params.spaceKey
1661
+ }, {
1662
+ file: "data-pipeline.ts",
1663
+ line: 176,
1664
+ scope: this,
1665
+ callSite: (f, a) => f(...a)
1666
+ });
1615
1667
  if (latestTimeframe) {
1616
- await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, latestTimeframe);
1668
+ const newTimeframe = import_timeframe3.Timeframe.merge((_b = this._targetTimeframe) != null ? _b : new import_timeframe3.Timeframe(), latestTimeframe);
1669
+ await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
1617
1670
  }
1618
1671
  }
1619
1672
  async close() {
1620
1673
  var _a, _b, _c;
1674
+ if (!this._isOpen) {
1675
+ return;
1676
+ }
1677
+ (0, import_log11.log)("close", {}, {
1678
+ file: "data-pipeline.ts",
1679
+ line: 188,
1680
+ scope: this,
1681
+ callSite: (f, a) => f(...a)
1682
+ });
1683
+ this._isOpen = false;
1621
1684
  try {
1622
1685
  await this._saveLatestTimeframe();
1623
1686
  await this._saveSnapshot();
1624
1687
  } catch (err) {
1625
1688
  import_log11.log.catch(err, {}, {
1626
- file: "data-pipeline-controller.ts",
1627
- line: 173,
1689
+ file: "data-pipeline.ts",
1690
+ line: 195,
1628
1691
  scope: this,
1629
1692
  callSite: (f, a) => f(...a)
1630
1693
  });
@@ -1637,6 +1700,7 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1637
1700
  }
1638
1701
  createSnapshot() {
1639
1702
  var _a, _b;
1703
+ (0, import_node_assert9.default)(this.databaseBackend, "Database backend is not initialized.");
1640
1704
  return {
1641
1705
  spaceKey: this._params.spaceKey.asUint8Array(),
1642
1706
  timeframe: (_b = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe) != null ? _b : new import_timeframe3.Timeframe(),
@@ -1650,8 +1714,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1650
1714
  (0, import_log11.log)("processing message", {
1651
1715
  msg
1652
1716
  }, {
1653
- file: "data-pipeline-controller.ts",
1654
- line: 194,
1717
+ file: "data-pipeline.ts",
1718
+ line: 217,
1655
1719
  scope: this,
1656
1720
  callSite: (f, a) => f(...a)
1657
1721
  });
@@ -1662,15 +1726,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1662
1726
  import_log11.log.error("Could not find feed.", {
1663
1727
  feedKey
1664
1728
  }, {
1665
- file: "data-pipeline-controller.ts",
1666
- line: 200,
1729
+ file: "data-pipeline.ts",
1730
+ line: 223,
1667
1731
  scope: this,
1668
1732
  callSite: (f, a) => f(...a)
1669
1733
  });
1670
1734
  continue;
1671
1735
  }
1672
1736
  await this.databaseBackend.echoProcessor({
1673
- data: data.payload.data.object,
1737
+ batch: data.payload.data.batch,
1674
1738
  meta: {
1675
1739
  feedKey,
1676
1740
  seq,
@@ -1682,8 +1746,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1682
1746
  }
1683
1747
  } catch (err) {
1684
1748
  import_log11.log.catch(err, {}, {
1685
- file: "data-pipeline-controller.ts",
1686
- line: 216,
1749
+ file: "data-pipeline.ts",
1750
+ line: 239,
1687
1751
  scope: this,
1688
1752
  callSite: (f, a) => f(...a)
1689
1753
  });
@@ -1695,9 +1759,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
1695
1759
  await this._pipeline.state.waitUntilTimeframe(timeframe);
1696
1760
  }
1697
1761
  };
1698
- DataPipelineControllerImpl = __decorate7([
1762
+ __decorate7([
1763
+ import_async8.synchronized
1764
+ ], DataPipeline.prototype, "open", null);
1765
+ __decorate7([
1766
+ import_async8.synchronized
1767
+ ], DataPipeline.prototype, "close", null);
1768
+ DataPipeline = __decorate7([
1699
1769
  (0, import_async8.trackLeaks)("open", "close")
1700
- ], DataPipelineControllerImpl);
1770
+ ], DataPipeline);
1701
1771
  var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
1702
1772
  return snapshotTimeframe.map(([key, seq]) => [
1703
1773
  key,
@@ -1709,7 +1779,7 @@ var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
1709
1779
  AuthExtension,
1710
1780
  AuthStatus,
1711
1781
  DBError,
1712
- DataPipelineControllerImpl,
1782
+ DataPipeline,
1713
1783
  DataServiceHost,
1714
1784
  DataServiceImpl,
1715
1785
  DataServiceSubscriptions,
@@ -1721,7 +1791,6 @@ var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
1721
1791
  MOCK_AUTH_PROVIDER,
1722
1792
  MOCK_AUTH_VERIFIER,
1723
1793
  MetadataStore,
1724
- NoopDataPipelineController,
1725
1794
  Pipeline,
1726
1795
  STORAGE_VERSION,
1727
1796
  SnapshotManager,