@dxos/echo-pipeline 0.1.50 → 0.1.51-main.04cd027

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/lib/browser/{chunk-CSFWTLPD.mjs → chunk-KTQITKUX.mjs} +191 -60
  2. package/dist/lib/browser/chunk-KTQITKUX.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +1 -1
  4. package/dist/lib/browser/meta.json +1 -1
  5. package/dist/lib/browser/testing/index.mjs +4 -3
  6. package/dist/lib/browser/testing/index.mjs.map +3 -3
  7. package/dist/lib/node/index.cjs +197 -66
  8. package/dist/lib/node/index.cjs.map +3 -3
  9. package/dist/lib/node/meta.json +1 -1
  10. package/dist/lib/node/testing/index.cjs +207 -75
  11. package/dist/lib/node/testing/index.cjs.map +3 -3
  12. package/dist/types/src/dbhost/snapshot-manager.d.ts +2 -1
  13. package/dist/types/src/dbhost/snapshot-manager.d.ts.map +1 -1
  14. package/dist/types/src/metadata/metadata-store.d.ts +2 -1
  15. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  16. package/dist/types/src/pipeline/pipeline.d.ts +1 -0
  17. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  18. package/dist/types/src/space/control-pipeline.d.ts +9 -1
  19. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  20. package/dist/types/src/space/data-pipeline.d.ts +10 -0
  21. package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
  22. package/dist/types/src/space/space.d.ts +2 -2
  23. package/dist/types/src/space/space.d.ts.map +1 -1
  24. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  25. package/package.json +30 -30
  26. package/src/dbhost/snapshot-manager.ts +4 -3
  27. package/src/metadata/metadata-store.ts +12 -1
  28. package/src/pipeline/pipeline.ts +24 -0
  29. package/src/space/control-pipeline.test.ts +4 -0
  30. package/src/space/control-pipeline.ts +36 -1
  31. package/src/space/data-pipeline.ts +62 -8
  32. package/src/space/space-manager.ts +1 -1
  33. package/src/space/space.test.ts +2 -5
  34. package/src/space/space.ts +11 -7
  35. package/src/testing/test-agent-builder.ts +4 -1
  36. package/src/testing/util.ts +1 -1
  37. package/dist/lib/browser/chunk-CSFWTLPD.mjs.map +0 -7
@@ -234,6 +234,10 @@ var MetadataStore = class {
234
234
  }
235
235
  }
236
236
  _getSpace(spaceKey) {
237
+ var _a;
238
+ if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
239
+ return this._metadata.identity.haloSpace;
240
+ }
237
241
  const space = this.spaces.find((space2) => space2.key === spaceKey);
238
242
  (0, import_node_assert.default)(space, "Space not found");
239
243
  return space;
@@ -244,7 +248,7 @@ var MetadataStore = class {
244
248
  async clear() {
245
249
  (0, import_log.log)("clearing all metadata", {}, {
246
250
  file: "metadata-store.ts",
247
- line: 131,
251
+ line: 136,
248
252
  scope: this,
249
253
  callSite: (f, a) => f(...a)
250
254
  });
@@ -265,10 +269,14 @@ var MetadataStore = class {
265
269
  ((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
266
270
  await this._save();
267
271
  }
268
- async setSpaceLatestTimeframe(spaceKey, timeframe) {
272
+ async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
269
273
  this._getSpace(spaceKey).dataTimeframe = timeframe;
270
274
  await this._save();
271
275
  }
276
+ async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
277
+ this._getSpace(spaceKey).controlTimeframe = timeframe;
278
+ await this._save();
279
+ }
272
280
  async setCache(spaceKey, cache) {
273
281
  this._getSpace(spaceKey).cache = cache;
274
282
  await this._save();
@@ -447,6 +455,7 @@ var PipelineState = class {
447
455
  this._timeframeClock = _timeframeClock;
448
456
  this.timeframeUpdate = this._timeframeClock.update;
449
457
  this.stalled = new import_async3.Event();
458
+ this._startTimeframe = new import_timeframe2.Timeframe();
450
459
  }
451
460
  /**
452
461
  * Latest theoretical timeframe based on the last mutation in each feed.
@@ -459,6 +468,9 @@ var PipelineState = class {
459
468
  index: feed.length - 1
460
469
  })));
461
470
  }
471
+ get startTimeframe() {
472
+ return this._startTimeframe;
473
+ }
462
474
  get timeframe() {
463
475
  return this._timeframeClock.timeframe;
464
476
  }
@@ -492,7 +504,7 @@ var PipelineState = class {
492
504
  target: this.targetTimeframe
493
505
  }, {
494
506
  file: "pipeline.ts",
495
- line: 109,
507
+ line: 118,
496
508
  scope: this,
497
509
  callSite: (f, a) => f(...a)
498
510
  });
@@ -522,7 +534,7 @@ var PipelineState = class {
522
534
  dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
523
535
  }, {
524
536
  file: "pipeline.ts",
525
- line: 135,
537
+ line: 144,
526
538
  scope: this,
527
539
  callSite: (f, a) => f(...a)
528
540
  });
@@ -579,7 +591,7 @@ var Pipeline = class {
579
591
  async start() {
580
592
  (0, import_log3.log)("starting...", {}, {
581
593
  file: "pipeline.ts",
582
- line: 254,
594
+ line: 263,
583
595
  scope: this,
584
596
  callSite: (f, a) => f(...a)
585
597
  });
@@ -587,7 +599,7 @@ var Pipeline = class {
587
599
  await this._feedSetIterator.open();
588
600
  (0, import_log3.log)("started", {}, {
589
601
  file: "pipeline.ts",
590
- line: 257,
602
+ line: 266,
591
603
  scope: this,
592
604
  callSite: (f, a) => f(...a)
593
605
  });
@@ -597,7 +609,7 @@ var Pipeline = class {
597
609
  var _a;
598
610
  (0, import_log3.log)("stopping...", {}, {
599
611
  file: "pipeline.ts",
600
- line: 263,
612
+ line: 272,
601
613
  scope: this,
602
614
  callSite: (f, a) => f(...a)
603
615
  });
@@ -606,7 +618,7 @@ var Pipeline = class {
606
618
  await this._processingTrigger.wait();
607
619
  (0, import_log3.log)("stopped", {}, {
608
620
  file: "pipeline.ts",
609
- line: 267,
621
+ line: 276,
610
622
  scope: this,
611
623
  callSite: (f, a) => f(...a)
612
624
  });
@@ -617,7 +629,35 @@ var Pipeline = class {
617
629
  */
618
630
  async setCursor(timeframe) {
619
631
  (0, import_node_assert4.default)(!this._isStarted || this._isPaused, "Invalid state.");
632
+ this._state._startTimeframe = timeframe;
620
633
  this._timeframeClock.setTimeframe(timeframe);
634
+ for (const [key, seq] of timeframe.frames()) {
635
+ const feed = this._feeds.get(key);
636
+ if (!feed) {
637
+ throw new Error("Feed not found");
638
+ }
639
+ feed.undownload({
640
+ callback: () => (0, import_log3.log)("Undownloaded", {}, {
641
+ file: "pipeline.ts",
642
+ line: 297,
643
+ scope: this,
644
+ callSite: (f, a) => f(...a)
645
+ })
646
+ });
647
+ feed.download({
648
+ start: seq + 1,
649
+ linear: true
650
+ }).catch((err) => {
651
+ (0, import_log3.log)("failed to download feed", {
652
+ err
653
+ }, {
654
+ file: "pipeline.ts",
655
+ line: 300,
656
+ scope: this,
657
+ callSite: (f, a) => f(...a)
658
+ });
659
+ });
660
+ }
621
661
  if (this._feedSetIterator) {
622
662
  await this._feedSetIterator.close();
623
663
  await this._initIterator();
@@ -680,7 +720,7 @@ var Pipeline = class {
680
720
  this._feedSetIterator.stalled.on((iterator) => {
681
721
  import_log3.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
682
722
  file: "pipeline.ts",
683
- line: 357,
723
+ line: 381,
684
724
  scope: this,
685
725
  callSite: (f, a) => f(...a)
686
726
  });
@@ -805,10 +845,15 @@ var import_credentials = require("@dxos/credentials");
805
845
  var import_keys2 = require("@dxos/keys");
806
846
  var import_log5 = require("@dxos/log");
807
847
  var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
848
+ var import_timeframe3 = require("@dxos/timeframe");
808
849
  var import_util2 = require("@dxos/util");
850
+ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
809
851
  var ControlPipeline = class {
810
- constructor({ spaceKey, genesisFeed, feedProvider }) {
852
+ constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
853
+ this._lastTimeframeSaveTime = Date.now();
811
854
  this.onFeedAdmitted = new import_util2.Callback();
855
+ this._spaceKey = spaceKey;
856
+ this._metadata = metadataStore;
812
857
  this._pipeline = new Pipeline();
813
858
  void this._pipeline.addFeed(genesisFeed);
814
859
  this._spaceStateMachine = new import_credentials.SpaceStateMachine(spaceKey);
@@ -817,7 +862,7 @@ var ControlPipeline = class {
817
862
  key: info.key
818
863
  }, {
819
864
  file: "control-pipeline.ts",
820
- line: 39,
865
+ line: 51,
821
866
  scope: this,
822
867
  callSite: (f, a) => f(...a)
823
868
  });
@@ -828,7 +873,7 @@ var ControlPipeline = class {
828
873
  } catch (err) {
829
874
  import_log5.log.catch(err, {}, {
830
875
  file: "control-pipeline.ts",
831
- line: 47,
876
+ line: 59,
832
877
  scope: this,
833
878
  callSite: (f, a) => f(...a)
834
879
  });
@@ -851,7 +896,7 @@ var ControlPipeline = class {
851
896
  async start() {
852
897
  (0, import_log5.log)("starting...", {}, {
853
898
  file: "control-pipeline.ts",
854
- line: 71,
899
+ line: 83,
855
900
  scope: this,
856
901
  callSite: (f, a) => f(...a)
857
902
  });
@@ -863,7 +908,7 @@ var ControlPipeline = class {
863
908
  seq: msg.seq
864
909
  }, {
865
910
  file: "control-pipeline.ts",
866
- line: 76,
911
+ line: 88,
867
912
  scope: this,
868
913
  callSite: (f, a) => f(...a)
869
914
  });
@@ -874,16 +919,18 @@ var ControlPipeline = class {
874
919
  msg
875
920
  }, {
876
921
  file: "control-pipeline.ts",
877
- line: 83,
922
+ line: 95,
878
923
  scope: this,
879
924
  callSite: (f, a) => f(...a)
880
925
  });
926
+ } else {
927
+ await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
881
928
  }
882
929
  }
883
930
  } catch (err) {
884
931
  import_log5.log.catch(err, {}, {
885
932
  file: "control-pipeline.ts",
886
- line: 87,
933
+ line: 101,
887
934
  scope: this,
888
935
  callSite: (f, a) => f(...a)
889
936
  });
@@ -893,36 +940,59 @@ var ControlPipeline = class {
893
940
  await this._pipeline.start();
894
941
  (0, import_log5.log)("started", {}, {
895
942
  file: "control-pipeline.ts",
896
- line: 93,
943
+ line: 107,
897
944
  scope: this,
898
945
  callSite: (f, a) => f(...a)
899
946
  });
900
947
  }
948
+ async _noteTargetStateIfNeeded(timeframe) {
949
+ if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
950
+ this._lastTimeframeSaveTime = Date.now();
951
+ await this._saveTargetTimeframe(timeframe);
952
+ }
953
+ }
901
954
  async stop() {
902
955
  (0, import_log5.log)("stopping...", {}, {
903
956
  file: "control-pipeline.ts",
904
- line: 97,
957
+ line: 121,
905
958
  scope: this,
906
959
  callSite: (f, a) => f(...a)
907
960
  });
908
961
  await this._pipeline.stop();
962
+ await this._saveTargetTimeframe(this._pipeline.state.timeframe);
909
963
  (0, import_log5.log)("stopped", {}, {
910
964
  file: "control-pipeline.ts",
911
- line: 99,
965
+ line: 124,
912
966
  scope: this,
913
967
  callSite: (f, a) => f(...a)
914
968
  });
915
969
  }
970
+ async _saveTargetTimeframe(timeframe) {
971
+ var _a;
972
+ try {
973
+ const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
974
+ await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
975
+ this._targetTimeframe = newTimeframe;
976
+ } catch (err) {
977
+ (0, import_log5.log)(err, {}, {
978
+ file: "control-pipeline.ts",
979
+ line: 133,
980
+ scope: this,
981
+ callSite: (f, a) => f(...a)
982
+ });
983
+ }
984
+ }
916
985
  };
917
986
 
918
987
  // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
919
988
  var import_node_assert8 = __toESM(require("node:assert"));
920
989
  var import_async6 = require("@dxos/async");
921
- var import_context3 = require("@dxos/context");
990
+ var import_context4 = require("@dxos/context");
922
991
  var import_credentials3 = require("@dxos/credentials");
923
992
  var import_echo_db3 = require("@dxos/echo-db");
993
+ var import_errors2 = require("@dxos/errors");
924
994
  var import_log9 = require("@dxos/log");
925
- var import_timeframe3 = require("@dxos/timeframe");
995
+ var import_timeframe4 = require("@dxos/timeframe");
926
996
 
927
997
  // packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
928
998
  var import_node_assert6 = __toESM(require("node:assert"));
@@ -1071,6 +1141,7 @@ var DatabaseHost = class {
1071
1141
 
1072
1142
  // packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
1073
1143
  var import_async5 = require("@dxos/async");
1144
+ var import_context3 = require("@dxos/context");
1074
1145
  var import_debug4 = require("@dxos/debug");
1075
1146
  var import_log7 = require("@dxos/log");
1076
1147
  var import_protocols4 = require("@dxos/protocols");
@@ -1097,7 +1168,7 @@ var SnapshotManager = class SnapshotManager2 {
1097
1168
  snapshot
1098
1169
  }, {
1099
1170
  file: "snapshot-manager.ts",
1100
- line: 30,
1171
+ line: 31,
1101
1172
  scope: this,
1102
1173
  callSite: (f, a) => f(...a)
1103
1174
  });
@@ -1114,7 +1185,7 @@ var SnapshotManager = class SnapshotManager2 {
1114
1185
  data
1115
1186
  }, {
1116
1187
  file: "snapshot-manager.ts",
1117
- line: 40,
1188
+ line: 41,
1118
1189
  scope: this,
1119
1190
  callSite: (f, a) => f(...a)
1120
1191
  });
@@ -1132,12 +1203,12 @@ var SnapshotManager = class SnapshotManager2 {
1132
1203
  async close() {
1133
1204
  await this._objectSync.close();
1134
1205
  }
1135
- async load(id) {
1136
- const local = await this._snapshotStore.loadSnapshot(id);
1206
+ async load(ctx, id) {
1207
+ const local = await (0, import_context3.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
1137
1208
  if (local) {
1138
1209
  return local;
1139
1210
  }
1140
- const remote = await this._objectSync.download(id);
1211
+ const remote = await (0, import_context3.cancelWithContext)(ctx, this._objectSync.download(ctx, id));
1141
1212
  return import_protocols4.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(remote.payload.value);
1142
1213
  }
1143
1214
  async store(snapshot) {
@@ -1268,15 +1339,16 @@ var __decorate5 = function(decorators, target, key, desc) {
1268
1339
  };
1269
1340
  var MESSAGES_PER_SNAPSHOT = 10;
1270
1341
  var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
1271
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1342
+ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
1272
1343
  var DataPipeline = class DataPipeline2 {
1273
1344
  constructor(_params) {
1274
1345
  this._params = _params;
1275
- this._ctx = new import_context3.Context();
1276
- this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
1346
+ this._ctx = new import_context4.Context();
1347
+ this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
1277
1348
  this._isOpen = false;
1278
1349
  this._lastTimeframeSaveTime = 0;
1279
1350
  this._lastSnapshotSaveTime = 0;
1351
+ this._lastProcessedEpoch = -1;
1280
1352
  this.onNewEpoch = new import_async6.Event();
1281
1353
  }
1282
1354
  get isOpen() {
@@ -1300,17 +1372,10 @@ var DataPipeline = class DataPipeline2 {
1300
1372
  if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
1301
1373
  return;
1302
1374
  }
1303
- (0, import_log9.log)("new epoch", {
1304
- credential
1305
- }, {
1306
- file: "data-pipeline.ts",
1307
- line: 109,
1308
- scope: this,
1309
- callSite: (f, a) => f(...a)
1310
- });
1311
- await this._processEpoch(credential.subject.assertion);
1312
1375
  this.currentEpoch = credential;
1313
- this.onNewEpoch.emit(credential);
1376
+ if (this._isOpen) {
1377
+ await this._processEpochInSeparateTask(credential);
1378
+ }
1314
1379
  }
1315
1380
  };
1316
1381
  }
@@ -1349,7 +1414,7 @@ var DataPipeline = class DataPipeline2 {
1349
1414
  }
1350
1415
  (0, import_log9.log)("close", {}, {
1351
1416
  file: "data-pipeline.ts",
1352
- line: 161,
1417
+ line: 174,
1353
1418
  scope: this,
1354
1419
  callSite: (f, a) => f(...a)
1355
1420
  });
@@ -1364,7 +1429,7 @@ var DataPipeline = class DataPipeline2 {
1364
1429
  } catch (err) {
1365
1430
  import_log9.log.catch(err, {}, {
1366
1431
  file: "data-pipeline.ts",
1367
- line: 174,
1432
+ line: 187,
1368
1433
  scope: this,
1369
1434
  callSite: (f, a) => f(...a)
1370
1435
  });
@@ -1374,6 +1439,11 @@ var DataPipeline = class DataPipeline2 {
1374
1439
  await this._params.snapshotManager.close();
1375
1440
  }
1376
1441
  async _consumePipeline() {
1442
+ if (this.currentEpoch) {
1443
+ const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
1444
+ await this._processEpochInSeparateTask(this.currentEpoch);
1445
+ await waitForOneEpoch;
1446
+ }
1377
1447
  (0, import_node_assert8.default)(this._pipeline, "Pipeline is not initialized.");
1378
1448
  for await (const msg of this._pipeline.consume()) {
1379
1449
  const { feedKey, seq, data } = msg;
@@ -1382,7 +1452,7 @@ var DataPipeline = class DataPipeline2 {
1382
1452
  seq
1383
1453
  }, {
1384
1454
  file: "data-pipeline.ts",
1385
- line: 186,
1455
+ line: 205,
1386
1456
  scope: this,
1387
1457
  callSite: (f, a) => f(...a)
1388
1458
  });
@@ -1394,7 +1464,7 @@ var DataPipeline = class DataPipeline2 {
1394
1464
  feedKey
1395
1465
  }, {
1396
1466
  file: "data-pipeline.ts",
1397
- line: 192,
1467
+ line: 211,
1398
1468
  scope: this,
1399
1469
  callSite: (f, a) => f(...a)
1400
1470
  });
@@ -1409,12 +1479,22 @@ var DataPipeline = class DataPipeline2 {
1409
1479
  memberKey: feedInfo.assertion.identityKey
1410
1480
  }
1411
1481
  });
1482
+ import_log9.log.trace("dxos.echo.data-pipeline.processed", {
1483
+ feedKey: feedKey.toHex(),
1484
+ seq,
1485
+ spaceKey: this._params.spaceKey.toHex()
1486
+ }, {
1487
+ file: "data-pipeline.ts",
1488
+ line: 225,
1489
+ scope: this,
1490
+ callSite: (f, a) => f(...a)
1491
+ });
1412
1492
  await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1413
1493
  }
1414
1494
  } catch (err) {
1415
1495
  import_log9.log.catch(err, {}, {
1416
1496
  file: "data-pipeline.ts",
1417
- line: 210,
1497
+ line: 235,
1418
1498
  scope: this,
1419
1499
  callSite: (f, a) => f(...a)
1420
1500
  });
@@ -1431,8 +1511,8 @@ var DataPipeline = class DataPipeline2 {
1431
1511
  }
1432
1512
  async _saveTargetTimeframe(timeframe) {
1433
1513
  var _a;
1434
- const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
1435
- await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
1514
+ const newTimeframe = import_timeframe4.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe4.Timeframe(), timeframe);
1515
+ await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
1436
1516
  this._targetTimeframe = newTimeframe;
1437
1517
  }
1438
1518
  async _saveCache() {
@@ -1449,7 +1529,7 @@ var DataPipeline = class DataPipeline2 {
1449
1529
  } catch (err) {
1450
1530
  import_log9.log.warn("Failed to cache properties", err, {
1451
1531
  file: "data-pipeline.ts",
1452
- line: 244,
1532
+ line: 269,
1453
1533
  scope: this,
1454
1534
  callSite: (f, a) => f(...a)
1455
1535
  });
@@ -1457,7 +1537,7 @@ var DataPipeline = class DataPipeline2 {
1457
1537
  await this._params.metadataStore.setCache(this._params.spaceKey, cache);
1458
1538
  }
1459
1539
  async _noteTargetStateIfNeeded(timeframe) {
1460
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1540
+ if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
1461
1541
  this._lastTimeframeSaveTime = Date.now();
1462
1542
  await this._saveTargetTimeframe(timeframe);
1463
1543
  }
@@ -1465,24 +1545,68 @@ var DataPipeline = class DataPipeline2 {
1465
1545
  await this._saveCache();
1466
1546
  }
1467
1547
  }
1468
- async _processEpoch(epoch) {
1469
- (0, import_node_assert8.default)(this._isOpen);
1548
+ async _processEpochInSeparateTask(epoch) {
1549
+ var _a;
1550
+ if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
1551
+ return;
1552
+ }
1553
+ await ((_a = this._epochCtx) == null ? void 0 : _a.dispose());
1554
+ const ctx = new import_context4.Context({
1555
+ onError: (err) => {
1556
+ if (err instanceof import_errors2.CancelledError) {
1557
+ (0, import_log9.log)("Epoch processing cancelled.", {}, {
1558
+ file: "data-pipeline.ts",
1559
+ line: 301,
1560
+ scope: this,
1561
+ callSite: (f, a) => f(...a)
1562
+ });
1563
+ } else {
1564
+ import_log9.log.catch(err, {}, {
1565
+ file: "data-pipeline.ts",
1566
+ line: 303,
1567
+ scope: this,
1568
+ callSite: (f, a) => f(...a)
1569
+ });
1570
+ }
1571
+ }
1572
+ });
1573
+ this._epochCtx = ctx;
1574
+ (0, import_async6.scheduleTask)(ctx, async () => {
1575
+ if (!this._isOpen) {
1576
+ return;
1577
+ }
1578
+ (0, import_log9.log)("process epoch", {
1579
+ epoch
1580
+ }, {
1581
+ file: "data-pipeline.ts",
1582
+ line: 313,
1583
+ scope: this,
1584
+ callSite: (f, a) => f(...a)
1585
+ });
1586
+ await this._processEpoch(ctx, epoch.subject.assertion);
1587
+ this.appliedEpoch = epoch;
1588
+ this.onNewEpoch.emit(epoch);
1589
+ });
1590
+ }
1591
+ async _processEpoch(ctx, epoch) {
1592
+ (0, import_node_assert8.default)(this._isOpen, "Space is closed.");
1470
1593
  (0, import_node_assert8.default)(this._pipeline);
1594
+ this._lastProcessedEpoch = epoch.number;
1471
1595
  (0, import_log9.log)("Processing epoch", {
1472
1596
  epoch
1473
1597
  }, {
1474
1598
  file: "data-pipeline.ts",
1475
- line: 273,
1599
+ line: 327,
1476
1600
  scope: this,
1477
1601
  callSite: (f, a) => f(...a)
1478
1602
  });
1479
1603
  if (epoch.snapshotCid) {
1480
- const snapshot = await this._params.snapshotManager.load(epoch.snapshotCid);
1604
+ const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
1481
1605
  this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
1482
1606
  }
1483
1607
  (0, import_log9.log)("restarting pipeline for epoch", {}, {
1484
1608
  file: "data-pipeline.ts",
1485
- line: 281,
1609
+ line: 335,
1486
1610
  scope: this,
1487
1611
  callSite: (f, a) => f(...a)
1488
1612
  });
@@ -1556,20 +1680,26 @@ var Space = class Space2 {
1556
1680
  this._controlPipeline = new ControlPipeline({
1557
1681
  spaceKey: params.spaceKey,
1558
1682
  genesisFeed: params.genesisFeed,
1559
- feedProvider: params.feedProvider
1683
+ feedProvider: params.feedProvider,
1684
+ metadataStore: params.metadataStore
1560
1685
  });
1561
1686
  this._controlPipeline.onFeedAdmitted.set(async (info) => {
1687
+ const sparse = info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA;
1562
1688
  if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA) {
1563
1689
  await this._addFeedLock.executeSynchronized(async () => {
1564
1690
  if (this._dataPipeline.pipeline) {
1565
1691
  if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
1566
- return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key));
1692
+ return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
1693
+ sparse
1694
+ }));
1567
1695
  }
1568
1696
  }
1569
1697
  });
1570
1698
  }
1571
1699
  if (!info.key.equals(params.genesisFeed.key)) {
1572
- this.protocol.addFeed(await params.feedProvider(info.key));
1700
+ this.protocol.addFeed(await params.feedProvider(info.key, {
1701
+ sparse
1702
+ }));
1573
1703
  }
1574
1704
  });
1575
1705
  this._controlPipeline.onCredentialProcessed.set(async (credential) => {
@@ -1578,7 +1708,7 @@ var Space = class Space2 {
1578
1708
  credential
1579
1709
  }, {
1580
1710
  file: "space.ts",
1581
- line: 106,
1711
+ line: 110,
1582
1712
  scope: this,
1583
1713
  callSite: (f, a) => f(...a)
1584
1714
  });
@@ -1601,7 +1731,9 @@ var Space = class Space2 {
1601
1731
  await this._addFeedLock.executeSynchronized(async () => {
1602
1732
  for (const feed of this._controlPipeline.spaceState.feeds.values()) {
1603
1733
  if (feed.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
1604
- await pipeline.addFeed(await this._feedProvider(feed.key));
1734
+ await pipeline.addFeed(await this._feedProvider(feed.key, {
1735
+ sparse: true
1736
+ }));
1605
1737
  }
1606
1738
  }
1607
1739
  });
@@ -1668,7 +1800,7 @@ var Space = class Space2 {
1668
1800
  async open() {
1669
1801
  (0, import_log10.log)("opening...", {}, {
1670
1802
  file: "space.ts",
1671
- line: 208,
1803
+ line: 212,
1672
1804
  scope: this,
1673
1805
  callSite: (f, a) => f(...a)
1674
1806
  });
@@ -1678,10 +1810,11 @@ var Space = class Space2 {
1678
1810
  await this._controlPipeline.start();
1679
1811
  await this.protocol.start();
1680
1812
  this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
1813
+ await this._dataPipelineCredentialConsumer.open();
1681
1814
  this._isOpen = true;
1682
1815
  (0, import_log10.log)("opened", {}, {
1683
1816
  file: "space.ts",
1684
- line: 222,
1817
+ line: 228,
1685
1818
  scope: this,
1686
1819
  callSite: (f, a) => f(...a)
1687
1820
  });
@@ -1692,7 +1825,7 @@ var Space = class Space2 {
1692
1825
  key: this._key
1693
1826
  }, {
1694
1827
  file: "space.ts",
1695
- line: 227,
1828
+ line: 233,
1696
1829
  scope: this,
1697
1830
  callSite: (f, a) => f(...a)
1698
1831
  });
@@ -1707,7 +1840,7 @@ var Space = class Space2 {
1707
1840
  this._isOpen = false;
1708
1841
  (0, import_log10.log)("closed", {}, {
1709
1842
  file: "space.ts",
1710
- line: 241,
1843
+ line: 247,
1711
1844
  scope: this,
1712
1845
  callSite: (f, a) => f(...a)
1713
1846
  });
@@ -1715,14 +1848,12 @@ var Space = class Space2 {
1715
1848
  async initializeDataPipeline() {
1716
1849
  (0, import_log10.log)("initializeDataPipeline", {}, {
1717
1850
  file: "space.ts",
1718
- line: 246,
1851
+ line: 252,
1719
1852
  scope: this,
1720
1853
  callSite: (f, a) => f(...a)
1721
1854
  });
1722
1855
  (0, import_node_assert9.default)(this._isOpen, "Space must be open to initialize data pipeline.");
1723
1856
  await this._dataPipeline.open();
1724
- (0, import_node_assert9.default)(this._dataPipelineCredentialConsumer);
1725
- await this._dataPipelineCredentialConsumer.open();
1726
1857
  }
1727
1858
  };
1728
1859
  __decorate6([
@@ -1997,7 +2128,7 @@ var SpaceManager = class SpaceManager2 {
1997
2128
  spaceKey,
1998
2129
  protocol,
1999
2130
  genesisFeed,
2000
- feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
2131
+ feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
2001
2132
  modelFactory: this._modelFactory,
2002
2133
  metadataStore: this._metadataStore,
2003
2134
  snapshotManager,