@dxos/echo-pipeline 0.1.51-next.57ad9f3 → 0.1.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/lib/browser/{chunk-HQNN3RQA.mjs → chunk-AXGWKSM3.mjs} +184 -151
  2. package/dist/lib/browser/chunk-AXGWKSM3.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +1 -1
  4. package/dist/lib/browser/meta.json +1 -1
  5. package/dist/lib/browser/testing/index.mjs +12 -4
  6. package/dist/lib/browser/testing/index.mjs.map +3 -3
  7. package/dist/lib/node/index.cjs +175 -142
  8. package/dist/lib/node/index.cjs.map +3 -3
  9. package/dist/lib/node/meta.json +1 -1
  10. package/dist/lib/node/testing/index.cjs +231 -190
  11. package/dist/lib/node/testing/index.cjs.map +3 -3
  12. package/dist/types/src/dbhost/snapshot-manager.d.ts +5 -6
  13. package/dist/types/src/dbhost/snapshot-manager.d.ts.map +1 -1
  14. package/dist/types/src/pipeline/pipeline.d.ts +2 -0
  15. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  16. package/dist/types/src/space/data-pipeline.d.ts +7 -0
  17. package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
  18. package/dist/types/src/space/space-manager.d.ts +7 -1
  19. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  20. package/dist/types/src/space/space-protocol.d.ts +7 -2
  21. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  22. package/dist/types/src/space/space.d.ts +2 -2
  23. package/dist/types/src/space/space.d.ts.map +1 -1
  24. package/dist/types/src/testing/test-agent-builder.d.ts +3 -0
  25. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  26. package/package.json +30 -30
  27. package/src/dbhost/snapshot-manager.ts +27 -42
  28. package/src/pipeline/pipeline.ts +27 -0
  29. package/src/space/data-pipeline.ts +19 -2
  30. package/src/space/space-manager.ts +20 -3
  31. package/src/space/space-protocol.test.ts +4 -0
  32. package/src/space/space-protocol.ts +20 -2
  33. package/src/space/space.test.ts +1 -1
  34. package/src/space/space.ts +8 -5
  35. package/src/testing/test-agent-builder.ts +12 -1
  36. package/src/testing/util.ts +1 -1
  37. package/dist/lib/browser/chunk-HQNN3RQA.mjs.map +0 -7
@@ -342,6 +342,7 @@ var PipelineState = class {
342
342
  this._timeframeClock = _timeframeClock;
343
343
  this.timeframeUpdate = this._timeframeClock.update;
344
344
  this.stalled = new Event3();
345
+ this._startTimeframe = new Timeframe2();
345
346
  }
346
347
  /**
347
348
  * Latest theoretical timeframe based on the last mutation in each feed.
@@ -354,6 +355,9 @@ var PipelineState = class {
354
355
  index: feed.length - 1
355
356
  })));
356
357
  }
358
+ get startTimeframe() {
359
+ return this._startTimeframe;
360
+ }
357
361
  get timeframe() {
358
362
  return this._timeframeClock.timeframe;
359
363
  }
@@ -387,7 +391,7 @@ var PipelineState = class {
387
391
  target: this.targetTimeframe
388
392
  }, {
389
393
  file: "pipeline.ts",
390
- line: 109,
394
+ line: 118,
391
395
  scope: this,
392
396
  callSite: (f, a) => f(...a)
393
397
  });
@@ -417,7 +421,7 @@ var PipelineState = class {
417
421
  dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
418
422
  }, {
419
423
  file: "pipeline.ts",
420
- line: 135,
424
+ line: 144,
421
425
  scope: this,
422
426
  callSite: (f, a) => f(...a)
423
427
  });
@@ -459,6 +463,7 @@ var Pipeline = class {
459
463
  if (this._feedSetIterator) {
460
464
  await this._feedSetIterator.addFeed(feed);
461
465
  }
466
+ this._setFeedDownloadState(feed);
462
467
  }
463
468
  hasFeed(feedKey) {
464
469
  return this._feeds.has(feedKey);
@@ -474,7 +479,7 @@ var Pipeline = class {
474
479
  async start() {
475
480
  log4("starting...", {}, {
476
481
  file: "pipeline.ts",
477
- line: 254,
482
+ line: 264,
478
483
  scope: this,
479
484
  callSite: (f, a) => f(...a)
480
485
  });
@@ -482,7 +487,7 @@ var Pipeline = class {
482
487
  await this._feedSetIterator.open();
483
488
  log4("started", {}, {
484
489
  file: "pipeline.ts",
485
- line: 257,
490
+ line: 267,
486
491
  scope: this,
487
492
  callSite: (f, a) => f(...a)
488
493
  });
@@ -492,7 +497,7 @@ var Pipeline = class {
492
497
  var _a;
493
498
  log4("stopping...", {}, {
494
499
  file: "pipeline.ts",
495
- line: 263,
500
+ line: 273,
496
501
  scope: this,
497
502
  callSite: (f, a) => f(...a)
498
503
  });
@@ -501,7 +506,7 @@ var Pipeline = class {
501
506
  await this._processingTrigger.wait();
502
507
  log4("stopped", {}, {
503
508
  file: "pipeline.ts",
504
- line: 267,
509
+ line: 277,
505
510
  scope: this,
506
511
  callSite: (f, a) => f(...a)
507
512
  });
@@ -512,7 +517,11 @@ var Pipeline = class {
512
517
  */
513
518
  async setCursor(timeframe) {
514
519
  assert4(!this._isStarted || this._isPaused, "Invalid state.");
520
+ this._state._startTimeframe = timeframe;
515
521
  this._timeframeClock.setTimeframe(timeframe);
522
+ for (const feed of this._feeds.values()) {
523
+ this._setFeedDownloadState(feed);
524
+ }
516
525
  if (this._feedSetIterator) {
517
526
  await this._feedSetIterator.close();
518
527
  await this._initIterator();
@@ -567,6 +576,32 @@ var Pipeline = class {
567
576
  }
568
577
  this._isOpen = false;
569
578
  }
579
+ _setFeedDownloadState(feed) {
580
+ var _a;
581
+ const timeframe = this._state._startTimeframe;
582
+ const seq = (_a = timeframe.get(feed.key)) != null ? _a : 0;
583
+ feed.undownload({
584
+ callback: () => log4("Undownloaded", {}, {
585
+ file: "pipeline.ts",
586
+ line: 371,
587
+ scope: this,
588
+ callSite: (f, a) => f(...a)
589
+ })
590
+ });
591
+ feed.download({
592
+ start: seq + 1,
593
+ linear: true
594
+ }).catch((err) => {
595
+ log4("failed to download feed", {
596
+ err
597
+ }, {
598
+ file: "pipeline.ts",
599
+ line: 374,
600
+ scope: this,
601
+ callSite: (f, a) => f(...a)
602
+ });
603
+ });
604
+ }
570
605
  async _initIterator() {
571
606
  this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
572
607
  start: startAfter(this._timeframeClock.timeframe),
@@ -575,7 +610,7 @@ var Pipeline = class {
575
610
  this._feedSetIterator.stalled.on((iterator) => {
576
611
  log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
577
612
  file: "pipeline.ts",
578
- line: 357,
613
+ line: 384,
579
614
  scope: this,
580
615
  callSite: (f, a) => f(...a)
581
616
  });
@@ -837,9 +872,9 @@ var DatabaseHost = class {
837
872
  import { trackLeaks } from "@dxos/async";
838
873
  import { cancelWithContext } from "@dxos/context";
839
874
  import { timed as timed2 } from "@dxos/debug";
840
- import { log as log7 } from "@dxos/log";
875
+ import { PublicKey as PublicKey2 } from "@dxos/keys";
841
876
  import { schema as schema4 } from "@dxos/protocols";
842
- import { ObjectSync } from "@dxos/teleport-extension-object-sync";
877
+ import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
843
878
  var __decorate4 = function(decorators, target, key, desc) {
844
879
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
845
880
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -852,62 +887,32 @@ var __decorate4 = function(decorators, target, key, desc) {
852
887
  };
853
888
  var SnapshotManager = class SnapshotManager2 {
854
889
  // prettier-ignore
855
- constructor(_snapshotStore) {
890
+ constructor(_snapshotStore, _blobStore, _blobSync) {
856
891
  this._snapshotStore = _snapshotStore;
857
- this._objectSync = new ObjectSync({
858
- getObject: async (id) => {
859
- const snapshot = await this._snapshotStore.loadSnapshot(id);
860
- log7("getObject", {
861
- id,
862
- snapshot
863
- }, {
864
- file: "snapshot-manager.ts",
865
- line: 31,
866
- scope: this,
867
- callSite: (f, a) => f(...a)
868
- });
869
- if (!snapshot) {
870
- return void 0;
871
- }
872
- return {
873
- id,
874
- payload: schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encodeAsAny(snapshot)
875
- };
876
- },
877
- setObject: async (data) => {
878
- log7("setObject", {
879
- data
880
- }, {
881
- file: "snapshot-manager.ts",
882
- line: 41,
883
- scope: this,
884
- callSite: (f, a) => f(...a)
885
- });
886
- const snapshot = schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(data.payload.value);
887
- await this._snapshotStore.saveSnapshot(snapshot);
888
- }
889
- });
892
+ this._blobStore = _blobStore;
893
+ this._blobSync = _blobSync;
890
894
  }
891
- get objectSync() {
892
- return this._objectSync;
893
- }
894
- async open() {
895
- await this._objectSync.open();
896
- }
897
- async close() {
898
- await this._objectSync.close();
895
+ async _getBlob(blobId) {
896
+ const blob = await this._blobStore.get(blobId);
897
+ return schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(blob);
899
898
  }
900
899
  async load(ctx, id) {
901
- const local = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
902
- if (local) {
903
- return local;
900
+ const blobId = PublicKey2.fromHex(id).asUint8Array();
901
+ const blobMeta = await this._blobStore.getMeta(blobId);
902
+ if (blobMeta && blobMeta.state === BlobMeta.State.FULLY_PRESENT) {
903
+ return this._getBlob(blobId);
904
904
  }
905
- const remote = await cancelWithContext(ctx, this._objectSync.download(ctx, id));
906
- return schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(remote.payload.value);
905
+ const fallbackStore = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
906
+ if (fallbackStore) {
907
+ return fallbackStore;
908
+ }
909
+ await this._blobSync.download(ctx, blobId);
910
+ return this._getBlob(blobId);
907
911
  }
908
912
  async store(snapshot) {
909
- const id = await this._snapshotStore.saveSnapshot(snapshot);
910
- return id;
913
+ const { id } = await this._blobStore.set(schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").encode(snapshot));
914
+ await this._blobSync.notifyBlobAdded(id);
915
+ return PublicKey2.from(id).toHex();
911
916
  }
912
917
  };
913
918
  __decorate4([
@@ -965,18 +970,18 @@ var SnapshotStore = class {
965
970
  // packages/core/echo/echo-pipeline/src/dbhost/data-service.ts
966
971
  import assert7 from "@dxos/node-std/assert";
967
972
  import { raise } from "@dxos/debug";
968
- import { PublicKey as PublicKey2 } from "@dxos/keys";
969
- import { log as log8 } from "@dxos/log";
973
+ import { PublicKey as PublicKey3 } from "@dxos/keys";
974
+ import { log as log7 } from "@dxos/log";
970
975
  import { ComplexMap as ComplexMap3 } from "@dxos/util";
971
976
  var DataServiceSubscriptions = class {
972
977
  constructor() {
973
- this._spaces = new ComplexMap3(PublicKey2.hash);
978
+ this._spaces = new ComplexMap3(PublicKey3.hash);
974
979
  }
975
980
  clear() {
976
981
  this._spaces.clear();
977
982
  }
978
983
  registerSpace(spaceKey, host) {
979
- log8("Registering space", {
984
+ log7("Registering space", {
980
985
  spaceKey
981
986
  }, {
982
987
  file: "data-service.ts",
@@ -987,7 +992,7 @@ var DataServiceSubscriptions = class {
987
992
  this._spaces.set(spaceKey, host);
988
993
  }
989
994
  unregisterSpace(spaceKey) {
990
- log8("Unregistering space", {
995
+ log7("Unregistering space", {
991
996
  spaceKey
992
997
  }, {
993
998
  file: "data-service.ts",
@@ -1027,7 +1032,7 @@ import { Context as Context3 } from "@dxos/context";
1027
1032
  import { checkCredentialType } from "@dxos/credentials";
1028
1033
  import { getStateMachineFromItem, ItemManager } from "@dxos/echo-db";
1029
1034
  import { CancelledError } from "@dxos/errors";
1030
- import { log as log9 } from "@dxos/log";
1035
+ import { log as log8 } from "@dxos/log";
1031
1036
  import { Timeframe as Timeframe3 } from "@dxos/timeframe";
1032
1037
  var __decorate5 = function(decorators, target, key, desc) {
1033
1038
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -1114,9 +1119,9 @@ var DataPipeline = class DataPipeline2 {
1114
1119
  if (!this._isOpen) {
1115
1120
  return;
1116
1121
  }
1117
- log9("close", {}, {
1122
+ log8("close", {}, {
1118
1123
  file: "data-pipeline.ts",
1119
- line: 164,
1124
+ line: 174,
1120
1125
  scope: this,
1121
1126
  callSite: (f, a) => f(...a)
1122
1127
  });
@@ -1129,32 +1134,31 @@ var DataPipeline = class DataPipeline2 {
1129
1134
  await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1130
1135
  }
1131
1136
  } catch (err) {
1132
- log9.catch(err, {}, {
1137
+ log8.catch(err, {}, {
1133
1138
  file: "data-pipeline.ts",
1134
- line: 177,
1139
+ line: 187,
1135
1140
  scope: this,
1136
1141
  callSite: (f, a) => f(...a)
1137
1142
  });
1138
1143
  }
1139
1144
  await ((_b = this.databaseHost) == null ? void 0 : _b.close());
1140
1145
  await ((_c = this.itemManager) == null ? void 0 : _c.destroy());
1141
- await this._params.snapshotManager.close();
1142
1146
  }
1143
1147
  async _consumePipeline() {
1144
1148
  if (this.currentEpoch) {
1145
- const waitForOneEpoch = this.onNewEpoch.waitFor(() => true);
1149
+ const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
1146
1150
  await this._processEpochInSeparateTask(this.currentEpoch);
1147
1151
  await waitForOneEpoch;
1148
1152
  }
1149
1153
  assert8(this._pipeline, "Pipeline is not initialized.");
1150
1154
  for await (const msg of this._pipeline.consume()) {
1151
1155
  const { feedKey, seq, data } = msg;
1152
- log9("processing message", {
1156
+ log8("processing message", {
1153
1157
  feedKey,
1154
1158
  seq
1155
1159
  }, {
1156
1160
  file: "data-pipeline.ts",
1157
- line: 195,
1161
+ line: 204,
1158
1162
  scope: this,
1159
1163
  callSite: (f, a) => f(...a)
1160
1164
  });
@@ -1162,11 +1166,11 @@ var DataPipeline = class DataPipeline2 {
1162
1166
  if (data.payload.data) {
1163
1167
  const feedInfo = this._params.feedInfoProvider(feedKey);
1164
1168
  if (!feedInfo) {
1165
- log9.error("Could not find feed.", {
1169
+ log8.error("Could not find feed.", {
1166
1170
  feedKey
1167
1171
  }, {
1168
1172
  file: "data-pipeline.ts",
1169
- line: 201,
1173
+ line: 210,
1170
1174
  scope: this,
1171
1175
  callSite: (f, a) => f(...a)
1172
1176
  });
@@ -1181,12 +1185,22 @@ var DataPipeline = class DataPipeline2 {
1181
1185
  memberKey: feedInfo.assertion.identityKey
1182
1186
  }
1183
1187
  });
1188
+ log8.trace("dxos.echo.data-pipeline.processed", {
1189
+ feedKey: feedKey.toHex(),
1190
+ seq,
1191
+ spaceKey: this._params.spaceKey.toHex()
1192
+ }, {
1193
+ file: "data-pipeline.ts",
1194
+ line: 224,
1195
+ scope: this,
1196
+ callSite: (f, a) => f(...a)
1197
+ });
1184
1198
  await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1185
1199
  }
1186
1200
  } catch (err) {
1187
- log9.catch(err, {}, {
1201
+ log8.catch(err, {}, {
1188
1202
  file: "data-pipeline.ts",
1189
- line: 219,
1203
+ line: 234,
1190
1204
  scope: this,
1191
1205
  callSite: (f, a) => f(...a)
1192
1206
  });
@@ -1219,9 +1233,9 @@ var DataPipeline = class DataPipeline2 {
1219
1233
  cache.properties = (_a = getStateMachineFromItem(propertiesItem)) == null ? void 0 : _a.snapshot();
1220
1234
  }
1221
1235
  } catch (err) {
1222
- log9.warn("Failed to cache properties", err, {
1236
+ log8.warn("Failed to cache properties", err, {
1223
1237
  file: "data-pipeline.ts",
1224
- line: 253,
1238
+ line: 268,
1225
1239
  scope: this,
1226
1240
  callSite: (f, a) => f(...a)
1227
1241
  });
@@ -1246,16 +1260,16 @@ var DataPipeline = class DataPipeline2 {
1246
1260
  const ctx = new Context3({
1247
1261
  onError: (err) => {
1248
1262
  if (err instanceof CancelledError) {
1249
- log9("Epoch processing cancelled.", {}, {
1263
+ log8("Epoch processing cancelled.", {}, {
1250
1264
  file: "data-pipeline.ts",
1251
- line: 285,
1265
+ line: 300,
1252
1266
  scope: this,
1253
1267
  callSite: (f, a) => f(...a)
1254
1268
  });
1255
1269
  } else {
1256
- log9.catch(err, {}, {
1270
+ log8.catch(err, {}, {
1257
1271
  file: "data-pipeline.ts",
1258
- line: 287,
1272
+ line: 302,
1259
1273
  scope: this,
1260
1274
  callSite: (f, a) => f(...a)
1261
1275
  });
@@ -1267,15 +1281,16 @@ var DataPipeline = class DataPipeline2 {
1267
1281
  if (!this._isOpen) {
1268
1282
  return;
1269
1283
  }
1270
- log9("process epoch", {
1284
+ log8("process epoch", {
1271
1285
  epoch
1272
1286
  }, {
1273
1287
  file: "data-pipeline.ts",
1274
- line: 297,
1288
+ line: 312,
1275
1289
  scope: this,
1276
1290
  callSite: (f, a) => f(...a)
1277
1291
  });
1278
1292
  await this._processEpoch(ctx, epoch.subject.assertion);
1293
+ this.appliedEpoch = epoch;
1279
1294
  this.onNewEpoch.emit(epoch);
1280
1295
  });
1281
1296
  }
@@ -1283,11 +1298,11 @@ var DataPipeline = class DataPipeline2 {
1283
1298
  assert8(this._isOpen, "Space is closed.");
1284
1299
  assert8(this._pipeline);
1285
1300
  this._lastProcessedEpoch = epoch.number;
1286
- log9("Processing epoch", {
1301
+ log8("Processing epoch", {
1287
1302
  epoch
1288
1303
  }, {
1289
1304
  file: "data-pipeline.ts",
1290
- line: 309,
1305
+ line: 326,
1291
1306
  scope: this,
1292
1307
  callSite: (f, a) => f(...a)
1293
1308
  });
@@ -1295,9 +1310,9 @@ var DataPipeline = class DataPipeline2 {
1295
1310
  const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
1296
1311
  this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
1297
1312
  }
1298
- log9("restarting pipeline for epoch", {}, {
1313
+ log8("restarting pipeline for epoch", {}, {
1299
1314
  file: "data-pipeline.ts",
1300
- line: 317,
1315
+ line: 334,
1301
1316
  scope: this,
1302
1317
  callSite: (f, a) => f(...a)
1303
1318
  });
@@ -1347,14 +1362,14 @@ DataPipeline = __decorate5([
1347
1362
  // packages/core/echo/echo-pipeline/src/space/space.ts
1348
1363
  import assert9 from "@dxos/node-std/assert";
1349
1364
  import { Event as Event5, synchronized as synchronized4, trackLeaks as trackLeaks3, Lock } from "@dxos/async";
1350
- import { log as log11, logInfo } from "@dxos/log";
1365
+ import { log as log10, logInfo } from "@dxos/log";
1351
1366
  import { AdmittedFeed as AdmittedFeed2 } from "@dxos/protocols/proto/dxos/halo/credentials";
1352
1367
  import { Callback as Callback2 } from "@dxos/util";
1353
1368
 
1354
1369
  // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
1355
1370
  import { SpaceStateMachine } from "@dxos/credentials";
1356
- import { PublicKey as PublicKey3 } from "@dxos/keys";
1357
- import { log as log10 } from "@dxos/log";
1371
+ import { PublicKey as PublicKey4 } from "@dxos/keys";
1372
+ import { log as log9 } from "@dxos/log";
1358
1373
  import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
1359
1374
  import { Timeframe as Timeframe4 } from "@dxos/timeframe";
1360
1375
  import { Callback } from "@dxos/util";
@@ -1369,7 +1384,7 @@ var ControlPipeline = class {
1369
1384
  void this._pipeline.addFeed(genesisFeed);
1370
1385
  this._spaceStateMachine = new SpaceStateMachine(spaceKey);
1371
1386
  this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
1372
- log10("feed admitted", {
1387
+ log9("feed admitted", {
1373
1388
  key: info.key
1374
1389
  }, {
1375
1390
  file: "control-pipeline.ts",
@@ -1382,7 +1397,7 @@ var ControlPipeline = class {
1382
1397
  const feed = await feedProvider(info.key);
1383
1398
  await this._pipeline.addFeed(feed);
1384
1399
  } catch (err) {
1385
- log10.catch(err, {}, {
1400
+ log9.catch(err, {}, {
1386
1401
  file: "control-pipeline.ts",
1387
1402
  line: 59,
1388
1403
  scope: this,
@@ -1405,7 +1420,7 @@ var ControlPipeline = class {
1405
1420
  this._pipeline.setWriteFeed(feed);
1406
1421
  }
1407
1422
  async start() {
1408
- log10("starting...", {}, {
1423
+ log9("starting...", {}, {
1409
1424
  file: "control-pipeline.ts",
1410
1425
  line: 83,
1411
1426
  scope: this,
@@ -1414,7 +1429,7 @@ var ControlPipeline = class {
1414
1429
  setTimeout(async () => {
1415
1430
  for await (const msg of this._pipeline.consume()) {
1416
1431
  try {
1417
- log10("processing", {
1432
+ log9("processing", {
1418
1433
  key: msg.feedKey,
1419
1434
  seq: msg.seq
1420
1435
  }, {
@@ -1424,9 +1439,9 @@ var ControlPipeline = class {
1424
1439
  callSite: (f, a) => f(...a)
1425
1440
  });
1426
1441
  if (msg.data.payload.credential) {
1427
- const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey3.from(msg.feedKey));
1442
+ const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey4.from(msg.feedKey));
1428
1443
  if (!result) {
1429
- log10.warn("processing failed", {
1444
+ log9.warn("processing failed", {
1430
1445
  msg
1431
1446
  }, {
1432
1447
  file: "control-pipeline.ts",
@@ -1439,7 +1454,7 @@ var ControlPipeline = class {
1439
1454
  }
1440
1455
  }
1441
1456
  } catch (err) {
1442
- log10.catch(err, {}, {
1457
+ log9.catch(err, {}, {
1443
1458
  file: "control-pipeline.ts",
1444
1459
  line: 101,
1445
1460
  scope: this,
@@ -1449,7 +1464,7 @@ var ControlPipeline = class {
1449
1464
  }
1450
1465
  });
1451
1466
  await this._pipeline.start();
1452
- log10("started", {}, {
1467
+ log9("started", {}, {
1453
1468
  file: "control-pipeline.ts",
1454
1469
  line: 107,
1455
1470
  scope: this,
@@ -1463,7 +1478,7 @@ var ControlPipeline = class {
1463
1478
  }
1464
1479
  }
1465
1480
  async stop() {
1466
- log10("stopping...", {}, {
1481
+ log9("stopping...", {}, {
1467
1482
  file: "control-pipeline.ts",
1468
1483
  line: 121,
1469
1484
  scope: this,
@@ -1471,7 +1486,7 @@ var ControlPipeline = class {
1471
1486
  });
1472
1487
  await this._pipeline.stop();
1473
1488
  await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1474
- log10("stopped", {}, {
1489
+ log9("stopped", {}, {
1475
1490
  file: "control-pipeline.ts",
1476
1491
  line: 124,
1477
1492
  scope: this,
@@ -1485,7 +1500,7 @@ var ControlPipeline = class {
1485
1500
  await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
1486
1501
  this._targetTimeframe = newTimeframe;
1487
1502
  } catch (err) {
1488
- log10(err, {}, {
1503
+ log9(err, {}, {
1489
1504
  file: "control-pipeline.ts",
1490
1505
  line: 133,
1491
1506
  scope: this,
@@ -1526,26 +1541,31 @@ var Space = class Space2 {
1526
1541
  metadataStore: params.metadataStore
1527
1542
  });
1528
1543
  this._controlPipeline.onFeedAdmitted.set(async (info) => {
1544
+ const sparse = info.assertion.designation === AdmittedFeed2.Designation.DATA;
1529
1545
  if (info.assertion.designation === AdmittedFeed2.Designation.DATA) {
1530
1546
  await this._addFeedLock.executeSynchronized(async () => {
1531
1547
  if (this._dataPipeline.pipeline) {
1532
1548
  if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
1533
- return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key));
1549
+ return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
1550
+ sparse
1551
+ }));
1534
1552
  }
1535
1553
  }
1536
1554
  });
1537
1555
  }
1538
1556
  if (!info.key.equals(params.genesisFeed.key)) {
1539
- this.protocol.addFeed(await params.feedProvider(info.key));
1557
+ this.protocol.addFeed(await params.feedProvider(info.key, {
1558
+ sparse
1559
+ }));
1540
1560
  }
1541
1561
  });
1542
1562
  this._controlPipeline.onCredentialProcessed.set(async (credential) => {
1543
1563
  await this.onCredentialProcessed.callIfSet(credential);
1544
- log11("onCredentialProcessed", {
1564
+ log10("onCredentialProcessed", {
1545
1565
  credential
1546
1566
  }, {
1547
1567
  file: "space.ts",
1548
- line: 107,
1568
+ line: 110,
1549
1569
  scope: this,
1550
1570
  callSite: (f, a) => f(...a)
1551
1571
  });
@@ -1568,7 +1588,9 @@ var Space = class Space2 {
1568
1588
  await this._addFeedLock.executeSynchronized(async () => {
1569
1589
  for (const feed of this._controlPipeline.spaceState.feeds.values()) {
1570
1590
  if (feed.assertion.designation === AdmittedFeed2.Designation.DATA && !pipeline.hasFeed(feed.key)) {
1571
- await pipeline.addFeed(await this._feedProvider(feed.key));
1591
+ await pipeline.addFeed(await this._feedProvider(feed.key, {
1592
+ sparse: true
1593
+ }));
1572
1594
  }
1573
1595
  }
1574
1596
  });
@@ -1633,9 +1655,9 @@ var Space = class Space2 {
1633
1655
  // return this._dataPipeline?.getFeeds();
1634
1656
  // }
1635
1657
  async open() {
1636
- log11("opening...", {}, {
1658
+ log10("opening...", {}, {
1637
1659
  file: "space.ts",
1638
- line: 209,
1660
+ line: 212,
1639
1661
  scope: this,
1640
1662
  callSite: (f, a) => f(...a)
1641
1663
  });
@@ -1647,20 +1669,20 @@ var Space = class Space2 {
1647
1669
  this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
1648
1670
  await this._dataPipelineCredentialConsumer.open();
1649
1671
  this._isOpen = true;
1650
- log11("opened", {}, {
1672
+ log10("opened", {}, {
1651
1673
  file: "space.ts",
1652
- line: 225,
1674
+ line: 228,
1653
1675
  scope: this,
1654
1676
  callSite: (f, a) => f(...a)
1655
1677
  });
1656
1678
  }
1657
1679
  async close() {
1658
1680
  var _a;
1659
- log11("closing...", {
1681
+ log10("closing...", {
1660
1682
  key: this._key
1661
1683
  }, {
1662
1684
  file: "space.ts",
1663
- line: 230,
1685
+ line: 233,
1664
1686
  scope: this,
1665
1687
  callSite: (f, a) => f(...a)
1666
1688
  });
@@ -1673,17 +1695,17 @@ var Space = class Space2 {
1673
1695
  await this.protocol.stop();
1674
1696
  await this._controlPipeline.stop();
1675
1697
  this._isOpen = false;
1676
- log11("closed", {}, {
1698
+ log10("closed", {}, {
1677
1699
  file: "space.ts",
1678
- line: 244,
1700
+ line: 247,
1679
1701
  scope: this,
1680
1702
  callSite: (f, a) => f(...a)
1681
1703
  });
1682
1704
  }
1683
1705
  async initializeDataPipeline() {
1684
- log11("initializeDataPipeline", {}, {
1706
+ log10("initializeDataPipeline", {}, {
1685
1707
  file: "space.ts",
1686
- line: 249,
1708
+ line: 252,
1687
1709
  scope: this,
1688
1710
  callSite: (f, a) => f(...a)
1689
1711
  });
@@ -1709,10 +1731,11 @@ Space = __decorate6([
1709
1731
 
1710
1732
  // packages/core/echo/echo-pipeline/src/space/space-protocol.ts
1711
1733
  import { discoveryKey, sha256 } from "@dxos/crypto";
1712
- import { PublicKey as PublicKey4 } from "@dxos/keys";
1713
- import { log as log12, logInfo as logInfo2 } from "@dxos/log";
1734
+ import { PublicKey as PublicKey5 } from "@dxos/keys";
1735
+ import { log as log11, logInfo as logInfo2 } from "@dxos/log";
1714
1736
  import { MMSTTopology } from "@dxos/network-manager";
1715
1737
  import { Teleport } from "@dxos/teleport";
1738
+ import { BlobSync } from "@dxos/teleport-extension-object-sync";
1716
1739
  import { ReplicatorExtension } from "@dxos/teleport-extension-replicator";
1717
1740
  import { ComplexMap as ComplexMap4 } from "@dxos/util";
1718
1741
  var __decorate7 = function(decorators, target, key, desc) {
@@ -1728,14 +1751,17 @@ var __decorate7 = function(decorators, target, key, desc) {
1728
1751
  var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
1729
1752
  var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
1730
1753
  var SpaceProtocol = class {
1731
- constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure }) {
1754
+ constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
1732
1755
  this._feeds = /* @__PURE__ */ new Set();
1733
- this._sessions = new ComplexMap4(PublicKey4.hash);
1756
+ this._sessions = new ComplexMap4(PublicKey5.hash);
1734
1757
  this._networkManager = networkManager;
1735
1758
  this._swarmIdentity = swarmIdentity;
1736
1759
  this._onSessionAuth = onSessionAuth;
1737
1760
  this._onAuthFailure = onAuthFailure;
1738
- this._topic = PublicKey4.from(discoveryKey(sha256(topic.toHex())));
1761
+ this.blobSync = new BlobSync({
1762
+ blobStore
1763
+ });
1764
+ this._topic = PublicKey5.from(discoveryKey(sha256(topic.toHex())));
1739
1765
  }
1740
1766
  get sessions() {
1741
1767
  return this._sessions;
@@ -1748,11 +1774,11 @@ var SpaceProtocol = class {
1748
1774
  }
1749
1775
  // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
1750
1776
  addFeed(feed) {
1751
- log12("addFeed", {
1777
+ log11("addFeed", {
1752
1778
  key: feed.key
1753
1779
  }, {
1754
1780
  file: "space-protocol.ts",
1755
- line: 87,
1781
+ line: 94,
1756
1782
  scope: this,
1757
1783
  callSite: (f, a) => f(...a)
1758
1784
  });
@@ -1771,9 +1797,10 @@ var SpaceProtocol = class {
1771
1797
  maxPeers: 10,
1772
1798
  sampleSize: 20
1773
1799
  };
1774
- log12("starting...", {}, {
1800
+ await this.blobSync.open();
1801
+ log11("starting...", {}, {
1775
1802
  file: "space-protocol.ts",
1776
- line: 110,
1803
+ line: 119,
1777
1804
  scope: this,
1778
1805
  callSite: (f, a) => f(...a)
1779
1806
  });
@@ -1784,25 +1811,26 @@ var SpaceProtocol = class {
1784
1811
  topology: new MMSTTopology(topologyConfig),
1785
1812
  label: `Protocol swarm: ${this._topic}`
1786
1813
  });
1787
- log12("started", {}, {
1814
+ log11("started", {}, {
1788
1815
  file: "space-protocol.ts",
1789
- line: 119,
1816
+ line: 128,
1790
1817
  scope: this,
1791
1818
  callSite: (f, a) => f(...a)
1792
1819
  });
1793
1820
  }
1794
1821
  async stop() {
1822
+ await this.blobSync.close();
1795
1823
  if (this._connection) {
1796
- log12("stopping...", {}, {
1824
+ log11("stopping...", {}, {
1797
1825
  file: "space-protocol.ts",
1798
- line: 124,
1826
+ line: 135,
1799
1827
  scope: this,
1800
1828
  callSite: (f, a) => f(...a)
1801
1829
  });
1802
1830
  await this._connection.close();
1803
- log12("stopped", {}, {
1831
+ log11("stopped", {}, {
1804
1832
  file: "space-protocol.ts",
1805
- line: 126,
1833
+ line: 137,
1806
1834
  scope: this,
1807
1835
  callSite: (f, a) => f(...a)
1808
1836
  });
@@ -1814,7 +1842,8 @@ var SpaceProtocol = class {
1814
1842
  wireParams,
1815
1843
  swarmIdentity: this._swarmIdentity,
1816
1844
  onSessionAuth: this._onSessionAuth,
1817
- onAuthFailure: this._onAuthFailure
1845
+ onAuthFailure: this._onAuthFailure,
1846
+ blobSync: this.blobSync
1818
1847
  });
1819
1848
  this._sessions.set(wireParams.remotePeerId, session);
1820
1849
  for (const feed of this._feeds) {
@@ -1838,7 +1867,7 @@ var AuthStatus;
1838
1867
  })(AuthStatus || (AuthStatus = {}));
1839
1868
  var SpaceProtocolSession = class {
1840
1869
  // TODO(dmaretskyi): Allow to pass in extra extensions.
1841
- constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure }) {
1870
+ constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }) {
1842
1871
  // TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
1843
1872
  this.replicator = new ReplicatorExtension().setOptions({
1844
1873
  upload: true
@@ -1848,6 +1877,7 @@ var SpaceProtocolSession = class {
1848
1877
  this._swarmIdentity = swarmIdentity;
1849
1878
  this._onSessionAuth = onSessionAuth;
1850
1879
  this._onAuthFailure = onAuthFailure;
1880
+ this._blobSync = blobSync;
1851
1881
  this._teleport = new Teleport(wireParams);
1852
1882
  }
1853
1883
  get authStatus() {
@@ -1863,9 +1893,9 @@ var SpaceProtocolSession = class {
1863
1893
  verifier: this._swarmIdentity.credentialAuthenticator,
1864
1894
  onAuthSuccess: () => {
1865
1895
  var _a;
1866
- log12("Peer authenticated", {}, {
1896
+ log11("Peer authenticated", {}, {
1867
1897
  file: "space-protocol.ts",
1868
- line: 213,
1898
+ line: 230,
1869
1899
  scope: this,
1870
1900
  callSite: (f, a) => f(...a)
1871
1901
  });
@@ -1879,6 +1909,7 @@ var SpaceProtocolSession = class {
1879
1909
  }
1880
1910
  }));
1881
1911
  this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
1912
+ this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
1882
1913
  }
1883
1914
  async destroy() {
1884
1915
  await this._teleport.close();
@@ -1894,8 +1925,8 @@ __decorate7([
1894
1925
  // packages/core/echo/echo-pipeline/src/space/space-manager.ts
1895
1926
  import { synchronized as synchronized5, trackLeaks as trackLeaks4 } from "@dxos/async";
1896
1927
  import { failUndefined as failUndefined2 } from "@dxos/debug";
1897
- import { PublicKey as PublicKey5 } from "@dxos/keys";
1898
- import { log as log13 } from "@dxos/log";
1928
+ import { PublicKey as PublicKey6 } from "@dxos/keys";
1929
+ import { log as log12 } from "@dxos/log";
1899
1930
  import { trace } from "@dxos/protocols";
1900
1931
  import { ComplexMap as ComplexMap5 } from "@dxos/util";
1901
1932
  var __decorate8 = function(decorators, target, key, desc) {
@@ -1909,14 +1940,15 @@ var __decorate8 = function(decorators, target, key, desc) {
1909
1940
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1910
1941
  };
1911
1942
  var SpaceManager = class SpaceManager2 {
1912
- constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore }) {
1913
- this._spaces = new ComplexMap5(PublicKey5.hash);
1914
- this._instanceId = PublicKey5.random().toHex();
1943
+ constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
1944
+ this._spaces = new ComplexMap5(PublicKey6.hash);
1945
+ this._instanceId = PublicKey6.random().toHex();
1915
1946
  this._feedStore = feedStore;
1916
1947
  this._networkManager = networkManager;
1917
1948
  this._modelFactory = modelFactory;
1918
1949
  this._metadataStore = metadataStore;
1919
1950
  this._snapshotStore = snapshotStore;
1951
+ this._blobStore = blobStore;
1920
1952
  }
1921
1953
  // TODO(burdon): Remove.
1922
1954
  get spaces() {
@@ -1931,19 +1963,19 @@ var SpaceManager = class SpaceManager2 {
1931
1963
  }
1932
1964
  async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
1933
1965
  var _a;
1934
- log13.trace("dxos.echo.space-manager.construct-space", trace.begin({
1966
+ log12.trace("dxos.echo.space-manager.construct-space", trace.begin({
1935
1967
  id: this._instanceId
1936
1968
  }), {
1937
1969
  file: "space-manager.ts",
1938
- line: 81,
1970
+ line: 97,
1939
1971
  scope: this,
1940
1972
  callSite: (f, a) => f(...a)
1941
1973
  });
1942
- log13("constructing space...", {
1974
+ log12("constructing space...", {
1943
1975
  spaceKey: metadata.genesisFeedKey
1944
1976
  }, {
1945
1977
  file: "space-manager.ts",
1946
- line: 82,
1978
+ line: 98,
1947
1979
  scope: this,
1948
1980
  callSite: (f, a) => f(...a)
1949
1981
  });
@@ -1954,25 +1986,26 @@ var SpaceManager = class SpaceManager2 {
1954
1986
  swarmIdentity,
1955
1987
  networkManager: this._networkManager,
1956
1988
  onSessionAuth: onNetworkConnection,
1957
- onAuthFailure
1989
+ onAuthFailure,
1990
+ blobStore: this._blobStore
1958
1991
  });
1959
- const snapshotManager = new SnapshotManager(this._snapshotStore);
1992
+ const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
1960
1993
  const space = new Space({
1961
1994
  spaceKey,
1962
1995
  protocol,
1963
1996
  genesisFeed,
1964
- feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
1997
+ feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
1965
1998
  modelFactory: this._modelFactory,
1966
1999
  metadataStore: this._metadataStore,
1967
2000
  snapshotManager,
1968
2001
  memberKey
1969
2002
  });
1970
2003
  this._spaces.set(space.key, space);
1971
- log13.trace("dxos.echo.space-manager.construct-space", trace.end({
2004
+ log12.trace("dxos.echo.space-manager.construct-space", trace.end({
1972
2005
  id: this._instanceId
1973
2006
  }), {
1974
2007
  file: "space-manager.ts",
1975
- line: 108,
2008
+ line: 125,
1976
2009
  scope: this,
1977
2010
  callSite: (f, a) => f(...a)
1978
2011
  });
@@ -2015,4 +2048,4 @@ export {
2015
2048
  SpaceProtocolSession,
2016
2049
  SpaceManager
2017
2050
  };
2018
- //# sourceMappingURL=chunk-HQNN3RQA.mjs.map
2051
+ //# sourceMappingURL=chunk-AXGWKSM3.mjs.map