@dxos/echo-pipeline 0.7.4 → 0.7.5-feature-compute.4d9d99a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/lib/browser/{chunk-LZK5YFYE.mjs → chunk-QBMTPEMY.mjs} +111 -38
  2. package/dist/lib/browser/chunk-QBMTPEMY.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +171 -77
  4. package/dist/lib/browser/index.mjs.map +4 -4
  5. package/dist/lib/browser/meta.json +1 -1
  6. package/dist/lib/browser/testing/index.mjs +1 -1
  7. package/dist/lib/node/{chunk-MACQJ2EP.cjs → chunk-NPZ57MV5.cjs} +110 -40
  8. package/dist/lib/node/chunk-NPZ57MV5.cjs.map +7 -0
  9. package/dist/lib/node/index.cjs +184 -94
  10. package/dist/lib/node/index.cjs.map +4 -4
  11. package/dist/lib/node/meta.json +1 -1
  12. package/dist/lib/node/testing/index.cjs +10 -10
  13. package/dist/lib/node-esm/{chunk-JIZPSASG.mjs → chunk-OY5N3ZIV.mjs} +111 -38
  14. package/dist/lib/node-esm/chunk-OY5N3ZIV.mjs.map +7 -0
  15. package/dist/lib/node-esm/index.mjs +171 -77
  16. package/dist/lib/node-esm/index.mjs.map +4 -4
  17. package/dist/lib/node-esm/meta.json +1 -1
  18. package/dist/lib/node-esm/testing/index.mjs +1 -1
  19. package/dist/types/src/automerge/automerge-host.d.ts +5 -1
  20. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  21. package/dist/types/src/automerge/collection-synchronizer.d.ts +1 -0
  22. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  23. package/dist/types/src/automerge/echo-network-adapter.d.ts +1 -0
  24. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  25. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +1 -1
  26. package/dist/types/src/db-host/echo-host.d.ts +3 -2
  27. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  28. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  29. package/dist/types/src/edge/inflight-request-limiter.d.ts +24 -0
  30. package/dist/types/src/edge/inflight-request-limiter.d.ts.map +1 -0
  31. package/dist/types/src/pipeline/pipeline.d.ts +1 -0
  32. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  33. package/dist/types/src/space/control-pipeline.d.ts +9 -0
  34. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  35. package/dist/types/src/space/space-manager.d.ts +1 -0
  36. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  37. package/dist/types/tsconfig.tsbuildinfo +1 -0
  38. package/package.json +34 -34
  39. package/src/automerge/automerge-host.ts +49 -14
  40. package/src/automerge/collection-synchronizer.ts +8 -4
  41. package/src/automerge/echo-network-adapter.ts +7 -0
  42. package/src/automerge/mesh-echo-replicator.ts +2 -2
  43. package/src/db-host/echo-host.ts +4 -1
  44. package/src/edge/echo-edge-replicator.ts +34 -18
  45. package/src/edge/inflight-request-limiter.ts +69 -0
  46. package/src/pipeline/pipeline.ts +9 -1
  47. package/src/space/control-pipeline.ts +25 -2
  48. package/src/space/space-manager.ts +17 -1
  49. package/dist/lib/browser/chunk-LZK5YFYE.mjs.map +0 -7
  50. package/dist/lib/node/chunk-MACQJ2EP.cjs.map +0 -7
  51. package/dist/lib/node-esm/chunk-JIZPSASG.mjs.map +0 -7
@@ -726,10 +726,22 @@ var Pipeline = class {
726
726
  payload
727
727
  }), feed.createFeedWriter());
728
728
  }
729
+ retryMessage(message) {
730
+ invariant4(this._feedSetIterator, "Iterator not initialized.", {
731
+ F: __dxlog_file5,
732
+ L: 283,
733
+ S: this,
734
+ A: [
735
+ "this._feedSetIterator",
736
+ "'Iterator not initialized.'"
737
+ ]
738
+ });
739
+ this._feedSetIterator.reiterateBlock(message);
740
+ }
729
741
  async start() {
730
742
  invariant4(!this._isStarted, "Pipeline is already started.", {
731
743
  F: __dxlog_file5,
732
- L: 284,
744
+ L: 289,
733
745
  S: this,
734
746
  A: [
735
747
  "!this._isStarted",
@@ -738,7 +750,7 @@ var Pipeline = class {
738
750
  });
739
751
  log4("starting...", void 0, {
740
752
  F: __dxlog_file5,
741
- L: 285,
753
+ L: 290,
742
754
  S: this,
743
755
  C: (f, a) => f(...a)
744
756
  });
@@ -747,7 +759,7 @@ var Pipeline = class {
747
759
  this._isStarted = true;
748
760
  log4("started", void 0, {
749
761
  F: __dxlog_file5,
750
- L: 289,
762
+ L: 294,
751
763
  S: this,
752
764
  C: (f, a) => f(...a)
753
765
  });
@@ -760,7 +772,7 @@ var Pipeline = class {
760
772
  async stop() {
761
773
  log4("stopping...", void 0, {
762
774
  F: __dxlog_file5,
763
- L: 300,
775
+ L: 305,
764
776
  S: this,
765
777
  C: (f, a) => f(...a)
766
778
  });
@@ -774,14 +786,14 @@ var Pipeline = class {
774
786
  await this._state._ctx.dispose();
775
787
  this._state._ctx = new Context2(void 0, {
776
788
  F: __dxlog_file5,
777
- L: 309
789
+ L: 314
778
790
  });
779
791
  this._state._reachedTargetPromise = void 0;
780
792
  this._state._reachedTarget = false;
781
793
  this._isStarted = false;
782
794
  log4("stopped", void 0, {
783
795
  F: __dxlog_file5,
784
- L: 313,
796
+ L: 318,
785
797
  S: this,
786
798
  C: (f, a) => f(...a)
787
799
  });
@@ -793,7 +805,7 @@ var Pipeline = class {
793
805
  async setCursor(timeframe) {
794
806
  invariant4(!this._isStarted || this._isPaused, "Invalid state.", {
795
807
  F: __dxlog_file5,
796
- L: 322,
808
+ L: 327,
797
809
  S: this,
798
810
  A: [
799
811
  "!this._isStarted || this._isPaused",
@@ -822,7 +834,7 @@ var Pipeline = class {
822
834
  async unpause() {
823
835
  invariant4(this._isPaused, "Pipeline is not paused.", {
824
836
  F: __dxlog_file5,
825
- L: 351,
837
+ L: 356,
826
838
  S: this,
827
839
  A: [
828
840
  "this._isPaused",
@@ -842,7 +854,7 @@ var Pipeline = class {
842
854
  async *consume() {
843
855
  invariant4(!this._isBeingConsumed, "Pipeline is already being consumed.", {
844
856
  F: __dxlog_file5,
845
- L: 366,
857
+ L: 371,
846
858
  S: this,
847
859
  A: [
848
860
  "!this._isBeingConsumed",
@@ -852,7 +864,7 @@ var Pipeline = class {
852
864
  this._isBeingConsumed = true;
853
865
  invariant4(this._feedSetIterator, "Iterator not initialized.", {
854
866
  F: __dxlog_file5,
855
- L: 369,
867
+ L: 374,
856
868
  S: this,
857
869
  A: [
858
870
  "this._feedSetIterator",
@@ -866,7 +878,7 @@ var Pipeline = class {
866
878
  if (lastFeedSetIterator !== this._feedSetIterator) {
867
879
  invariant4(this._feedSetIterator, "Iterator not initialized.", {
868
880
  F: __dxlog_file5,
869
- L: 378,
881
+ L: 383,
870
882
  S: this,
871
883
  A: [
872
884
  "this._feedSetIterator",
@@ -901,7 +913,7 @@ var Pipeline = class {
901
913
  length: feed.length
902
914
  }, {
903
915
  F: __dxlog_file5,
904
- L: 407,
916
+ L: 412,
905
917
  S: this,
906
918
  C: (f, a) => f(...a)
907
919
  });
@@ -915,7 +927,7 @@ var Pipeline = class {
915
927
  data
916
928
  }, {
917
929
  F: __dxlog_file5,
918
- L: 412,
930
+ L: 417,
919
931
  S: this,
920
932
  C: (f, a) => f(...a)
921
933
  });
@@ -929,9 +941,12 @@ var Pipeline = class {
929
941
  stallTimeout: 1e3
930
942
  });
931
943
  this._feedSetIterator.stalled.on((iterator) => {
932
- log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
944
+ log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {
945
+ currentTimeframe: this._timeframeClock.timeframe,
946
+ targetTimeframe: this._state.targetTimeframe
947
+ }, {
933
948
  F: __dxlog_file5,
934
- L: 426,
949
+ L: 431,
935
950
  S: this,
936
951
  C: (f, a) => f(...a)
937
952
  });
@@ -1086,7 +1101,7 @@ import { log as log6 } from "@dxos/log";
1086
1101
  import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
1087
1102
  import { Timeframe as Timeframe3 } from "@dxos/timeframe";
1088
1103
  import { TimeSeriesCounter, TimeUsageCounter, trace } from "@dxos/tracing";
1089
- import { Callback, tracer } from "@dxos/util";
1104
+ import { Callback, ComplexSet, tracer } from "@dxos/util";
1090
1105
  function _ts_decorate4(decorators, target, key, desc) {
1091
1106
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1092
1107
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
@@ -1103,6 +1118,10 @@ var ControlPipeline = class {
1103
1118
  F: __dxlog_file7,
1104
1119
  L: 47
1105
1120
  });
1121
+ /**
1122
+ * Map to keep track of failed messages.
1123
+ */
1124
+ this._failedMessages = new ComplexSet((message) => `${message.feedKey.toHex()}:${message.seq}`);
1106
1125
  this._lastTimeframeSaveTime = Date.now();
1107
1126
  this.onFeedAdmitted = new Callback();
1108
1127
  this._usage = new TimeUsageCounter();
@@ -1121,7 +1140,7 @@ var ControlPipeline = class {
1121
1140
  key: info.key
1122
1141
  }, {
1123
1142
  F: __dxlog_file7,
1124
- L: 82,
1143
+ L: 89,
1125
1144
  S: this,
1126
1145
  C: (f, a) => f(...a)
1127
1146
  });
@@ -1135,7 +1154,7 @@ var ControlPipeline = class {
1135
1154
  } catch (err) {
1136
1155
  log6.catch(err, void 0, {
1137
1156
  F: __dxlog_file7,
1138
- L: 93,
1157
+ L: 100,
1139
1158
  S: this,
1140
1159
  C: (f, a) => f(...a)
1141
1160
  });
@@ -1167,7 +1186,7 @@ var ControlPipeline = class {
1167
1186
  tf: snapshot?.timeframe
1168
1187
  }, {
1169
1188
  F: __dxlog_file7,
1170
- L: 123,
1189
+ L: 130,
1171
1190
  S: this,
1172
1191
  C: (f, a) => f(...a)
1173
1192
  });
@@ -1176,20 +1195,20 @@ var ControlPipeline = class {
1176
1195
  }
1177
1196
  log6("starting...", void 0, {
1178
1197
  F: __dxlog_file7,
1179
- L: 128,
1198
+ L: 135,
1180
1199
  S: this,
1181
1200
  C: (f, a) => f(...a)
1182
1201
  });
1183
1202
  setTimeout(async () => {
1184
1203
  void this._consumePipeline(new Context4(void 0, {
1185
1204
  F: __dxlog_file7,
1186
- L: 130
1205
+ L: 137
1187
1206
  }));
1188
1207
  });
1189
1208
  await this._pipeline.start();
1190
1209
  log6("started", void 0, {
1191
1210
  F: __dxlog_file7,
1192
- L: 134,
1211
+ L: 141,
1193
1212
  S: this,
1194
1213
  C: (f, a) => f(...a)
1195
1214
  });
@@ -1206,7 +1225,7 @@ var ControlPipeline = class {
1206
1225
  message
1207
1226
  }, {
1208
1227
  F: __dxlog_file7,
1209
- L: 147,
1228
+ L: 154,
1210
1229
  S: this,
1211
1230
  C: (f, a) => f(...a)
1212
1231
  });
@@ -1228,7 +1247,7 @@ var ControlPipeline = class {
1228
1247
  snapshot: getSnapshotLoggerContext(snapshot)
1229
1248
  }, {
1230
1249
  F: __dxlog_file7,
1231
- L: 163,
1250
+ L: 170,
1232
1251
  S: this,
1233
1252
  C: (f, a) => f(...a)
1234
1253
  });
@@ -1243,7 +1262,7 @@ var ControlPipeline = class {
1243
1262
  } catch (err) {
1244
1263
  log6.catch(err, void 0, {
1245
1264
  F: __dxlog_file7,
1246
- L: 176,
1265
+ L: 183,
1247
1266
  S: this,
1248
1267
  C: (f, a) => f(...a)
1249
1268
  });
@@ -1257,7 +1276,7 @@ var ControlPipeline = class {
1257
1276
  seq: msg.seq
1258
1277
  }, {
1259
1278
  F: __dxlog_file7,
1260
- L: 186,
1279
+ L: 192,
1261
1280
  S: this,
1262
1281
  C: (f, a) => f(...a)
1263
1282
  });
@@ -1272,16 +1291,52 @@ var ControlPipeline = class {
1272
1291
  msg
1273
1292
  }, {
1274
1293
  F: __dxlog_file7,
1275
- L: 195,
1294
+ L: 201,
1276
1295
  S: this,
1277
1296
  C: (f, a) => f(...a)
1278
1297
  });
1298
+ this._retryMessage(msg);
1279
1299
  } else {
1280
1300
  await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1281
1301
  }
1282
1302
  this._snapshotTask.schedule();
1303
+ return result;
1283
1304
  }
1284
1305
  }
1306
+ /**
1307
+ * If it first failure, it will be retried once the pipeline is processed fully.
1308
+ * If it fails again, it will be ignored.
1309
+ */
1310
+ _retryMessage(message) {
1311
+ if (this._failedMessages.has({
1312
+ feedKey: message.feedKey,
1313
+ seq: message.seq
1314
+ })) {
1315
+ log6.warn("message processing failed twice", {
1316
+ message
1317
+ }, {
1318
+ F: __dxlog_file7,
1319
+ L: 218,
1320
+ S: this,
1321
+ C: (f, a) => f(...a)
1322
+ });
1323
+ return;
1324
+ }
1325
+ log6("message will be retried", {
1326
+ feedKey: message.feedKey.toHex(),
1327
+ seq: message.seq
1328
+ }, {
1329
+ F: __dxlog_file7,
1330
+ L: 222,
1331
+ S: this,
1332
+ C: (f, a) => f(...a)
1333
+ });
1334
+ this._failedMessages.add({
1335
+ feedKey: message.feedKey,
1336
+ seq: message.seq
1337
+ });
1338
+ this._pipeline.retryMessage(message);
1339
+ }
1285
1340
  async _noteTargetStateIfNeeded(timeframe) {
1286
1341
  if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1287
1342
  this._lastTimeframeSaveTime = Date.now();
@@ -1291,7 +1346,7 @@ var ControlPipeline = class {
1291
1346
  async stop() {
1292
1347
  log6("stopping...", void 0, {
1293
1348
  F: __dxlog_file7,
1294
- L: 215,
1349
+ L: 238,
1295
1350
  S: this,
1296
1351
  C: (f, a) => f(...a)
1297
1352
  });
@@ -1300,7 +1355,7 @@ var ControlPipeline = class {
1300
1355
  await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1301
1356
  log6("stopped", void 0, {
1302
1357
  F: __dxlog_file7,
1303
- L: 219,
1358
+ L: 242,
1304
1359
  S: this,
1305
1360
  C: (f, a) => f(...a)
1306
1361
  });
@@ -1313,7 +1368,7 @@ var ControlPipeline = class {
1313
1368
  } catch (err) {
1314
1369
  log6(err, void 0, {
1315
1370
  F: __dxlog_file7,
1316
- L: 228,
1371
+ L: 251,
1317
1372
  S: this,
1318
1373
  C: (f, a) => f(...a)
1319
1374
  });
@@ -1869,6 +1924,8 @@ _ts_decorate6([
1869
1924
 
1870
1925
  // packages/core/echo/echo-pipeline/src/space/space-manager.ts
1871
1926
  import { synchronized as synchronized4, trackLeaks as trackLeaks3, Trigger as Trigger2 } from "@dxos/async";
1927
+ import { parseAutomergeUrl } from "@dxos/automerge/automerge-repo";
1928
+ import { getCredentialAssertion } from "@dxos/credentials";
1872
1929
  import { failUndefined as failUndefined2 } from "@dxos/debug";
1873
1930
  import { PublicKey as PublicKey6 } from "@dxos/keys";
1874
1931
  import { log as log9 } from "@dxos/log";
@@ -1907,7 +1964,7 @@ var SpaceManager = class {
1907
1964
  id: this._instanceId
1908
1965
  }), {
1909
1966
  F: __dxlog_file11,
1910
- L: 100,
1967
+ L: 101,
1911
1968
  S: this,
1912
1969
  C: (f, a) => f(...a)
1913
1970
  });
@@ -1915,7 +1972,7 @@ var SpaceManager = class {
1915
1972
  spaceKey: metadata.genesisFeedKey
1916
1973
  }, {
1917
1974
  F: __dxlog_file11,
1918
- L: 101,
1975
+ L: 102,
1919
1976
  S: this,
1920
1977
  C: (f, a) => f(...a)
1921
1978
  });
@@ -1947,7 +2004,7 @@ var SpaceManager = class {
1947
2004
  id: this._instanceId
1948
2005
  }), {
1949
2006
  F: __dxlog_file11,
1950
- L: 131,
2007
+ L: 132,
1951
2008
  S: this,
1952
2009
  C: (f, a) => f(...a)
1953
2010
  });
@@ -1959,7 +2016,7 @@ var SpaceManager = class {
1959
2016
  id: this._instanceId
1960
2017
  }), {
1961
2018
  F: __dxlog_file11,
1962
- L: 137,
2019
+ L: 138,
1963
2020
  S: this,
1964
2021
  C: (f, a) => f(...a)
1965
2022
  });
@@ -1967,7 +2024,7 @@ var SpaceManager = class {
1967
2024
  spaceKey: params.spaceKey
1968
2025
  }, {
1969
2026
  F: __dxlog_file11,
1970
- L: 138,
2027
+ L: 139,
1971
2028
  S: this,
1972
2029
  C: (f, a) => f(...a)
1973
2030
  });
@@ -1995,7 +2052,7 @@ var SpaceManager = class {
1995
2052
  id: this._instanceId
1996
2053
  }), {
1997
2054
  F: __dxlog_file11,
1998
- L: 162,
2055
+ L: 163,
1999
2056
  S: this,
2000
2057
  C: (f, a) => f(...a)
2001
2058
  });
@@ -2006,7 +2063,7 @@ var SpaceManager = class {
2006
2063
  error: err
2007
2064
  }), {
2008
2065
  F: __dxlog_file11,
2009
- L: 165,
2066
+ L: 166,
2010
2067
  S: this,
2011
2068
  C: (f, a) => f(...a)
2012
2069
  });
@@ -2015,6 +2072,22 @@ var SpaceManager = class {
2015
2072
  await protocol.stop();
2016
2073
  }
2017
2074
  }
2075
+ findSpaceByRootDocumentId(documentId) {
2076
+ return [
2077
+ ...this._spaces.values()
2078
+ ].find((space) => {
2079
+ return space.spaceState.credentials.some((credential) => {
2080
+ const assertion = getCredentialAssertion(credential);
2081
+ if (assertion["@type"] !== "dxos.halo.credentials.Epoch") {
2082
+ return false;
2083
+ }
2084
+ if (!assertion?.automergeRoot) {
2085
+ return false;
2086
+ }
2087
+ return parseAutomergeUrl(assertion.automergeRoot).documentId === documentId;
2088
+ });
2089
+ });
2090
+ }
2018
2091
  };
2019
2092
  _ts_decorate7([
2020
2093
  synchronized4
@@ -2049,4 +2122,4 @@ export {
2049
2122
  SpaceProtocolSession,
2050
2123
  SpaceManager
2051
2124
  };
2052
- //# sourceMappingURL=chunk-LZK5YFYE.mjs.map
2125
+ //# sourceMappingURL=chunk-QBMTPEMY.mjs.map