@dxos/echo-pipeline 0.1.50 → 0.1.51-main.04cd027
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-CSFWTLPD.mjs → chunk-KTQITKUX.mjs} +191 -60
- package/dist/lib/browser/chunk-KTQITKUX.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +4 -3
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/index.cjs +197 -66
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +207 -75
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/dbhost/snapshot-manager.d.ts +2 -1
- package/dist/types/src/dbhost/snapshot-manager.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +2 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +9 -1
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +10 -0
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +2 -2
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/package.json +30 -30
- package/src/dbhost/snapshot-manager.ts +4 -3
- package/src/metadata/metadata-store.ts +12 -1
- package/src/pipeline/pipeline.ts +24 -0
- package/src/space/control-pipeline.test.ts +4 -0
- package/src/space/control-pipeline.ts +36 -1
- package/src/space/data-pipeline.ts +62 -8
- package/src/space/space-manager.ts +1 -1
- package/src/space/space.test.ts +2 -5
- package/src/space/space.ts +11 -7
- package/src/testing/test-agent-builder.ts +4 -1
- package/src/testing/util.ts +1 -1
- package/dist/lib/browser/chunk-CSFWTLPD.mjs.map +0 -7
package/dist/lib/node/index.cjs
CHANGED
|
@@ -234,6 +234,10 @@ var MetadataStore = class {
|
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
236
|
_getSpace(spaceKey) {
|
|
237
|
+
var _a;
|
|
238
|
+
if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
|
|
239
|
+
return this._metadata.identity.haloSpace;
|
|
240
|
+
}
|
|
237
241
|
const space = this.spaces.find((space2) => space2.key === spaceKey);
|
|
238
242
|
(0, import_node_assert.default)(space, "Space not found");
|
|
239
243
|
return space;
|
|
@@ -244,7 +248,7 @@ var MetadataStore = class {
|
|
|
244
248
|
async clear() {
|
|
245
249
|
(0, import_log.log)("clearing all metadata", {}, {
|
|
246
250
|
file: "metadata-store.ts",
|
|
247
|
-
line:
|
|
251
|
+
line: 136,
|
|
248
252
|
scope: this,
|
|
249
253
|
callSite: (f, a) => f(...a)
|
|
250
254
|
});
|
|
@@ -265,10 +269,14 @@ var MetadataStore = class {
|
|
|
265
269
|
((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
|
|
266
270
|
await this._save();
|
|
267
271
|
}
|
|
268
|
-
async
|
|
272
|
+
async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
|
|
269
273
|
this._getSpace(spaceKey).dataTimeframe = timeframe;
|
|
270
274
|
await this._save();
|
|
271
275
|
}
|
|
276
|
+
async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
|
|
277
|
+
this._getSpace(spaceKey).controlTimeframe = timeframe;
|
|
278
|
+
await this._save();
|
|
279
|
+
}
|
|
272
280
|
async setCache(spaceKey, cache) {
|
|
273
281
|
this._getSpace(spaceKey).cache = cache;
|
|
274
282
|
await this._save();
|
|
@@ -447,6 +455,7 @@ var PipelineState = class {
|
|
|
447
455
|
this._timeframeClock = _timeframeClock;
|
|
448
456
|
this.timeframeUpdate = this._timeframeClock.update;
|
|
449
457
|
this.stalled = new import_async3.Event();
|
|
458
|
+
this._startTimeframe = new import_timeframe2.Timeframe();
|
|
450
459
|
}
|
|
451
460
|
/**
|
|
452
461
|
* Latest theoretical timeframe based on the last mutation in each feed.
|
|
@@ -459,6 +468,9 @@ var PipelineState = class {
|
|
|
459
468
|
index: feed.length - 1
|
|
460
469
|
})));
|
|
461
470
|
}
|
|
471
|
+
get startTimeframe() {
|
|
472
|
+
return this._startTimeframe;
|
|
473
|
+
}
|
|
462
474
|
get timeframe() {
|
|
463
475
|
return this._timeframeClock.timeframe;
|
|
464
476
|
}
|
|
@@ -492,7 +504,7 @@ var PipelineState = class {
|
|
|
492
504
|
target: this.targetTimeframe
|
|
493
505
|
}, {
|
|
494
506
|
file: "pipeline.ts",
|
|
495
|
-
line:
|
|
507
|
+
line: 118,
|
|
496
508
|
scope: this,
|
|
497
509
|
callSite: (f, a) => f(...a)
|
|
498
510
|
});
|
|
@@ -522,7 +534,7 @@ var PipelineState = class {
|
|
|
522
534
|
dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
|
|
523
535
|
}, {
|
|
524
536
|
file: "pipeline.ts",
|
|
525
|
-
line:
|
|
537
|
+
line: 144,
|
|
526
538
|
scope: this,
|
|
527
539
|
callSite: (f, a) => f(...a)
|
|
528
540
|
});
|
|
@@ -579,7 +591,7 @@ var Pipeline = class {
|
|
|
579
591
|
async start() {
|
|
580
592
|
(0, import_log3.log)("starting...", {}, {
|
|
581
593
|
file: "pipeline.ts",
|
|
582
|
-
line:
|
|
594
|
+
line: 263,
|
|
583
595
|
scope: this,
|
|
584
596
|
callSite: (f, a) => f(...a)
|
|
585
597
|
});
|
|
@@ -587,7 +599,7 @@ var Pipeline = class {
|
|
|
587
599
|
await this._feedSetIterator.open();
|
|
588
600
|
(0, import_log3.log)("started", {}, {
|
|
589
601
|
file: "pipeline.ts",
|
|
590
|
-
line:
|
|
602
|
+
line: 266,
|
|
591
603
|
scope: this,
|
|
592
604
|
callSite: (f, a) => f(...a)
|
|
593
605
|
});
|
|
@@ -597,7 +609,7 @@ var Pipeline = class {
|
|
|
597
609
|
var _a;
|
|
598
610
|
(0, import_log3.log)("stopping...", {}, {
|
|
599
611
|
file: "pipeline.ts",
|
|
600
|
-
line:
|
|
612
|
+
line: 272,
|
|
601
613
|
scope: this,
|
|
602
614
|
callSite: (f, a) => f(...a)
|
|
603
615
|
});
|
|
@@ -606,7 +618,7 @@ var Pipeline = class {
|
|
|
606
618
|
await this._processingTrigger.wait();
|
|
607
619
|
(0, import_log3.log)("stopped", {}, {
|
|
608
620
|
file: "pipeline.ts",
|
|
609
|
-
line:
|
|
621
|
+
line: 276,
|
|
610
622
|
scope: this,
|
|
611
623
|
callSite: (f, a) => f(...a)
|
|
612
624
|
});
|
|
@@ -617,7 +629,35 @@ var Pipeline = class {
|
|
|
617
629
|
*/
|
|
618
630
|
async setCursor(timeframe) {
|
|
619
631
|
(0, import_node_assert4.default)(!this._isStarted || this._isPaused, "Invalid state.");
|
|
632
|
+
this._state._startTimeframe = timeframe;
|
|
620
633
|
this._timeframeClock.setTimeframe(timeframe);
|
|
634
|
+
for (const [key, seq] of timeframe.frames()) {
|
|
635
|
+
const feed = this._feeds.get(key);
|
|
636
|
+
if (!feed) {
|
|
637
|
+
throw new Error("Feed not found");
|
|
638
|
+
}
|
|
639
|
+
feed.undownload({
|
|
640
|
+
callback: () => (0, import_log3.log)("Undownloaded", {}, {
|
|
641
|
+
file: "pipeline.ts",
|
|
642
|
+
line: 297,
|
|
643
|
+
scope: this,
|
|
644
|
+
callSite: (f, a) => f(...a)
|
|
645
|
+
})
|
|
646
|
+
});
|
|
647
|
+
feed.download({
|
|
648
|
+
start: seq + 1,
|
|
649
|
+
linear: true
|
|
650
|
+
}).catch((err) => {
|
|
651
|
+
(0, import_log3.log)("failed to download feed", {
|
|
652
|
+
err
|
|
653
|
+
}, {
|
|
654
|
+
file: "pipeline.ts",
|
|
655
|
+
line: 300,
|
|
656
|
+
scope: this,
|
|
657
|
+
callSite: (f, a) => f(...a)
|
|
658
|
+
});
|
|
659
|
+
});
|
|
660
|
+
}
|
|
621
661
|
if (this._feedSetIterator) {
|
|
622
662
|
await this._feedSetIterator.close();
|
|
623
663
|
await this._initIterator();
|
|
@@ -680,7 +720,7 @@ var Pipeline = class {
|
|
|
680
720
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
681
721
|
import_log3.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
|
|
682
722
|
file: "pipeline.ts",
|
|
683
|
-
line:
|
|
723
|
+
line: 381,
|
|
684
724
|
scope: this,
|
|
685
725
|
callSite: (f, a) => f(...a)
|
|
686
726
|
});
|
|
@@ -805,10 +845,15 @@ var import_credentials = require("@dxos/credentials");
|
|
|
805
845
|
var import_keys2 = require("@dxos/keys");
|
|
806
846
|
var import_log5 = require("@dxos/log");
|
|
807
847
|
var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
848
|
+
var import_timeframe3 = require("@dxos/timeframe");
|
|
808
849
|
var import_util2 = require("@dxos/util");
|
|
850
|
+
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
809
851
|
var ControlPipeline = class {
|
|
810
|
-
constructor({ spaceKey, genesisFeed, feedProvider }) {
|
|
852
|
+
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
853
|
+
this._lastTimeframeSaveTime = Date.now();
|
|
811
854
|
this.onFeedAdmitted = new import_util2.Callback();
|
|
855
|
+
this._spaceKey = spaceKey;
|
|
856
|
+
this._metadata = metadataStore;
|
|
812
857
|
this._pipeline = new Pipeline();
|
|
813
858
|
void this._pipeline.addFeed(genesisFeed);
|
|
814
859
|
this._spaceStateMachine = new import_credentials.SpaceStateMachine(spaceKey);
|
|
@@ -817,7 +862,7 @@ var ControlPipeline = class {
|
|
|
817
862
|
key: info.key
|
|
818
863
|
}, {
|
|
819
864
|
file: "control-pipeline.ts",
|
|
820
|
-
line:
|
|
865
|
+
line: 51,
|
|
821
866
|
scope: this,
|
|
822
867
|
callSite: (f, a) => f(...a)
|
|
823
868
|
});
|
|
@@ -828,7 +873,7 @@ var ControlPipeline = class {
|
|
|
828
873
|
} catch (err) {
|
|
829
874
|
import_log5.log.catch(err, {}, {
|
|
830
875
|
file: "control-pipeline.ts",
|
|
831
|
-
line:
|
|
876
|
+
line: 59,
|
|
832
877
|
scope: this,
|
|
833
878
|
callSite: (f, a) => f(...a)
|
|
834
879
|
});
|
|
@@ -851,7 +896,7 @@ var ControlPipeline = class {
|
|
|
851
896
|
async start() {
|
|
852
897
|
(0, import_log5.log)("starting...", {}, {
|
|
853
898
|
file: "control-pipeline.ts",
|
|
854
|
-
line:
|
|
899
|
+
line: 83,
|
|
855
900
|
scope: this,
|
|
856
901
|
callSite: (f, a) => f(...a)
|
|
857
902
|
});
|
|
@@ -863,7 +908,7 @@ var ControlPipeline = class {
|
|
|
863
908
|
seq: msg.seq
|
|
864
909
|
}, {
|
|
865
910
|
file: "control-pipeline.ts",
|
|
866
|
-
line:
|
|
911
|
+
line: 88,
|
|
867
912
|
scope: this,
|
|
868
913
|
callSite: (f, a) => f(...a)
|
|
869
914
|
});
|
|
@@ -874,16 +919,18 @@ var ControlPipeline = class {
|
|
|
874
919
|
msg
|
|
875
920
|
}, {
|
|
876
921
|
file: "control-pipeline.ts",
|
|
877
|
-
line:
|
|
922
|
+
line: 95,
|
|
878
923
|
scope: this,
|
|
879
924
|
callSite: (f, a) => f(...a)
|
|
880
925
|
});
|
|
926
|
+
} else {
|
|
927
|
+
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
881
928
|
}
|
|
882
929
|
}
|
|
883
930
|
} catch (err) {
|
|
884
931
|
import_log5.log.catch(err, {}, {
|
|
885
932
|
file: "control-pipeline.ts",
|
|
886
|
-
line:
|
|
933
|
+
line: 101,
|
|
887
934
|
scope: this,
|
|
888
935
|
callSite: (f, a) => f(...a)
|
|
889
936
|
});
|
|
@@ -893,36 +940,59 @@ var ControlPipeline = class {
|
|
|
893
940
|
await this._pipeline.start();
|
|
894
941
|
(0, import_log5.log)("started", {}, {
|
|
895
942
|
file: "control-pipeline.ts",
|
|
896
|
-
line:
|
|
943
|
+
line: 107,
|
|
897
944
|
scope: this,
|
|
898
945
|
callSite: (f, a) => f(...a)
|
|
899
946
|
});
|
|
900
947
|
}
|
|
948
|
+
async _noteTargetStateIfNeeded(timeframe) {
|
|
949
|
+
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
|
|
950
|
+
this._lastTimeframeSaveTime = Date.now();
|
|
951
|
+
await this._saveTargetTimeframe(timeframe);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
901
954
|
async stop() {
|
|
902
955
|
(0, import_log5.log)("stopping...", {}, {
|
|
903
956
|
file: "control-pipeline.ts",
|
|
904
|
-
line:
|
|
957
|
+
line: 121,
|
|
905
958
|
scope: this,
|
|
906
959
|
callSite: (f, a) => f(...a)
|
|
907
960
|
});
|
|
908
961
|
await this._pipeline.stop();
|
|
962
|
+
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
909
963
|
(0, import_log5.log)("stopped", {}, {
|
|
910
964
|
file: "control-pipeline.ts",
|
|
911
|
-
line:
|
|
965
|
+
line: 124,
|
|
912
966
|
scope: this,
|
|
913
967
|
callSite: (f, a) => f(...a)
|
|
914
968
|
});
|
|
915
969
|
}
|
|
970
|
+
async _saveTargetTimeframe(timeframe) {
|
|
971
|
+
var _a;
|
|
972
|
+
try {
|
|
973
|
+
const newTimeframe = import_timeframe3.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe3.Timeframe(), timeframe);
|
|
974
|
+
await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
|
|
975
|
+
this._targetTimeframe = newTimeframe;
|
|
976
|
+
} catch (err) {
|
|
977
|
+
(0, import_log5.log)(err, {}, {
|
|
978
|
+
file: "control-pipeline.ts",
|
|
979
|
+
line: 133,
|
|
980
|
+
scope: this,
|
|
981
|
+
callSite: (f, a) => f(...a)
|
|
982
|
+
});
|
|
983
|
+
}
|
|
984
|
+
}
|
|
916
985
|
};
|
|
917
986
|
|
|
918
987
|
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
919
988
|
var import_node_assert8 = __toESM(require("node:assert"));
|
|
920
989
|
var import_async6 = require("@dxos/async");
|
|
921
|
-
var
|
|
990
|
+
var import_context4 = require("@dxos/context");
|
|
922
991
|
var import_credentials3 = require("@dxos/credentials");
|
|
923
992
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
993
|
+
var import_errors2 = require("@dxos/errors");
|
|
924
994
|
var import_log9 = require("@dxos/log");
|
|
925
|
-
var
|
|
995
|
+
var import_timeframe4 = require("@dxos/timeframe");
|
|
926
996
|
|
|
927
997
|
// packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
|
|
928
998
|
var import_node_assert6 = __toESM(require("node:assert"));
|
|
@@ -1071,6 +1141,7 @@ var DatabaseHost = class {
|
|
|
1071
1141
|
|
|
1072
1142
|
// packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
|
|
1073
1143
|
var import_async5 = require("@dxos/async");
|
|
1144
|
+
var import_context3 = require("@dxos/context");
|
|
1074
1145
|
var import_debug4 = require("@dxos/debug");
|
|
1075
1146
|
var import_log7 = require("@dxos/log");
|
|
1076
1147
|
var import_protocols4 = require("@dxos/protocols");
|
|
@@ -1097,7 +1168,7 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
1097
1168
|
snapshot
|
|
1098
1169
|
}, {
|
|
1099
1170
|
file: "snapshot-manager.ts",
|
|
1100
|
-
line:
|
|
1171
|
+
line: 31,
|
|
1101
1172
|
scope: this,
|
|
1102
1173
|
callSite: (f, a) => f(...a)
|
|
1103
1174
|
});
|
|
@@ -1114,7 +1185,7 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
1114
1185
|
data
|
|
1115
1186
|
}, {
|
|
1116
1187
|
file: "snapshot-manager.ts",
|
|
1117
|
-
line:
|
|
1188
|
+
line: 41,
|
|
1118
1189
|
scope: this,
|
|
1119
1190
|
callSite: (f, a) => f(...a)
|
|
1120
1191
|
});
|
|
@@ -1132,12 +1203,12 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
1132
1203
|
async close() {
|
|
1133
1204
|
await this._objectSync.close();
|
|
1134
1205
|
}
|
|
1135
|
-
async load(id) {
|
|
1136
|
-
const local = await this._snapshotStore.loadSnapshot(id);
|
|
1206
|
+
async load(ctx, id) {
|
|
1207
|
+
const local = await (0, import_context3.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
|
|
1137
1208
|
if (local) {
|
|
1138
1209
|
return local;
|
|
1139
1210
|
}
|
|
1140
|
-
const remote = await this._objectSync.download(id);
|
|
1211
|
+
const remote = await (0, import_context3.cancelWithContext)(ctx, this._objectSync.download(ctx, id));
|
|
1141
1212
|
return import_protocols4.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot").decode(remote.payload.value);
|
|
1142
1213
|
}
|
|
1143
1214
|
async store(snapshot) {
|
|
@@ -1268,15 +1339,16 @@ var __decorate5 = function(decorators, target, key, desc) {
|
|
|
1268
1339
|
};
|
|
1269
1340
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1270
1341
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1271
|
-
var
|
|
1342
|
+
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
1272
1343
|
var DataPipeline = class DataPipeline2 {
|
|
1273
1344
|
constructor(_params) {
|
|
1274
1345
|
this._params = _params;
|
|
1275
|
-
this._ctx = new
|
|
1276
|
-
this._lastAutomaticSnapshotTimeframe = new
|
|
1346
|
+
this._ctx = new import_context4.Context();
|
|
1347
|
+
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
1277
1348
|
this._isOpen = false;
|
|
1278
1349
|
this._lastTimeframeSaveTime = 0;
|
|
1279
1350
|
this._lastSnapshotSaveTime = 0;
|
|
1351
|
+
this._lastProcessedEpoch = -1;
|
|
1280
1352
|
this.onNewEpoch = new import_async6.Event();
|
|
1281
1353
|
}
|
|
1282
1354
|
get isOpen() {
|
|
@@ -1300,17 +1372,10 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1300
1372
|
if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
|
|
1301
1373
|
return;
|
|
1302
1374
|
}
|
|
1303
|
-
(0, import_log9.log)("new epoch", {
|
|
1304
|
-
credential
|
|
1305
|
-
}, {
|
|
1306
|
-
file: "data-pipeline.ts",
|
|
1307
|
-
line: 109,
|
|
1308
|
-
scope: this,
|
|
1309
|
-
callSite: (f, a) => f(...a)
|
|
1310
|
-
});
|
|
1311
|
-
await this._processEpoch(credential.subject.assertion);
|
|
1312
1375
|
this.currentEpoch = credential;
|
|
1313
|
-
this.
|
|
1376
|
+
if (this._isOpen) {
|
|
1377
|
+
await this._processEpochInSeparateTask(credential);
|
|
1378
|
+
}
|
|
1314
1379
|
}
|
|
1315
1380
|
};
|
|
1316
1381
|
}
|
|
@@ -1349,7 +1414,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1349
1414
|
}
|
|
1350
1415
|
(0, import_log9.log)("close", {}, {
|
|
1351
1416
|
file: "data-pipeline.ts",
|
|
1352
|
-
line:
|
|
1417
|
+
line: 174,
|
|
1353
1418
|
scope: this,
|
|
1354
1419
|
callSite: (f, a) => f(...a)
|
|
1355
1420
|
});
|
|
@@ -1364,7 +1429,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1364
1429
|
} catch (err) {
|
|
1365
1430
|
import_log9.log.catch(err, {}, {
|
|
1366
1431
|
file: "data-pipeline.ts",
|
|
1367
|
-
line:
|
|
1432
|
+
line: 187,
|
|
1368
1433
|
scope: this,
|
|
1369
1434
|
callSite: (f, a) => f(...a)
|
|
1370
1435
|
});
|
|
@@ -1374,6 +1439,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1374
1439
|
await this._params.snapshotManager.close();
|
|
1375
1440
|
}
|
|
1376
1441
|
async _consumePipeline() {
|
|
1442
|
+
if (this.currentEpoch) {
|
|
1443
|
+
const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
|
|
1444
|
+
await this._processEpochInSeparateTask(this.currentEpoch);
|
|
1445
|
+
await waitForOneEpoch;
|
|
1446
|
+
}
|
|
1377
1447
|
(0, import_node_assert8.default)(this._pipeline, "Pipeline is not initialized.");
|
|
1378
1448
|
for await (const msg of this._pipeline.consume()) {
|
|
1379
1449
|
const { feedKey, seq, data } = msg;
|
|
@@ -1382,7 +1452,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1382
1452
|
seq
|
|
1383
1453
|
}, {
|
|
1384
1454
|
file: "data-pipeline.ts",
|
|
1385
|
-
line:
|
|
1455
|
+
line: 205,
|
|
1386
1456
|
scope: this,
|
|
1387
1457
|
callSite: (f, a) => f(...a)
|
|
1388
1458
|
});
|
|
@@ -1394,7 +1464,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1394
1464
|
feedKey
|
|
1395
1465
|
}, {
|
|
1396
1466
|
file: "data-pipeline.ts",
|
|
1397
|
-
line:
|
|
1467
|
+
line: 211,
|
|
1398
1468
|
scope: this,
|
|
1399
1469
|
callSite: (f, a) => f(...a)
|
|
1400
1470
|
});
|
|
@@ -1409,12 +1479,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1409
1479
|
memberKey: feedInfo.assertion.identityKey
|
|
1410
1480
|
}
|
|
1411
1481
|
});
|
|
1482
|
+
import_log9.log.trace("dxos.echo.data-pipeline.processed", {
|
|
1483
|
+
feedKey: feedKey.toHex(),
|
|
1484
|
+
seq,
|
|
1485
|
+
spaceKey: this._params.spaceKey.toHex()
|
|
1486
|
+
}, {
|
|
1487
|
+
file: "data-pipeline.ts",
|
|
1488
|
+
line: 225,
|
|
1489
|
+
scope: this,
|
|
1490
|
+
callSite: (f, a) => f(...a)
|
|
1491
|
+
});
|
|
1412
1492
|
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1413
1493
|
}
|
|
1414
1494
|
} catch (err) {
|
|
1415
1495
|
import_log9.log.catch(err, {}, {
|
|
1416
1496
|
file: "data-pipeline.ts",
|
|
1417
|
-
line:
|
|
1497
|
+
line: 235,
|
|
1418
1498
|
scope: this,
|
|
1419
1499
|
callSite: (f, a) => f(...a)
|
|
1420
1500
|
});
|
|
@@ -1431,8 +1511,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1431
1511
|
}
|
|
1432
1512
|
async _saveTargetTimeframe(timeframe) {
|
|
1433
1513
|
var _a;
|
|
1434
|
-
const newTimeframe =
|
|
1435
|
-
await this._params.metadataStore.
|
|
1514
|
+
const newTimeframe = import_timeframe4.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe4.Timeframe(), timeframe);
|
|
1515
|
+
await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
|
|
1436
1516
|
this._targetTimeframe = newTimeframe;
|
|
1437
1517
|
}
|
|
1438
1518
|
async _saveCache() {
|
|
@@ -1449,7 +1529,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1449
1529
|
} catch (err) {
|
|
1450
1530
|
import_log9.log.warn("Failed to cache properties", err, {
|
|
1451
1531
|
file: "data-pipeline.ts",
|
|
1452
|
-
line:
|
|
1532
|
+
line: 269,
|
|
1453
1533
|
scope: this,
|
|
1454
1534
|
callSite: (f, a) => f(...a)
|
|
1455
1535
|
});
|
|
@@ -1457,7 +1537,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1457
1537
|
await this._params.metadataStore.setCache(this._params.spaceKey, cache);
|
|
1458
1538
|
}
|
|
1459
1539
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
1460
|
-
if (Date.now() - this._lastTimeframeSaveTime >
|
|
1540
|
+
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
|
|
1461
1541
|
this._lastTimeframeSaveTime = Date.now();
|
|
1462
1542
|
await this._saveTargetTimeframe(timeframe);
|
|
1463
1543
|
}
|
|
@@ -1465,24 +1545,68 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1465
1545
|
await this._saveCache();
|
|
1466
1546
|
}
|
|
1467
1547
|
}
|
|
1468
|
-
async
|
|
1469
|
-
|
|
1548
|
+
async _processEpochInSeparateTask(epoch) {
|
|
1549
|
+
var _a;
|
|
1550
|
+
if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
await ((_a = this._epochCtx) == null ? void 0 : _a.dispose());
|
|
1554
|
+
const ctx = new import_context4.Context({
|
|
1555
|
+
onError: (err) => {
|
|
1556
|
+
if (err instanceof import_errors2.CancelledError) {
|
|
1557
|
+
(0, import_log9.log)("Epoch processing cancelled.", {}, {
|
|
1558
|
+
file: "data-pipeline.ts",
|
|
1559
|
+
line: 301,
|
|
1560
|
+
scope: this,
|
|
1561
|
+
callSite: (f, a) => f(...a)
|
|
1562
|
+
});
|
|
1563
|
+
} else {
|
|
1564
|
+
import_log9.log.catch(err, {}, {
|
|
1565
|
+
file: "data-pipeline.ts",
|
|
1566
|
+
line: 303,
|
|
1567
|
+
scope: this,
|
|
1568
|
+
callSite: (f, a) => f(...a)
|
|
1569
|
+
});
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
});
|
|
1573
|
+
this._epochCtx = ctx;
|
|
1574
|
+
(0, import_async6.scheduleTask)(ctx, async () => {
|
|
1575
|
+
if (!this._isOpen) {
|
|
1576
|
+
return;
|
|
1577
|
+
}
|
|
1578
|
+
(0, import_log9.log)("process epoch", {
|
|
1579
|
+
epoch
|
|
1580
|
+
}, {
|
|
1581
|
+
file: "data-pipeline.ts",
|
|
1582
|
+
line: 313,
|
|
1583
|
+
scope: this,
|
|
1584
|
+
callSite: (f, a) => f(...a)
|
|
1585
|
+
});
|
|
1586
|
+
await this._processEpoch(ctx, epoch.subject.assertion);
|
|
1587
|
+
this.appliedEpoch = epoch;
|
|
1588
|
+
this.onNewEpoch.emit(epoch);
|
|
1589
|
+
});
|
|
1590
|
+
}
|
|
1591
|
+
async _processEpoch(ctx, epoch) {
|
|
1592
|
+
(0, import_node_assert8.default)(this._isOpen, "Space is closed.");
|
|
1470
1593
|
(0, import_node_assert8.default)(this._pipeline);
|
|
1594
|
+
this._lastProcessedEpoch = epoch.number;
|
|
1471
1595
|
(0, import_log9.log)("Processing epoch", {
|
|
1472
1596
|
epoch
|
|
1473
1597
|
}, {
|
|
1474
1598
|
file: "data-pipeline.ts",
|
|
1475
|
-
line:
|
|
1599
|
+
line: 327,
|
|
1476
1600
|
scope: this,
|
|
1477
1601
|
callSite: (f, a) => f(...a)
|
|
1478
1602
|
});
|
|
1479
1603
|
if (epoch.snapshotCid) {
|
|
1480
|
-
const snapshot = await this._params.snapshotManager.load(epoch.snapshotCid);
|
|
1604
|
+
const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
|
|
1481
1605
|
this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
|
|
1482
1606
|
}
|
|
1483
1607
|
(0, import_log9.log)("restarting pipeline for epoch", {}, {
|
|
1484
1608
|
file: "data-pipeline.ts",
|
|
1485
|
-
line:
|
|
1609
|
+
line: 335,
|
|
1486
1610
|
scope: this,
|
|
1487
1611
|
callSite: (f, a) => f(...a)
|
|
1488
1612
|
});
|
|
@@ -1556,20 +1680,26 @@ var Space = class Space2 {
|
|
|
1556
1680
|
this._controlPipeline = new ControlPipeline({
|
|
1557
1681
|
spaceKey: params.spaceKey,
|
|
1558
1682
|
genesisFeed: params.genesisFeed,
|
|
1559
|
-
feedProvider: params.feedProvider
|
|
1683
|
+
feedProvider: params.feedProvider,
|
|
1684
|
+
metadataStore: params.metadataStore
|
|
1560
1685
|
});
|
|
1561
1686
|
this._controlPipeline.onFeedAdmitted.set(async (info) => {
|
|
1687
|
+
const sparse = info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA;
|
|
1562
1688
|
if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA) {
|
|
1563
1689
|
await this._addFeedLock.executeSynchronized(async () => {
|
|
1564
1690
|
if (this._dataPipeline.pipeline) {
|
|
1565
1691
|
if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
|
|
1566
|
-
return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key
|
|
1692
|
+
return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
|
|
1693
|
+
sparse
|
|
1694
|
+
}));
|
|
1567
1695
|
}
|
|
1568
1696
|
}
|
|
1569
1697
|
});
|
|
1570
1698
|
}
|
|
1571
1699
|
if (!info.key.equals(params.genesisFeed.key)) {
|
|
1572
|
-
this.protocol.addFeed(await params.feedProvider(info.key
|
|
1700
|
+
this.protocol.addFeed(await params.feedProvider(info.key, {
|
|
1701
|
+
sparse
|
|
1702
|
+
}));
|
|
1573
1703
|
}
|
|
1574
1704
|
});
|
|
1575
1705
|
this._controlPipeline.onCredentialProcessed.set(async (credential) => {
|
|
@@ -1578,7 +1708,7 @@ var Space = class Space2 {
|
|
|
1578
1708
|
credential
|
|
1579
1709
|
}, {
|
|
1580
1710
|
file: "space.ts",
|
|
1581
|
-
line:
|
|
1711
|
+
line: 110,
|
|
1582
1712
|
scope: this,
|
|
1583
1713
|
callSite: (f, a) => f(...a)
|
|
1584
1714
|
});
|
|
@@ -1601,7 +1731,9 @@ var Space = class Space2 {
|
|
|
1601
1731
|
await this._addFeedLock.executeSynchronized(async () => {
|
|
1602
1732
|
for (const feed of this._controlPipeline.spaceState.feeds.values()) {
|
|
1603
1733
|
if (feed.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
|
|
1604
|
-
await pipeline.addFeed(await this._feedProvider(feed.key
|
|
1734
|
+
await pipeline.addFeed(await this._feedProvider(feed.key, {
|
|
1735
|
+
sparse: true
|
|
1736
|
+
}));
|
|
1605
1737
|
}
|
|
1606
1738
|
}
|
|
1607
1739
|
});
|
|
@@ -1668,7 +1800,7 @@ var Space = class Space2 {
|
|
|
1668
1800
|
async open() {
|
|
1669
1801
|
(0, import_log10.log)("opening...", {}, {
|
|
1670
1802
|
file: "space.ts",
|
|
1671
|
-
line:
|
|
1803
|
+
line: 212,
|
|
1672
1804
|
scope: this,
|
|
1673
1805
|
callSite: (f, a) => f(...a)
|
|
1674
1806
|
});
|
|
@@ -1678,10 +1810,11 @@ var Space = class Space2 {
|
|
|
1678
1810
|
await this._controlPipeline.start();
|
|
1679
1811
|
await this.protocol.start();
|
|
1680
1812
|
this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(this._dataPipeline.createCredentialProcessor());
|
|
1813
|
+
await this._dataPipelineCredentialConsumer.open();
|
|
1681
1814
|
this._isOpen = true;
|
|
1682
1815
|
(0, import_log10.log)("opened", {}, {
|
|
1683
1816
|
file: "space.ts",
|
|
1684
|
-
line:
|
|
1817
|
+
line: 228,
|
|
1685
1818
|
scope: this,
|
|
1686
1819
|
callSite: (f, a) => f(...a)
|
|
1687
1820
|
});
|
|
@@ -1692,7 +1825,7 @@ var Space = class Space2 {
|
|
|
1692
1825
|
key: this._key
|
|
1693
1826
|
}, {
|
|
1694
1827
|
file: "space.ts",
|
|
1695
|
-
line:
|
|
1828
|
+
line: 233,
|
|
1696
1829
|
scope: this,
|
|
1697
1830
|
callSite: (f, a) => f(...a)
|
|
1698
1831
|
});
|
|
@@ -1707,7 +1840,7 @@ var Space = class Space2 {
|
|
|
1707
1840
|
this._isOpen = false;
|
|
1708
1841
|
(0, import_log10.log)("closed", {}, {
|
|
1709
1842
|
file: "space.ts",
|
|
1710
|
-
line:
|
|
1843
|
+
line: 247,
|
|
1711
1844
|
scope: this,
|
|
1712
1845
|
callSite: (f, a) => f(...a)
|
|
1713
1846
|
});
|
|
@@ -1715,14 +1848,12 @@ var Space = class Space2 {
|
|
|
1715
1848
|
async initializeDataPipeline() {
|
|
1716
1849
|
(0, import_log10.log)("initializeDataPipeline", {}, {
|
|
1717
1850
|
file: "space.ts",
|
|
1718
|
-
line:
|
|
1851
|
+
line: 252,
|
|
1719
1852
|
scope: this,
|
|
1720
1853
|
callSite: (f, a) => f(...a)
|
|
1721
1854
|
});
|
|
1722
1855
|
(0, import_node_assert9.default)(this._isOpen, "Space must be open to initialize data pipeline.");
|
|
1723
1856
|
await this._dataPipeline.open();
|
|
1724
|
-
(0, import_node_assert9.default)(this._dataPipelineCredentialConsumer);
|
|
1725
|
-
await this._dataPipelineCredentialConsumer.open();
|
|
1726
1857
|
}
|
|
1727
1858
|
};
|
|
1728
1859
|
__decorate6([
|
|
@@ -1997,7 +2128,7 @@ var SpaceManager = class SpaceManager2 {
|
|
|
1997
2128
|
spaceKey,
|
|
1998
2129
|
protocol,
|
|
1999
2130
|
genesisFeed,
|
|
2000
|
-
feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
|
|
2131
|
+
feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
|
|
2001
2132
|
modelFactory: this._modelFactory,
|
|
2002
2133
|
metadataStore: this._metadataStore,
|
|
2003
2134
|
snapshotManager,
|