@dxos/echo-pipeline 0.1.31-next.ea8f0bc → 0.1.32-next.77d513e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-L44NG4N6.mjs → chunk-EM5HSJ7J.mjs} +192 -122
- package/dist/lib/browser/chunk-EM5HSJ7J.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +3 -5
- package/dist/lib/browser/index.mjs.map +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +34 -17
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/index.cjs +189 -120
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +175 -119
- package/dist/lib/node/testing/index.cjs.map +4 -4
- package/dist/types/src/dbhost/data-service-host.d.ts.map +1 -1
- package/dist/types/src/dbhost/database-backend.d.ts +2 -2
- package/dist/types/src/dbhost/database-backend.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +1 -2
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +10 -1
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/{data-pipeline-controller.d.ts → data-pipeline.d.ts} +10 -18
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -0
- package/dist/types/src/space/genesis.d.ts.map +1 -1
- package/dist/types/src/space/index.d.ts +1 -1
- package/dist/types/src/space/index.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +6 -4
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts +6 -5
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/util.d.ts +2 -2
- package/dist/types/src/testing/util.d.ts.map +1 -1
- package/package.json +29 -29
- package/src/dbhost/data-service-host.ts +19 -25
- package/src/dbhost/database-backend.ts +1 -1
- package/src/metadata/metadata-store.ts +1 -2
- package/src/pipeline/pipeline.test.ts +12 -4
- package/src/pipeline/pipeline.ts +18 -3
- package/src/space/{data-pipeline-controller.ts → data-pipeline.ts} +62 -39
- package/src/space/genesis.ts +47 -7
- package/src/space/index.ts +1 -1
- package/src/space/space-protocol.browser.test.ts +8 -6
- package/src/space/space-protocol.test.ts +8 -6
- package/src/space/space.ts +22 -23
- package/src/testing/database-test-rig.ts +1 -1
- package/src/testing/test-agent-builder.ts +23 -11
- package/src/testing/util.ts +13 -12
- package/src/tests/database-unit.test.ts +25 -0
- package/src/tests/database.test.ts +5 -5
- package/dist/lib/browser/chunk-L44NG4N6.mjs.map +0 -7
- package/dist/types/src/space/data-pipeline-controller.d.ts.map +0 -1
package/dist/lib/node/index.cjs
CHANGED
|
@@ -33,7 +33,7 @@ __export(src_exports, {
|
|
|
33
33
|
AuthExtension: () => AuthExtension,
|
|
34
34
|
AuthStatus: () => AuthStatus,
|
|
35
35
|
DBError: () => DBError,
|
|
36
|
-
|
|
36
|
+
DataPipeline: () => DataPipeline,
|
|
37
37
|
DataServiceHost: () => DataServiceHost,
|
|
38
38
|
DataServiceImpl: () => DataServiceImpl,
|
|
39
39
|
DataServiceSubscriptions: () => DataServiceSubscriptions,
|
|
@@ -45,7 +45,6 @@ __export(src_exports, {
|
|
|
45
45
|
MOCK_AUTH_PROVIDER: () => MOCK_AUTH_PROVIDER,
|
|
46
46
|
MOCK_AUTH_VERIFIER: () => MOCK_AUTH_VERIFIER,
|
|
47
47
|
MetadataStore: () => MetadataStore,
|
|
48
|
-
NoopDataPipelineController: () => NoopDataPipelineController,
|
|
49
48
|
Pipeline: () => Pipeline,
|
|
50
49
|
STORAGE_VERSION: () => STORAGE_VERSION,
|
|
51
50
|
SnapshotManager: () => SnapshotManager,
|
|
@@ -170,7 +169,7 @@ var MetadataStore = class {
|
|
|
170
169
|
size: dataSize
|
|
171
170
|
}, {
|
|
172
171
|
file: "metadata-store.ts",
|
|
173
|
-
line:
|
|
172
|
+
line: 67,
|
|
174
173
|
scope: this,
|
|
175
174
|
callSite: (f, a) => f(...a)
|
|
176
175
|
});
|
|
@@ -186,7 +185,7 @@ var MetadataStore = class {
|
|
|
186
185
|
err
|
|
187
186
|
}, {
|
|
188
187
|
file: "metadata-store.ts",
|
|
189
|
-
line:
|
|
188
|
+
line: 79,
|
|
190
189
|
scope: this,
|
|
191
190
|
callSite: (f, a) => f(...a)
|
|
192
191
|
});
|
|
@@ -211,7 +210,7 @@ var MetadataStore = class {
|
|
|
211
210
|
size: encoded.length
|
|
212
211
|
}, {
|
|
213
212
|
file: "metadata-store.ts",
|
|
214
|
-
line:
|
|
213
|
+
line: 102,
|
|
215
214
|
scope: this,
|
|
216
215
|
callSite: (f, a) => f(...a)
|
|
217
216
|
});
|
|
@@ -226,7 +225,7 @@ var MetadataStore = class {
|
|
|
226
225
|
async clear() {
|
|
227
226
|
(0, import_log.log)("clearing all metadata", {}, {
|
|
228
227
|
file: "metadata-store.ts",
|
|
229
|
-
line:
|
|
228
|
+
line: 115,
|
|
230
229
|
scope: this,
|
|
231
230
|
callSite: (f, a) => f(...a)
|
|
232
231
|
});
|
|
@@ -282,6 +281,7 @@ var fromBytesInt32 = (buf) => buf.readInt32LE(0);
|
|
|
282
281
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
283
282
|
var import_node_assert4 = __toESM(require("node:assert"));
|
|
284
283
|
var import_async3 = require("@dxos/async");
|
|
284
|
+
var import_context = require("@dxos/context");
|
|
285
285
|
var import_feed_store = require("@dxos/feed-store");
|
|
286
286
|
var import_keys = require("@dxos/keys");
|
|
287
287
|
var import_log3 = require("@dxos/log");
|
|
@@ -416,7 +416,7 @@ var PipelineState = class {
|
|
|
416
416
|
return this._timeframeClock.timeframe;
|
|
417
417
|
}
|
|
418
418
|
get targetTimeframe() {
|
|
419
|
-
return this._targetTimeframe ?
|
|
419
|
+
return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
|
|
420
420
|
}
|
|
421
421
|
async waitUntilTimeframe(target) {
|
|
422
422
|
await this._timeframeClock.waitUntilReached(target);
|
|
@@ -431,7 +431,7 @@ var PipelineState = class {
|
|
|
431
431
|
*
|
|
432
432
|
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
433
433
|
*/
|
|
434
|
-
async waitUntilReachedTargetTimeframe({ timeout } = {}) {
|
|
434
|
+
async waitUntilReachedTargetTimeframe({ ctx = new import_context.Context(), timeout, breakOnStall = true } = {}) {
|
|
435
435
|
var _a;
|
|
436
436
|
(0, import_log3.log)("waitUntilReachedTargetTimeframe", {
|
|
437
437
|
timeout,
|
|
@@ -439,7 +439,7 @@ var PipelineState = class {
|
|
|
439
439
|
target: this.targetTimeframe
|
|
440
440
|
}, {
|
|
441
441
|
file: "pipeline.ts",
|
|
442
|
-
line:
|
|
442
|
+
line: 96,
|
|
443
443
|
scope: this,
|
|
444
444
|
callSite: (f, a) => f(...a)
|
|
445
445
|
});
|
|
@@ -447,11 +447,14 @@ var PipelineState = class {
|
|
|
447
447
|
this._timeframeClock.update.waitForCondition(() => {
|
|
448
448
|
return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
|
|
449
449
|
}),
|
|
450
|
-
|
|
450
|
+
...breakOnStall ? [
|
|
451
|
+
this._iterator.stalled.discardParameter().waitForCount(1)
|
|
452
|
+
] : []
|
|
451
453
|
]);
|
|
452
454
|
let done = false;
|
|
453
455
|
if (timeout) {
|
|
454
456
|
return Promise.race([
|
|
457
|
+
(0, import_context.rejectOnDispose)(ctx),
|
|
455
458
|
this._reachedTargetPromise.then(() => {
|
|
456
459
|
done = true;
|
|
457
460
|
}),
|
|
@@ -466,7 +469,7 @@ var PipelineState = class {
|
|
|
466
469
|
dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
|
|
467
470
|
}, {
|
|
468
471
|
file: "pipeline.ts",
|
|
469
|
-
line:
|
|
472
|
+
line: 122,
|
|
470
473
|
scope: this,
|
|
471
474
|
callSite: (f, a) => f(...a)
|
|
472
475
|
});
|
|
@@ -492,7 +495,7 @@ var Pipeline = class {
|
|
|
492
495
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
493
496
|
import_log3.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, {}, {
|
|
494
497
|
file: "pipeline.ts",
|
|
495
|
-
line:
|
|
498
|
+
line: 198,
|
|
496
499
|
scope: this,
|
|
497
500
|
callSite: (f, a) => f(...a)
|
|
498
501
|
});
|
|
@@ -522,14 +525,14 @@ var Pipeline = class {
|
|
|
522
525
|
async start() {
|
|
523
526
|
(0, import_log3.log)("starting...", {}, {
|
|
524
527
|
file: "pipeline.ts",
|
|
525
|
-
line:
|
|
528
|
+
line: 233,
|
|
526
529
|
scope: this,
|
|
527
530
|
callSite: (f, a) => f(...a)
|
|
528
531
|
});
|
|
529
532
|
await this._feedSetIterator.open();
|
|
530
533
|
(0, import_log3.log)("started", {}, {
|
|
531
534
|
file: "pipeline.ts",
|
|
532
|
-
line:
|
|
535
|
+
line: 235,
|
|
533
536
|
scope: this,
|
|
534
537
|
callSite: (f, a) => f(...a)
|
|
535
538
|
});
|
|
@@ -537,7 +540,7 @@ var Pipeline = class {
|
|
|
537
540
|
async stop() {
|
|
538
541
|
(0, import_log3.log)("stopping...", {}, {
|
|
539
542
|
file: "pipeline.ts",
|
|
540
|
-
line:
|
|
543
|
+
line: 239,
|
|
541
544
|
scope: this,
|
|
542
545
|
callSite: (f, a) => f(...a)
|
|
543
546
|
});
|
|
@@ -545,7 +548,7 @@ var Pipeline = class {
|
|
|
545
548
|
await this._processingTrigger.wait();
|
|
546
549
|
(0, import_log3.log)("stopped", {}, {
|
|
547
550
|
file: "pipeline.ts",
|
|
548
|
-
line:
|
|
551
|
+
line: 242,
|
|
549
552
|
scope: this,
|
|
550
553
|
callSite: (f, a) => f(...a)
|
|
551
554
|
});
|
|
@@ -570,7 +573,7 @@ var Pipeline = class {
|
|
|
570
573
|
// packages/core/echo/echo-pipeline/src/space/auth.ts
|
|
571
574
|
var import_node_assert5 = __toESM(require("node:assert"));
|
|
572
575
|
var import_async4 = require("@dxos/async");
|
|
573
|
-
var
|
|
576
|
+
var import_context2 = require("@dxos/context");
|
|
574
577
|
var import_crypto = require("@dxos/crypto");
|
|
575
578
|
var import_log4 = require("@dxos/log");
|
|
576
579
|
var import_protocols3 = require("@dxos/protocols");
|
|
@@ -588,7 +591,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
588
591
|
// Long timeout because auth can wait for sync in certain cases.
|
|
589
592
|
});
|
|
590
593
|
this._authParams = _authParams;
|
|
591
|
-
this._ctx = new
|
|
594
|
+
this._ctx = new import_context2.Context({
|
|
592
595
|
onError: (err) => {
|
|
593
596
|
import_log4.log.catch(err, {}, {
|
|
594
597
|
file: "auth.ts",
|
|
@@ -817,7 +820,7 @@ var Space = class Space2 {
|
|
|
817
820
|
credential
|
|
818
821
|
}, {
|
|
819
822
|
file: "space.ts",
|
|
820
|
-
line:
|
|
823
|
+
line: 79,
|
|
821
824
|
scope: this,
|
|
822
825
|
callSite: (f, a) => f(...a)
|
|
823
826
|
});
|
|
@@ -880,7 +883,7 @@ var Space = class Space2 {
|
|
|
880
883
|
async open() {
|
|
881
884
|
(0, import_log6.log)("opening...", {}, {
|
|
882
885
|
file: "space.ts",
|
|
883
|
-
line:
|
|
886
|
+
line: 149,
|
|
884
887
|
scope: this,
|
|
885
888
|
callSite: (f, a) => f(...a)
|
|
886
889
|
});
|
|
@@ -892,18 +895,17 @@ var Space = class Space2 {
|
|
|
892
895
|
this._isOpen = true;
|
|
893
896
|
(0, import_log6.log)("opened", {}, {
|
|
894
897
|
file: "space.ts",
|
|
895
|
-
line:
|
|
898
|
+
line: 159,
|
|
896
899
|
scope: this,
|
|
897
900
|
callSite: (f, a) => f(...a)
|
|
898
901
|
});
|
|
899
902
|
}
|
|
900
903
|
async close() {
|
|
901
|
-
var _a;
|
|
902
904
|
(0, import_log6.log)("closing...", {
|
|
903
905
|
key: this._key
|
|
904
906
|
}, {
|
|
905
907
|
file: "space.ts",
|
|
906
|
-
line:
|
|
908
|
+
line: 164,
|
|
907
909
|
scope: this,
|
|
908
910
|
callSite: (f, a) => f(...a)
|
|
909
911
|
});
|
|
@@ -911,37 +913,30 @@ var Space = class Space2 {
|
|
|
911
913
|
return;
|
|
912
914
|
}
|
|
913
915
|
await this.protocol.stop();
|
|
914
|
-
await ((_a = this._dataPipelineController) == null ? void 0 : _a.close());
|
|
915
916
|
await this._controlPipeline.stop();
|
|
916
917
|
this._isOpen = false;
|
|
917
918
|
(0, import_log6.log)("closed", {}, {
|
|
918
919
|
file: "space.ts",
|
|
919
|
-
line:
|
|
920
|
+
line: 174,
|
|
920
921
|
scope: this,
|
|
921
922
|
callSite: (f, a) => f(...a)
|
|
922
923
|
});
|
|
923
924
|
}
|
|
924
|
-
|
|
925
|
+
// TODO(dmaretskyi): Make reusable.
|
|
926
|
+
async createDataPipeline({ start }) {
|
|
925
927
|
(0, import_node_assert6.default)(this._isOpen, "Space must be open to initialize data pipeline.");
|
|
926
|
-
(0, import_node_assert6.default)(!this._dataPipelineController, "Data pipeline already initialized.");
|
|
927
928
|
(0, import_node_assert6.default)(!this._dataPipeline, "Data pipeline already initialized.");
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
for (const feed of this._controlPipeline.spaceState.feeds.values()) {
|
|
937
|
-
if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
|
|
938
|
-
await this._dataPipeline.addFeed(await this._feedProvider(feed.key));
|
|
939
|
-
}
|
|
940
|
-
}
|
|
941
|
-
await this._dataPipeline.start();
|
|
942
|
-
return this._dataPipeline;
|
|
929
|
+
const pipeline = new Pipeline(start);
|
|
930
|
+
if (this._dataFeed) {
|
|
931
|
+
pipeline.setWriteFeed(this._dataFeed);
|
|
932
|
+
}
|
|
933
|
+
this._dataPipeline = pipeline;
|
|
934
|
+
for (const feed of this._controlPipeline.spaceState.feeds.values()) {
|
|
935
|
+
if (feed.assertion.designation === import_credentials3.AdmittedFeed.Designation.DATA) {
|
|
936
|
+
await pipeline.addFeed(await this._feedProvider(feed.key));
|
|
943
937
|
}
|
|
944
|
-
}
|
|
938
|
+
}
|
|
939
|
+
return pipeline;
|
|
945
940
|
}
|
|
946
941
|
};
|
|
947
942
|
__decorate3([
|
|
@@ -1220,11 +1215,49 @@ var import_credentials4 = require("@dxos/credentials");
|
|
|
1220
1215
|
var import_debug4 = require("@dxos/debug");
|
|
1221
1216
|
var import_credentials5 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
1222
1217
|
var spaceGenesis = async (keyring, signingContext, space) => {
|
|
1223
|
-
var _a, _b;
|
|
1224
|
-
const generator = new import_credentials4.CredentialGenerator(keyring, signingContext.identityKey, signingContext.deviceKey);
|
|
1218
|
+
var _a, _b, _c;
|
|
1225
1219
|
const credentials = [
|
|
1226
|
-
|
|
1227
|
-
|
|
1220
|
+
await (0, import_credentials4.createCredential)({
|
|
1221
|
+
signer: keyring,
|
|
1222
|
+
issuer: space.key,
|
|
1223
|
+
subject: space.key,
|
|
1224
|
+
assertion: {
|
|
1225
|
+
"@type": "dxos.halo.credentials.SpaceGenesis",
|
|
1226
|
+
spaceKey: space.key
|
|
1227
|
+
}
|
|
1228
|
+
}),
|
|
1229
|
+
await (0, import_credentials4.createCredential)({
|
|
1230
|
+
signer: keyring,
|
|
1231
|
+
issuer: space.key,
|
|
1232
|
+
subject: signingContext.identityKey,
|
|
1233
|
+
assertion: {
|
|
1234
|
+
"@type": "dxos.halo.credentials.SpaceMember",
|
|
1235
|
+
spaceKey: space.key,
|
|
1236
|
+
role: import_credentials5.SpaceMember.Role.ADMIN,
|
|
1237
|
+
profile: signingContext.profile,
|
|
1238
|
+
genesisFeedKey: (_a = space.controlFeedKey) != null ? _a : (0, import_debug4.failUndefined)()
|
|
1239
|
+
}
|
|
1240
|
+
}),
|
|
1241
|
+
await signingContext.credentialSigner.createCredential({
|
|
1242
|
+
subject: (_b = space.controlFeedKey) != null ? _b : (0, import_debug4.failUndefined)(),
|
|
1243
|
+
assertion: {
|
|
1244
|
+
"@type": "dxos.halo.credentials.AdmittedFeed",
|
|
1245
|
+
spaceKey: space.key,
|
|
1246
|
+
identityKey: signingContext.identityKey,
|
|
1247
|
+
deviceKey: signingContext.deviceKey,
|
|
1248
|
+
designation: import_credentials5.AdmittedFeed.Designation.CONTROL
|
|
1249
|
+
}
|
|
1250
|
+
}),
|
|
1251
|
+
await signingContext.credentialSigner.createCredential({
|
|
1252
|
+
subject: (_c = space.dataFeedKey) != null ? _c : (0, import_debug4.failUndefined)(),
|
|
1253
|
+
assertion: {
|
|
1254
|
+
"@type": "dxos.halo.credentials.AdmittedFeed",
|
|
1255
|
+
spaceKey: space.key,
|
|
1256
|
+
identityKey: signingContext.identityKey,
|
|
1257
|
+
deviceKey: signingContext.deviceKey,
|
|
1258
|
+
designation: import_credentials5.AdmittedFeed.Designation.DATA
|
|
1259
|
+
}
|
|
1260
|
+
})
|
|
1228
1261
|
];
|
|
1229
1262
|
for (const credential of credentials) {
|
|
1230
1263
|
await space.controlPipeline.writer.write({
|
|
@@ -1236,10 +1269,10 @@ var spaceGenesis = async (keyring, signingContext, space) => {
|
|
|
1236
1269
|
return credentials;
|
|
1237
1270
|
};
|
|
1238
1271
|
|
|
1239
|
-
// packages/core/echo/echo-pipeline/src/space/data-pipeline
|
|
1272
|
+
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1240
1273
|
var import_node_assert9 = __toESM(require("node:assert"));
|
|
1241
1274
|
var import_async8 = require("@dxos/async");
|
|
1242
|
-
var
|
|
1275
|
+
var import_context3 = require("@dxos/context");
|
|
1243
1276
|
var import_debug6 = require("@dxos/debug");
|
|
1244
1277
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
1245
1278
|
var import_log11 = require("@dxos/log");
|
|
@@ -1269,56 +1302,53 @@ var DataServiceHost = class {
|
|
|
1269
1302
|
objects
|
|
1270
1303
|
}
|
|
1271
1304
|
});
|
|
1272
|
-
this._itemDemuxer.mutation.on(ctx, (
|
|
1305
|
+
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
1273
1306
|
var _a;
|
|
1307
|
+
const { batch, meta } = message;
|
|
1274
1308
|
(0, import_log9.log)("Object update", {
|
|
1275
|
-
|
|
1309
|
+
batch,
|
|
1310
|
+
meta
|
|
1276
1311
|
}, {
|
|
1277
1312
|
file: "data-service-host.ts",
|
|
1278
|
-
line:
|
|
1313
|
+
line: 50,
|
|
1279
1314
|
scope: this,
|
|
1280
1315
|
callSite: (f, a) => f(...a)
|
|
1281
1316
|
});
|
|
1282
1317
|
const clientTag = this._clientTagMap.get([
|
|
1283
|
-
|
|
1284
|
-
|
|
1318
|
+
message.meta.feedKey,
|
|
1319
|
+
message.meta.seq
|
|
1285
1320
|
]);
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
...mutation.data,
|
|
1290
|
-
mutations: (_a = mutation.data.mutations) == null ? void 0 : _a.map((m, mutationIdx) => ({
|
|
1291
|
-
...m,
|
|
1292
|
-
meta: mutation.meta
|
|
1293
|
-
})),
|
|
1294
|
-
meta: mutation.meta
|
|
1295
|
-
}
|
|
1296
|
-
]
|
|
1297
|
-
};
|
|
1321
|
+
(_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
|
|
1322
|
+
(0, import_echo_db.setMetadataOnObject)(object, meta);
|
|
1323
|
+
});
|
|
1298
1324
|
if (clientTag) {
|
|
1299
1325
|
(0, import_echo_db.tagMutationsInBatch)(batch, clientTag);
|
|
1300
1326
|
}
|
|
1301
1327
|
next({
|
|
1302
1328
|
clientTag,
|
|
1303
|
-
feedKey:
|
|
1304
|
-
seq:
|
|
1329
|
+
feedKey: message.meta.feedKey,
|
|
1330
|
+
seq: message.meta.seq,
|
|
1305
1331
|
batch
|
|
1306
1332
|
});
|
|
1307
1333
|
});
|
|
1308
1334
|
});
|
|
1309
1335
|
}
|
|
1310
1336
|
async write(request) {
|
|
1311
|
-
var _a
|
|
1337
|
+
var _a;
|
|
1312
1338
|
(0, import_node_assert7.default)(this._writeStream, "Cannot write mutations in readonly mode");
|
|
1313
|
-
(0, import_node_assert7.default)(((_a = request.batch.objects) == null ? void 0 : _a.length) === 1, "Only single object mutations are supported");
|
|
1314
1339
|
const receipt = await this._writeStream.write({
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1340
|
+
batch: {
|
|
1341
|
+
objects: (_a = request.batch.objects) == null ? void 0 : _a.map((object) => {
|
|
1342
|
+
var _a2;
|
|
1343
|
+
return {
|
|
1344
|
+
...object,
|
|
1345
|
+
mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((m) => ({
|
|
1346
|
+
...m,
|
|
1347
|
+
meta: void 0
|
|
1348
|
+
})),
|
|
1349
|
+
meta: void 0
|
|
1350
|
+
};
|
|
1351
|
+
})
|
|
1322
1352
|
}
|
|
1323
1353
|
});
|
|
1324
1354
|
if (request.clientTag) {
|
|
@@ -1521,7 +1551,7 @@ var DataServiceImpl = class {
|
|
|
1521
1551
|
}
|
|
1522
1552
|
};
|
|
1523
1553
|
|
|
1524
|
-
// packages/core/echo/echo-pipeline/src/space/data-pipeline
|
|
1554
|
+
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1525
1555
|
var __decorate7 = function(decorators, target, key, desc) {
|
|
1526
1556
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1527
1557
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
@@ -1532,22 +1562,21 @@ var __decorate7 = function(decorators, target, key, desc) {
|
|
|
1532
1562
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1533
1563
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1534
1564
|
};
|
|
1535
|
-
var NoopDataPipelineController = class {
|
|
1536
|
-
async open(context) {
|
|
1537
|
-
}
|
|
1538
|
-
async close() {
|
|
1539
|
-
}
|
|
1540
|
-
};
|
|
1541
1565
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1542
1566
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1543
1567
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
1544
|
-
var
|
|
1568
|
+
var DISABLE_SNAPSHOT_CACHE = false;
|
|
1569
|
+
var DataPipeline = class DataPipeline2 {
|
|
1545
1570
|
constructor(_params) {
|
|
1546
1571
|
this._params = _params;
|
|
1547
|
-
this._ctx = new
|
|
1572
|
+
this._ctx = new import_context3.Context();
|
|
1548
1573
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe3.Timeframe();
|
|
1574
|
+
this._isOpen = false;
|
|
1549
1575
|
this.onTimeframeReached = new import_async8.Event();
|
|
1550
1576
|
}
|
|
1577
|
+
get isOpen() {
|
|
1578
|
+
return this._isOpen;
|
|
1579
|
+
}
|
|
1551
1580
|
get pipelineState() {
|
|
1552
1581
|
var _a;
|
|
1553
1582
|
return (_a = this._pipeline) == null ? void 0 : _a.state;
|
|
@@ -1560,14 +1589,25 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1560
1589
|
var _a;
|
|
1561
1590
|
return snapshotTimeframeToStartingTimeframe((_a = this.snapshotTimeframe) != null ? _a : new import_timeframe3.Timeframe());
|
|
1562
1591
|
}
|
|
1592
|
+
setTargetTimeframe(timeframe) {
|
|
1593
|
+
var _a;
|
|
1594
|
+
this._targetTimeframe = timeframe;
|
|
1595
|
+
(_a = this._pipeline) == null ? void 0 : _a.state.setTargetTimeframe(timeframe);
|
|
1596
|
+
}
|
|
1563
1597
|
async open(spaceContext) {
|
|
1564
1598
|
var _a, _b, _c, _d;
|
|
1599
|
+
if (this._isOpen) {
|
|
1600
|
+
return;
|
|
1601
|
+
}
|
|
1565
1602
|
this._spaceContext = spaceContext;
|
|
1566
1603
|
if (this._params.snapshotId) {
|
|
1567
1604
|
this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
|
|
1568
1605
|
this._lastAutomaticSnapshotTimeframe = (_b = (_a = this._snapshot) == null ? void 0 : _a.timeframe) != null ? _b : new import_timeframe3.Timeframe();
|
|
1569
1606
|
}
|
|
1570
1607
|
this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
|
|
1608
|
+
if (this._targetTimeframe) {
|
|
1609
|
+
this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
|
|
1610
|
+
}
|
|
1571
1611
|
const feedWriter = createMappedFeedWriter((data) => ({
|
|
1572
1612
|
data
|
|
1573
1613
|
}), (_c = this._pipeline.writer) != null ? _c : (0, import_debug6.failUndefined)());
|
|
@@ -1578,30 +1618,33 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1578
1618
|
await this._consumePipeline();
|
|
1579
1619
|
});
|
|
1580
1620
|
this._createPeriodicSnapshots();
|
|
1621
|
+
this._isOpen = true;
|
|
1581
1622
|
}
|
|
1582
1623
|
_createPeriodicSnapshots() {
|
|
1583
1624
|
this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
|
|
1584
1625
|
await this._saveLatestTimeframe();
|
|
1585
1626
|
});
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1627
|
+
if (!DISABLE_SNAPSHOT_CACHE) {
|
|
1628
|
+
this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
|
|
1629
|
+
var _a, _b;
|
|
1630
|
+
const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
|
|
1631
|
+
if (!latestTimeframe) {
|
|
1632
|
+
return;
|
|
1633
|
+
}
|
|
1634
|
+
if (latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
|
|
1635
|
+
const snapshot = await this._saveSnapshot();
|
|
1636
|
+
this._lastAutomaticSnapshotTimeframe = (_b = snapshot.timeframe) != null ? _b : (0, import_debug6.failUndefined)();
|
|
1637
|
+
(0, import_log11.log)("save", {
|
|
1638
|
+
snapshot
|
|
1639
|
+
}, {
|
|
1640
|
+
file: "data-pipeline.ts",
|
|
1641
|
+
line: 161,
|
|
1642
|
+
scope: this,
|
|
1643
|
+
callSite: (f, a) => f(...a)
|
|
1644
|
+
});
|
|
1645
|
+
}
|
|
1646
|
+
});
|
|
1647
|
+
}
|
|
1605
1648
|
}
|
|
1606
1649
|
async _saveSnapshot() {
|
|
1607
1650
|
const snapshot = await this.createSnapshot();
|
|
@@ -1610,21 +1653,41 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1610
1653
|
return snapshot;
|
|
1611
1654
|
}
|
|
1612
1655
|
async _saveLatestTimeframe() {
|
|
1613
|
-
var _a;
|
|
1656
|
+
var _a, _b;
|
|
1614
1657
|
const latestTimeframe = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe;
|
|
1658
|
+
(0, import_log11.log)("save latest timeframe", {
|
|
1659
|
+
latestTimeframe,
|
|
1660
|
+
spaceKey: this._params.spaceKey
|
|
1661
|
+
}, {
|
|
1662
|
+
file: "data-pipeline.ts",
|
|
1663
|
+
line: 176,
|
|
1664
|
+
scope: this,
|
|
1665
|
+
callSite: (f, a) => f(...a)
|
|
1666
|
+
});
|
|
1615
1667
|
if (latestTimeframe) {
|
|
1616
|
-
|
|
1668
|
+
const newTimeframe = import_timeframe3.Timeframe.merge((_b = this._targetTimeframe) != null ? _b : new import_timeframe3.Timeframe(), latestTimeframe);
|
|
1669
|
+
await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
|
|
1617
1670
|
}
|
|
1618
1671
|
}
|
|
1619
1672
|
async close() {
|
|
1620
1673
|
var _a, _b, _c;
|
|
1674
|
+
if (!this._isOpen) {
|
|
1675
|
+
return;
|
|
1676
|
+
}
|
|
1677
|
+
(0, import_log11.log)("close", {}, {
|
|
1678
|
+
file: "data-pipeline.ts",
|
|
1679
|
+
line: 188,
|
|
1680
|
+
scope: this,
|
|
1681
|
+
callSite: (f, a) => f(...a)
|
|
1682
|
+
});
|
|
1683
|
+
this._isOpen = false;
|
|
1621
1684
|
try {
|
|
1622
1685
|
await this._saveLatestTimeframe();
|
|
1623
1686
|
await this._saveSnapshot();
|
|
1624
1687
|
} catch (err) {
|
|
1625
1688
|
import_log11.log.catch(err, {}, {
|
|
1626
|
-
file: "data-pipeline
|
|
1627
|
-
line:
|
|
1689
|
+
file: "data-pipeline.ts",
|
|
1690
|
+
line: 195,
|
|
1628
1691
|
scope: this,
|
|
1629
1692
|
callSite: (f, a) => f(...a)
|
|
1630
1693
|
});
|
|
@@ -1637,6 +1700,7 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1637
1700
|
}
|
|
1638
1701
|
createSnapshot() {
|
|
1639
1702
|
var _a, _b;
|
|
1703
|
+
(0, import_node_assert9.default)(this.databaseBackend, "Database backend is not initialized.");
|
|
1640
1704
|
return {
|
|
1641
1705
|
spaceKey: this._params.spaceKey.asUint8Array(),
|
|
1642
1706
|
timeframe: (_b = (_a = this._pipeline) == null ? void 0 : _a.state.timeframe) != null ? _b : new import_timeframe3.Timeframe(),
|
|
@@ -1650,8 +1714,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1650
1714
|
(0, import_log11.log)("processing message", {
|
|
1651
1715
|
msg
|
|
1652
1716
|
}, {
|
|
1653
|
-
file: "data-pipeline
|
|
1654
|
-
line:
|
|
1717
|
+
file: "data-pipeline.ts",
|
|
1718
|
+
line: 217,
|
|
1655
1719
|
scope: this,
|
|
1656
1720
|
callSite: (f, a) => f(...a)
|
|
1657
1721
|
});
|
|
@@ -1662,15 +1726,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1662
1726
|
import_log11.log.error("Could not find feed.", {
|
|
1663
1727
|
feedKey
|
|
1664
1728
|
}, {
|
|
1665
|
-
file: "data-pipeline
|
|
1666
|
-
line:
|
|
1729
|
+
file: "data-pipeline.ts",
|
|
1730
|
+
line: 223,
|
|
1667
1731
|
scope: this,
|
|
1668
1732
|
callSite: (f, a) => f(...a)
|
|
1669
1733
|
});
|
|
1670
1734
|
continue;
|
|
1671
1735
|
}
|
|
1672
1736
|
await this.databaseBackend.echoProcessor({
|
|
1673
|
-
|
|
1737
|
+
batch: data.payload.data.batch,
|
|
1674
1738
|
meta: {
|
|
1675
1739
|
feedKey,
|
|
1676
1740
|
seq,
|
|
@@ -1682,8 +1746,8 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1682
1746
|
}
|
|
1683
1747
|
} catch (err) {
|
|
1684
1748
|
import_log11.log.catch(err, {}, {
|
|
1685
|
-
file: "data-pipeline
|
|
1686
|
-
line:
|
|
1749
|
+
file: "data-pipeline.ts",
|
|
1750
|
+
line: 239,
|
|
1687
1751
|
scope: this,
|
|
1688
1752
|
callSite: (f, a) => f(...a)
|
|
1689
1753
|
});
|
|
@@ -1695,9 +1759,15 @@ var DataPipelineControllerImpl = class DataPipelineControllerImpl2 {
|
|
|
1695
1759
|
await this._pipeline.state.waitUntilTimeframe(timeframe);
|
|
1696
1760
|
}
|
|
1697
1761
|
};
|
|
1698
|
-
|
|
1762
|
+
__decorate7([
|
|
1763
|
+
import_async8.synchronized
|
|
1764
|
+
], DataPipeline.prototype, "open", null);
|
|
1765
|
+
__decorate7([
|
|
1766
|
+
import_async8.synchronized
|
|
1767
|
+
], DataPipeline.prototype, "close", null);
|
|
1768
|
+
DataPipeline = __decorate7([
|
|
1699
1769
|
(0, import_async8.trackLeaks)("open", "close")
|
|
1700
|
-
],
|
|
1770
|
+
], DataPipeline);
|
|
1701
1771
|
var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
|
|
1702
1772
|
return snapshotTimeframe.map(([key, seq]) => [
|
|
1703
1773
|
key,
|
|
@@ -1709,7 +1779,7 @@ var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
|
|
|
1709
1779
|
AuthExtension,
|
|
1710
1780
|
AuthStatus,
|
|
1711
1781
|
DBError,
|
|
1712
|
-
|
|
1782
|
+
DataPipeline,
|
|
1713
1783
|
DataServiceHost,
|
|
1714
1784
|
DataServiceImpl,
|
|
1715
1785
|
DataServiceSubscriptions,
|
|
@@ -1721,7 +1791,6 @@ var snapshotTimeframeToStartingTimeframe = (snapshotTimeframe) => {
|
|
|
1721
1791
|
MOCK_AUTH_PROVIDER,
|
|
1722
1792
|
MOCK_AUTH_VERIFIER,
|
|
1723
1793
|
MetadataStore,
|
|
1724
|
-
NoopDataPipelineController,
|
|
1725
1794
|
Pipeline,
|
|
1726
1795
|
STORAGE_VERSION,
|
|
1727
1796
|
SnapshotManager,
|