@dxos/echo-pipeline 0.1.56-main.c19b7cd → 0.1.56-main.c277f68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-EJ35NHZV.mjs → chunk-GDF4DQU2.mjs} +296 -272
- package/dist/lib/browser/chunk-GDF4DQU2.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/index.mjs.map +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +12 -17
- package/dist/lib/browser/testing/index.mjs.map +1 -1
- package/dist/lib/node/index.cjs +300 -276
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +313 -294
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/db-host/data-service-host.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts +2 -0
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts.map +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts +4 -3
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +4 -0
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +2 -0
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts +4 -1
- package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
- package/package.json +32 -32
- package/src/db-host/data-service-host.ts +9 -2
- package/src/metadata/metadata-store.ts +1 -1
- package/src/pipeline/pipeline-stress.test.ts +226 -0
- package/src/pipeline/pipeline.test.ts +2 -214
- package/src/pipeline/pipeline.ts +47 -22
- package/src/space/control-pipeline.ts +47 -22
- package/src/space/data-pipeline.ts +20 -10
- package/src/space/space-manager.ts +1 -0
- package/src/space/space-protocol.ts +3 -1
- package/src/space/space.ts +1 -0
- package/src/testing/database-test-rig.ts +27 -2
- package/src/tests/database-unit.test.ts +39 -0
- package/dist/lib/browser/chunk-EJ35NHZV.mjs.map +0 -7
|
@@ -42,7 +42,7 @@ __export(testing_exports, {
|
|
|
42
42
|
module.exports = __toCommonJS(testing_exports);
|
|
43
43
|
|
|
44
44
|
// packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
|
|
45
|
-
var
|
|
45
|
+
var import_context7 = require("@dxos/context");
|
|
46
46
|
var import_credentials5 = require("@dxos/credentials");
|
|
47
47
|
var import_document_model = require("@dxos/document-model");
|
|
48
48
|
var import_keys7 = require("@dxos/keys");
|
|
@@ -61,6 +61,7 @@ var import_context = require("@dxos/context");
|
|
|
61
61
|
var import_echo_db = require("@dxos/echo-db");
|
|
62
62
|
var import_invariant = require("@dxos/invariant");
|
|
63
63
|
var import_log = require("@dxos/log");
|
|
64
|
+
var import_service = require("@dxos/protocols/proto/dxos/echo/service");
|
|
64
65
|
var import_util = require("@dxos/util");
|
|
65
66
|
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
|
|
66
67
|
var DataServiceHost = class {
|
|
@@ -88,12 +89,19 @@ var DataServiceHost = class {
|
|
|
88
89
|
objects
|
|
89
90
|
}
|
|
90
91
|
});
|
|
92
|
+
this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
|
|
93
|
+
next({
|
|
94
|
+
action: import_service.EchoEvent.DatabaseAction.RESET,
|
|
95
|
+
batch: {
|
|
96
|
+
objects: snapshot.items
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
});
|
|
91
100
|
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
92
|
-
var _a;
|
|
93
101
|
const { batch, meta } = message;
|
|
94
102
|
(0, import_invariant.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
|
|
95
103
|
F: __dxlog_file,
|
|
96
|
-
L:
|
|
104
|
+
L: 68,
|
|
97
105
|
S: this,
|
|
98
106
|
A: [
|
|
99
107
|
"!(meta as any).clientTag",
|
|
@@ -101,11 +109,11 @@ var DataServiceHost = class {
|
|
|
101
109
|
]
|
|
102
110
|
});
|
|
103
111
|
(0, import_log.log)("message", {
|
|
104
|
-
batch,
|
|
112
|
+
batch: batch.objects?.length,
|
|
105
113
|
meta
|
|
106
114
|
}, {
|
|
107
115
|
F: __dxlog_file,
|
|
108
|
-
L:
|
|
116
|
+
L: 69,
|
|
109
117
|
S: this,
|
|
110
118
|
C: (f, a) => f(...a)
|
|
111
119
|
});
|
|
@@ -113,7 +121,7 @@ var DataServiceHost = class {
|
|
|
113
121
|
message.meta.feedKey,
|
|
114
122
|
message.meta.seq
|
|
115
123
|
]);
|
|
116
|
-
|
|
124
|
+
batch.objects?.forEach((object) => {
|
|
117
125
|
(0, import_echo_db.setMetadataOnObject)(object, {
|
|
118
126
|
...meta
|
|
119
127
|
});
|
|
@@ -131,10 +139,9 @@ var DataServiceHost = class {
|
|
|
131
139
|
});
|
|
132
140
|
}
|
|
133
141
|
async write(request) {
|
|
134
|
-
var _a, _b;
|
|
135
142
|
(0, import_invariant.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
|
|
136
143
|
F: __dxlog_file,
|
|
137
|
-
L:
|
|
144
|
+
L: 95,
|
|
138
145
|
S: this,
|
|
139
146
|
A: [
|
|
140
147
|
"!this._ctx.disposed",
|
|
@@ -143,7 +150,7 @@ var DataServiceHost = class {
|
|
|
143
150
|
});
|
|
144
151
|
(0, import_invariant.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
|
|
145
152
|
F: __dxlog_file,
|
|
146
|
-
L:
|
|
153
|
+
L: 96,
|
|
147
154
|
S: this,
|
|
148
155
|
A: [
|
|
149
156
|
"this._writeStream",
|
|
@@ -152,10 +159,10 @@ var DataServiceHost = class {
|
|
|
152
159
|
});
|
|
153
160
|
(0, import_log.log)("write", {
|
|
154
161
|
clientTag: request.clientTag,
|
|
155
|
-
objectCount:
|
|
162
|
+
objectCount: request.batch.objects?.length ?? 0
|
|
156
163
|
}, {
|
|
157
164
|
F: __dxlog_file,
|
|
158
|
-
L:
|
|
165
|
+
L: 98,
|
|
159
166
|
S: this,
|
|
160
167
|
C: (f, a) => f(...a)
|
|
161
168
|
});
|
|
@@ -169,7 +176,7 @@ var DataServiceHost = class {
|
|
|
169
176
|
seq: receipt2.seq
|
|
170
177
|
}, {
|
|
171
178
|
F: __dxlog_file,
|
|
172
|
-
L:
|
|
179
|
+
L: 107,
|
|
173
180
|
S: this,
|
|
174
181
|
C: (f, a) => f(...a)
|
|
175
182
|
});
|
|
@@ -183,24 +190,18 @@ var DataServiceHost = class {
|
|
|
183
190
|
return receipt;
|
|
184
191
|
}
|
|
185
192
|
};
|
|
186
|
-
var createDataMessage = (batch) => {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
meta: void 0
|
|
199
|
-
};
|
|
200
|
-
})
|
|
201
|
-
}
|
|
202
|
-
};
|
|
203
|
-
};
|
|
193
|
+
var createDataMessage = (batch) => ({
|
|
194
|
+
batch: {
|
|
195
|
+
objects: batch.objects?.map((object) => ({
|
|
196
|
+
...object,
|
|
197
|
+
mutations: object.mutations?.map((mutation) => ({
|
|
198
|
+
...mutation,
|
|
199
|
+
meta: void 0
|
|
200
|
+
})),
|
|
201
|
+
meta: void 0
|
|
202
|
+
}))
|
|
203
|
+
}
|
|
204
|
+
});
|
|
204
205
|
|
|
205
206
|
// packages/core/echo/echo-pipeline/src/db-host/database-host.ts
|
|
206
207
|
var import_echo_db2 = require("@dxos/echo-db");
|
|
@@ -229,8 +230,7 @@ var DatabaseHost = class {
|
|
|
229
230
|
return this._itemDemuxer.createSnapshot();
|
|
230
231
|
}
|
|
231
232
|
createDataServiceHost() {
|
|
232
|
-
|
|
233
|
-
return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
|
|
233
|
+
return new DataServiceHost(this._itemManager, this._itemDemuxer, this._outboundStream ?? void 0);
|
|
234
234
|
}
|
|
235
235
|
};
|
|
236
236
|
|
|
@@ -376,7 +376,7 @@ var DataServiceSubscriptions = class {
|
|
|
376
376
|
C: (f, a) => f(...a)
|
|
377
377
|
});
|
|
378
378
|
const host = this._spaces.get(spaceKey);
|
|
379
|
-
await
|
|
379
|
+
await host?.close();
|
|
380
380
|
this._spaces.delete(spaceKey);
|
|
381
381
|
}
|
|
382
382
|
getDataService(spaceKey) {
|
|
@@ -388,7 +388,6 @@ var DataServiceImpl = class {
|
|
|
388
388
|
this._subscriptions = _subscriptions;
|
|
389
389
|
}
|
|
390
390
|
subscribe(request) {
|
|
391
|
-
var _a;
|
|
392
391
|
(0, import_invariant2.invariant)(request.spaceKey, void 0, {
|
|
393
392
|
F: __dxlog_file2,
|
|
394
393
|
L: 56,
|
|
@@ -398,11 +397,10 @@ var DataServiceImpl = class {
|
|
|
398
397
|
""
|
|
399
398
|
]
|
|
400
399
|
});
|
|
401
|
-
const host =
|
|
400
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
402
401
|
return host.subscribe();
|
|
403
402
|
}
|
|
404
403
|
write(request) {
|
|
405
|
-
var _a;
|
|
406
404
|
(0, import_invariant2.invariant)(request.spaceKey, void 0, {
|
|
407
405
|
F: __dxlog_file2,
|
|
408
406
|
L: 63,
|
|
@@ -421,7 +419,7 @@ var DataServiceImpl = class {
|
|
|
421
419
|
""
|
|
422
420
|
]
|
|
423
421
|
});
|
|
424
|
-
const host =
|
|
422
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
425
423
|
return host.write(request);
|
|
426
424
|
}
|
|
427
425
|
};
|
|
@@ -449,8 +447,8 @@ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipelin
|
|
|
449
447
|
var emptyEchoMetadata = () => ({
|
|
450
448
|
version: import_protocols3.STORAGE_VERSION,
|
|
451
449
|
spaces: [],
|
|
452
|
-
created: new Date(),
|
|
453
|
-
updated: new Date()
|
|
450
|
+
created: /* @__PURE__ */ new Date(),
|
|
451
|
+
updated: /* @__PURE__ */ new Date()
|
|
454
452
|
});
|
|
455
453
|
var EchoMetadata = import_protocols3.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
456
454
|
var MetadataStore = class {
|
|
@@ -464,22 +462,19 @@ var MetadataStore = class {
|
|
|
464
462
|
return this._metadata;
|
|
465
463
|
}
|
|
466
464
|
get version() {
|
|
467
|
-
|
|
468
|
-
return (_a = this._metadata.version) != null ? _a : 0;
|
|
465
|
+
return this._metadata.version ?? 0;
|
|
469
466
|
}
|
|
470
467
|
/**
|
|
471
468
|
* Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
|
|
472
469
|
* addSpaceFeed functions.
|
|
473
470
|
*/
|
|
474
471
|
get spaces() {
|
|
475
|
-
|
|
476
|
-
return (_a = this._metadata.spaces) != null ? _a : [];
|
|
472
|
+
return this._metadata.spaces ?? [];
|
|
477
473
|
}
|
|
478
474
|
/**
|
|
479
475
|
* Loads metadata from persistent storage.
|
|
480
476
|
*/
|
|
481
477
|
async load() {
|
|
482
|
-
var _a;
|
|
483
478
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
484
479
|
try {
|
|
485
480
|
const { size: fileLength } = await file.stat();
|
|
@@ -498,7 +493,10 @@ var MetadataStore = class {
|
|
|
498
493
|
C: (f, a) => f(...a)
|
|
499
494
|
});
|
|
500
495
|
if (fileLength < dataSize + 8) {
|
|
501
|
-
throw new import_errors.DataCorruptionError("Metadata size is smaller than expected."
|
|
496
|
+
throw new import_errors.DataCorruptionError("Metadata size is smaller than expected.", {
|
|
497
|
+
fileLength,
|
|
498
|
+
dataSize
|
|
499
|
+
});
|
|
502
500
|
}
|
|
503
501
|
const data = await file.read(8, dataSize);
|
|
504
502
|
const calculatedChecksum = import_crc_32.default.buf(data);
|
|
@@ -506,9 +504,8 @@ var MetadataStore = class {
|
|
|
506
504
|
throw new import_errors.DataCorruptionError("Metadata checksum is invalid.");
|
|
507
505
|
}
|
|
508
506
|
this._metadata = EchoMetadata.decode(data);
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
(_a2 = space.state) != null ? _a2 : space.state = import_services.SpaceState.ACTIVE;
|
|
507
|
+
this._metadata.spaces?.forEach((space) => {
|
|
508
|
+
space.state ??= import_services.SpaceState.ACTIVE;
|
|
512
509
|
});
|
|
513
510
|
} catch (err) {
|
|
514
511
|
import_log3.log.error("failed to load metadata", {
|
|
@@ -525,12 +522,11 @@ var MetadataStore = class {
|
|
|
525
522
|
}
|
|
526
523
|
}
|
|
527
524
|
async _save() {
|
|
528
|
-
var _a;
|
|
529
525
|
const data = {
|
|
530
526
|
...this._metadata,
|
|
531
527
|
version: import_protocols3.STORAGE_VERSION,
|
|
532
|
-
created:
|
|
533
|
-
updated: new Date()
|
|
528
|
+
created: this._metadata.created ?? /* @__PURE__ */ new Date(),
|
|
529
|
+
updated: /* @__PURE__ */ new Date()
|
|
534
530
|
};
|
|
535
531
|
this.update.emit(data);
|
|
536
532
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
@@ -556,8 +552,7 @@ var MetadataStore = class {
|
|
|
556
552
|
}
|
|
557
553
|
}
|
|
558
554
|
_getSpace(spaceKey) {
|
|
559
|
-
|
|
560
|
-
if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
|
|
555
|
+
if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
|
|
561
556
|
return this._metadata.identity.haloSpace;
|
|
562
557
|
}
|
|
563
558
|
const space = this.spaces.find((space2) => space2.key === spaceKey);
|
|
@@ -602,8 +597,7 @@ var MetadataStore = class {
|
|
|
602
597
|
await this._save();
|
|
603
598
|
}
|
|
604
599
|
async addSpace(record) {
|
|
605
|
-
|
|
606
|
-
(0, import_invariant3.invariant)(!((_a = this._metadata.spaces) != null ? _a : []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
|
|
600
|
+
(0, import_invariant3.invariant)(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
|
|
607
601
|
F: __dxlog_file3,
|
|
608
602
|
L: 162,
|
|
609
603
|
S: this,
|
|
@@ -612,7 +606,7 @@ var MetadataStore = class {
|
|
|
612
606
|
"'Cannot overwrite existing space in metadata'"
|
|
613
607
|
]
|
|
614
608
|
});
|
|
615
|
-
(
|
|
609
|
+
(this._metadata.spaces ??= []).push(record);
|
|
616
610
|
await this._save();
|
|
617
611
|
}
|
|
618
612
|
async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
|
|
@@ -711,7 +705,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
711
705
|
const { credential } = await this.rpc.AuthService.authenticate({
|
|
712
706
|
challenge
|
|
713
707
|
});
|
|
714
|
-
(0, import_invariant4.invariant)(
|
|
708
|
+
(0, import_invariant4.invariant)(credential?.length > 0, "invalid credential", {
|
|
715
709
|
F: __dxlog_file4,
|
|
716
710
|
L: 69,
|
|
717
711
|
S: this,
|
|
@@ -754,15 +748,17 @@ var import_async7 = require("@dxos/async");
|
|
|
754
748
|
var import_invariant9 = require("@dxos/invariant");
|
|
755
749
|
var import_log10 = require("@dxos/log");
|
|
756
750
|
var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
757
|
-
var
|
|
751
|
+
var import_tracing3 = require("@dxos/tracing");
|
|
758
752
|
var import_util7 = require("@dxos/util");
|
|
759
753
|
|
|
760
754
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
755
|
+
var import_context5 = require("@dxos/context");
|
|
761
756
|
var import_credentials = require("@dxos/credentials");
|
|
762
757
|
var import_keys4 = require("@dxos/keys");
|
|
763
758
|
var import_log8 = require("@dxos/log");
|
|
764
759
|
var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
765
760
|
var import_timeframe3 = require("@dxos/timeframe");
|
|
761
|
+
var import_tracing = require("@dxos/tracing");
|
|
766
762
|
var import_util5 = require("@dxos/util");
|
|
767
763
|
|
|
768
764
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
@@ -995,7 +991,6 @@ var PipelineState = class {
|
|
|
995
991
|
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
996
992
|
*/
|
|
997
993
|
async waitUntilReachedTargetTimeframe({ ctx = new import_context4.Context(), timeout, breakOnStall = true } = {}) {
|
|
998
|
-
var _a;
|
|
999
994
|
(0, import_log7.log)("waitUntilReachedTargetTimeframe", {
|
|
1000
995
|
timeout,
|
|
1001
996
|
current: this.timeframe,
|
|
@@ -1006,7 +1001,7 @@ var PipelineState = class {
|
|
|
1006
1001
|
S: this,
|
|
1007
1002
|
C: (f, a) => f(...a)
|
|
1008
1003
|
});
|
|
1009
|
-
|
|
1004
|
+
this._reachedTargetPromise ??= Promise.race([
|
|
1010
1005
|
this._timeframeClock.update.waitForCondition(() => {
|
|
1011
1006
|
return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
|
|
1012
1007
|
}),
|
|
@@ -1053,9 +1048,11 @@ var Pipeline = class {
|
|
|
1053
1048
|
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
1054
1049
|
this._processingTrigger = new import_async5.Trigger().wake();
|
|
1055
1050
|
this._pauseTrigger = new import_async5.Trigger().wake();
|
|
1051
|
+
// Pending downloads.
|
|
1052
|
+
this._downloads = new import_util4.ComplexMap((value) => import_keys3.PublicKey.hash(value.key));
|
|
1056
1053
|
this._isStopping = false;
|
|
1057
1054
|
this._isStarted = false;
|
|
1058
|
-
this.
|
|
1055
|
+
this._isBeingConsumed = false;
|
|
1059
1056
|
this._isPaused = false;
|
|
1060
1057
|
}
|
|
1061
1058
|
get state() {
|
|
@@ -1064,7 +1061,7 @@ var Pipeline = class {
|
|
|
1064
1061
|
get writer() {
|
|
1065
1062
|
(0, import_invariant7.invariant)(this._writer, "Writer not set.", {
|
|
1066
1063
|
F: __dxlog_file8,
|
|
1067
|
-
L:
|
|
1064
|
+
L: 233,
|
|
1068
1065
|
S: this,
|
|
1069
1066
|
A: [
|
|
1070
1067
|
"this._writer",
|
|
@@ -1086,12 +1083,14 @@ var Pipeline = class {
|
|
|
1086
1083
|
if (this._feedSetIterator) {
|
|
1087
1084
|
await this._feedSetIterator.addFeed(feed);
|
|
1088
1085
|
}
|
|
1089
|
-
this.
|
|
1086
|
+
if (this._isStarted && !this._isPaused) {
|
|
1087
|
+
this._setFeedDownloadState(feed);
|
|
1088
|
+
}
|
|
1090
1089
|
}
|
|
1091
1090
|
setWriteFeed(feed) {
|
|
1092
1091
|
(0, import_invariant7.invariant)(!this._writer, "Writer already set.", {
|
|
1093
1092
|
F: __dxlog_file8,
|
|
1094
|
-
L:
|
|
1093
|
+
L: 260,
|
|
1095
1094
|
S: this,
|
|
1096
1095
|
A: [
|
|
1097
1096
|
"!this._writer",
|
|
@@ -1100,7 +1099,7 @@ var Pipeline = class {
|
|
|
1100
1099
|
});
|
|
1101
1100
|
(0, import_invariant7.invariant)(feed.properties.writable, "Feed must be writable.", {
|
|
1102
1101
|
F: __dxlog_file8,
|
|
1103
|
-
L:
|
|
1102
|
+
L: 261,
|
|
1104
1103
|
S: this,
|
|
1105
1104
|
A: [
|
|
1106
1105
|
"feed.properties.writable",
|
|
@@ -1115,7 +1114,7 @@ var Pipeline = class {
|
|
|
1115
1114
|
async start() {
|
|
1116
1115
|
(0, import_invariant7.invariant)(!this._isStarted, "Pipeline is already started.", {
|
|
1117
1116
|
F: __dxlog_file8,
|
|
1118
|
-
L:
|
|
1117
|
+
L: 274,
|
|
1119
1118
|
S: this,
|
|
1120
1119
|
A: [
|
|
1121
1120
|
"!this._isStarted",
|
|
@@ -1124,7 +1123,7 @@ var Pipeline = class {
|
|
|
1124
1123
|
});
|
|
1125
1124
|
(0, import_log7.log)("starting...", void 0, {
|
|
1126
1125
|
F: __dxlog_file8,
|
|
1127
|
-
L:
|
|
1126
|
+
L: 275,
|
|
1128
1127
|
S: this,
|
|
1129
1128
|
C: (f, a) => f(...a)
|
|
1130
1129
|
});
|
|
@@ -1133,21 +1132,29 @@ var Pipeline = class {
|
|
|
1133
1132
|
this._isStarted = true;
|
|
1134
1133
|
(0, import_log7.log)("started", void 0, {
|
|
1135
1134
|
F: __dxlog_file8,
|
|
1136
|
-
L:
|
|
1135
|
+
L: 279,
|
|
1137
1136
|
S: this,
|
|
1138
1137
|
C: (f, a) => f(...a)
|
|
1139
1138
|
});
|
|
1139
|
+
if (!this._isPaused) {
|
|
1140
|
+
for (const feed of this._feeds.values()) {
|
|
1141
|
+
this._setFeedDownloadState(feed);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1140
1144
|
}
|
|
1141
1145
|
async stop() {
|
|
1142
|
-
var _a;
|
|
1143
1146
|
(0, import_log7.log)("stopping...", void 0, {
|
|
1144
1147
|
F: __dxlog_file8,
|
|
1145
|
-
L:
|
|
1148
|
+
L: 290,
|
|
1146
1149
|
S: this,
|
|
1147
1150
|
C: (f, a) => f(...a)
|
|
1148
1151
|
});
|
|
1149
1152
|
this._isStopping = true;
|
|
1150
|
-
|
|
1153
|
+
for (const [feed, handle] of this._downloads.entries()) {
|
|
1154
|
+
feed.undownload(handle);
|
|
1155
|
+
}
|
|
1156
|
+
this._downloads.clear();
|
|
1157
|
+
await this._feedSetIterator?.close();
|
|
1151
1158
|
await this._processingTrigger.wait();
|
|
1152
1159
|
await this._state._ctx.dispose();
|
|
1153
1160
|
this._state._ctx = new import_context4.Context();
|
|
@@ -1155,7 +1162,7 @@ var Pipeline = class {
|
|
|
1155
1162
|
this._isStarted = false;
|
|
1156
1163
|
(0, import_log7.log)("stopped", void 0, {
|
|
1157
1164
|
F: __dxlog_file8,
|
|
1158
|
-
L:
|
|
1165
|
+
L: 302,
|
|
1159
1166
|
S: this,
|
|
1160
1167
|
C: (f, a) => f(...a)
|
|
1161
1168
|
});
|
|
@@ -1167,7 +1174,7 @@ var Pipeline = class {
|
|
|
1167
1174
|
async setCursor(timeframe) {
|
|
1168
1175
|
(0, import_invariant7.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
|
|
1169
1176
|
F: __dxlog_file8,
|
|
1170
|
-
L:
|
|
1177
|
+
L: 311,
|
|
1171
1178
|
S: this,
|
|
1172
1179
|
A: [
|
|
1173
1180
|
"!this._isStarted || this._isPaused",
|
|
@@ -1176,9 +1183,6 @@ var Pipeline = class {
|
|
|
1176
1183
|
});
|
|
1177
1184
|
this._state._startTimeframe = timeframe;
|
|
1178
1185
|
this._timeframeClock.setTimeframe(timeframe);
|
|
1179
|
-
for (const feed of this._feeds.values()) {
|
|
1180
|
-
this._setFeedDownloadState(feed);
|
|
1181
|
-
}
|
|
1182
1186
|
if (this._feedSetIterator) {
|
|
1183
1187
|
await this._feedSetIterator.close();
|
|
1184
1188
|
await this._initIterator();
|
|
@@ -1189,15 +1193,6 @@ var Pipeline = class {
|
|
|
1189
1193
|
* Calling pause while processing will cause a deadlock.
|
|
1190
1194
|
*/
|
|
1191
1195
|
async pause() {
|
|
1192
|
-
(0, import_invariant7.invariant)(this._isStarted, "Pipeline is not open.", {
|
|
1193
|
-
F: __dxlog_file8,
|
|
1194
|
-
L: 316,
|
|
1195
|
-
S: this,
|
|
1196
|
-
A: [
|
|
1197
|
-
"this._isStarted",
|
|
1198
|
-
"'Pipeline is not open.'"
|
|
1199
|
-
]
|
|
1200
|
-
});
|
|
1201
1196
|
if (this._isPaused) {
|
|
1202
1197
|
return;
|
|
1203
1198
|
}
|
|
@@ -1206,18 +1201,9 @@ var Pipeline = class {
|
|
|
1206
1201
|
this._isPaused = true;
|
|
1207
1202
|
}
|
|
1208
1203
|
async unpause() {
|
|
1209
|
-
(0, import_invariant7.invariant)(this._isStarted, "Pipeline is not open.", {
|
|
1210
|
-
F: __dxlog_file8,
|
|
1211
|
-
L: 328,
|
|
1212
|
-
S: this,
|
|
1213
|
-
A: [
|
|
1214
|
-
"this._isStarted",
|
|
1215
|
-
"'Pipeline is not open.'"
|
|
1216
|
-
]
|
|
1217
|
-
});
|
|
1218
1204
|
(0, import_invariant7.invariant)(this._isPaused, "Pipeline is not paused.", {
|
|
1219
1205
|
F: __dxlog_file8,
|
|
1220
|
-
L:
|
|
1206
|
+
L: 340,
|
|
1221
1207
|
S: this,
|
|
1222
1208
|
A: [
|
|
1223
1209
|
"this._isPaused",
|
|
@@ -1226,25 +1212,28 @@ var Pipeline = class {
|
|
|
1226
1212
|
});
|
|
1227
1213
|
this._pauseTrigger.wake();
|
|
1228
1214
|
this._isPaused = false;
|
|
1215
|
+
for (const feed of this._feeds.values()) {
|
|
1216
|
+
this._setFeedDownloadState(feed);
|
|
1217
|
+
}
|
|
1229
1218
|
}
|
|
1230
1219
|
/**
|
|
1231
1220
|
* Starts to iterate over the ordered messages from the added feeds.
|
|
1232
1221
|
* Updates the timeframe clock after the message has bee processed.
|
|
1233
1222
|
*/
|
|
1234
1223
|
async *consume() {
|
|
1235
|
-
(0, import_invariant7.invariant)(!this.
|
|
1224
|
+
(0, import_invariant7.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
|
|
1236
1225
|
F: __dxlog_file8,
|
|
1237
|
-
L:
|
|
1226
|
+
L: 355,
|
|
1238
1227
|
S: this,
|
|
1239
1228
|
A: [
|
|
1240
|
-
"!this.
|
|
1229
|
+
"!this._isBeingConsumed",
|
|
1241
1230
|
"'Pipeline is already being consumed.'"
|
|
1242
1231
|
]
|
|
1243
1232
|
});
|
|
1244
|
-
this.
|
|
1233
|
+
this._isBeingConsumed = true;
|
|
1245
1234
|
(0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1246
1235
|
F: __dxlog_file8,
|
|
1247
|
-
L:
|
|
1236
|
+
L: 358,
|
|
1248
1237
|
S: this,
|
|
1249
1238
|
A: [
|
|
1250
1239
|
"this._feedSetIterator",
|
|
@@ -1258,7 +1247,7 @@ var Pipeline = class {
|
|
|
1258
1247
|
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
1259
1248
|
(0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1260
1249
|
F: __dxlog_file8,
|
|
1261
|
-
L:
|
|
1250
|
+
L: 367,
|
|
1262
1251
|
S: this,
|
|
1263
1252
|
A: [
|
|
1264
1253
|
"this._feedSetIterator",
|
|
@@ -1270,7 +1259,7 @@ var Pipeline = class {
|
|
|
1270
1259
|
}
|
|
1271
1260
|
const { done, value } = await iterable.next();
|
|
1272
1261
|
if (!done) {
|
|
1273
|
-
const block = value
|
|
1262
|
+
const block = value ?? (0, import_debug3.failUndefined)();
|
|
1274
1263
|
this._processingTrigger.reset();
|
|
1275
1264
|
this._timeframeClock.updatePendingTimeframe(import_keys3.PublicKey.from(block.feedKey), block.seq);
|
|
1276
1265
|
yield block;
|
|
@@ -1278,33 +1267,40 @@ var Pipeline = class {
|
|
|
1278
1267
|
this._timeframeClock.updateTimeframe();
|
|
1279
1268
|
}
|
|
1280
1269
|
}
|
|
1281
|
-
this.
|
|
1270
|
+
this._isBeingConsumed = false;
|
|
1282
1271
|
}
|
|
1283
1272
|
_setFeedDownloadState(feed) {
|
|
1284
|
-
|
|
1273
|
+
let handle = this._downloads.get(feed);
|
|
1274
|
+
if (handle) {
|
|
1275
|
+
feed.undownload(handle);
|
|
1276
|
+
}
|
|
1285
1277
|
const timeframe = this._state._startTimeframe;
|
|
1286
|
-
const seq =
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1278
|
+
const seq = timeframe.get(feed.key) ?? -1;
|
|
1279
|
+
import_log7.log.info("download", {
|
|
1280
|
+
feed: feed.key.truncate(),
|
|
1281
|
+
seq,
|
|
1282
|
+
length: feed.length
|
|
1283
|
+
}, {
|
|
1284
|
+
F: __dxlog_file8,
|
|
1285
|
+
L: 396,
|
|
1286
|
+
S: this,
|
|
1287
|
+
C: (f, a) => f(...a)
|
|
1294
1288
|
});
|
|
1295
|
-
feed.download({
|
|
1289
|
+
handle = feed.download({
|
|
1296
1290
|
start: seq + 1,
|
|
1297
1291
|
linear: true
|
|
1298
|
-
}
|
|
1299
|
-
(
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1292
|
+
}, (err, data) => {
|
|
1293
|
+
if (err) {
|
|
1294
|
+
} else {
|
|
1295
|
+
import_log7.log.info("data", data, {
|
|
1296
|
+
F: __dxlog_file8,
|
|
1297
|
+
L: 401,
|
|
1298
|
+
S: this,
|
|
1299
|
+
C: (f, a) => f(...a)
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1307
1302
|
});
|
|
1303
|
+
this._downloads.set(feed, handle);
|
|
1308
1304
|
}
|
|
1309
1305
|
async _initIterator() {
|
|
1310
1306
|
this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
|
|
@@ -1314,7 +1310,7 @@ var Pipeline = class {
|
|
|
1314
1310
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
1315
1311
|
import_log7.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
1316
1312
|
F: __dxlog_file8,
|
|
1317
|
-
L:
|
|
1313
|
+
L: 415,
|
|
1318
1314
|
S: this,
|
|
1319
1315
|
C: (f, a) => f(...a)
|
|
1320
1316
|
});
|
|
@@ -1342,12 +1338,24 @@ _ts_decorate4([
|
|
|
1342
1338
|
], Pipeline.prototype, "unpause", null);
|
|
1343
1339
|
|
|
1344
1340
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
1341
|
+
function _ts_decorate5(decorators, target, key, desc) {
|
|
1342
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1343
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1344
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
1345
|
+
else
|
|
1346
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
1347
|
+
if (d = decorators[i])
|
|
1348
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1349
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1350
|
+
}
|
|
1345
1351
|
var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
|
|
1346
1352
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
1347
|
-
var ControlPipeline = class {
|
|
1353
|
+
var ControlPipeline = class ControlPipeline2 {
|
|
1348
1354
|
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
1349
1355
|
this._lastTimeframeSaveTime = Date.now();
|
|
1350
1356
|
this.onFeedAdmitted = new import_util5.Callback();
|
|
1357
|
+
this._usage = new import_tracing.TimeUsageCounter();
|
|
1358
|
+
this._mutations = new import_tracing.TimeSeriesCounter();
|
|
1351
1359
|
this._spaceKey = spaceKey;
|
|
1352
1360
|
this._metadata = metadataStore;
|
|
1353
1361
|
this._pipeline = new Pipeline();
|
|
@@ -1358,7 +1366,7 @@ var ControlPipeline = class {
|
|
|
1358
1366
|
key: info.key
|
|
1359
1367
|
}, {
|
|
1360
1368
|
F: __dxlog_file9,
|
|
1361
|
-
L:
|
|
1369
|
+
L: 61,
|
|
1362
1370
|
S: this,
|
|
1363
1371
|
C: (f, a) => f(...a)
|
|
1364
1372
|
});
|
|
@@ -1369,7 +1377,7 @@ var ControlPipeline = class {
|
|
|
1369
1377
|
} catch (err) {
|
|
1370
1378
|
import_log8.log.catch(err, void 0, {
|
|
1371
1379
|
F: __dxlog_file9,
|
|
1372
|
-
L:
|
|
1380
|
+
L: 69,
|
|
1373
1381
|
S: this,
|
|
1374
1382
|
C: (f, a) => f(...a)
|
|
1375
1383
|
});
|
|
@@ -1392,56 +1400,65 @@ var ControlPipeline = class {
|
|
|
1392
1400
|
async start() {
|
|
1393
1401
|
(0, import_log8.log)("starting...", void 0, {
|
|
1394
1402
|
F: __dxlog_file9,
|
|
1395
|
-
L:
|
|
1403
|
+
L: 93,
|
|
1396
1404
|
S: this,
|
|
1397
1405
|
C: (f, a) => f(...a)
|
|
1398
1406
|
});
|
|
1399
1407
|
setTimeout(async () => {
|
|
1400
|
-
|
|
1401
|
-
try {
|
|
1402
|
-
(0, import_log8.log)("processing", {
|
|
1403
|
-
key: msg.feedKey,
|
|
1404
|
-
seq: msg.seq
|
|
1405
|
-
}, {
|
|
1406
|
-
F: __dxlog_file9,
|
|
1407
|
-
L: 88,
|
|
1408
|
-
S: this,
|
|
1409
|
-
C: (f, a) => f(...a)
|
|
1410
|
-
});
|
|
1411
|
-
if (msg.data.payload.credential) {
|
|
1412
|
-
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1413
|
-
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys4.PublicKey.from(msg.feedKey));
|
|
1414
|
-
timer.end();
|
|
1415
|
-
if (!result) {
|
|
1416
|
-
import_log8.log.warn("processing failed", {
|
|
1417
|
-
msg
|
|
1418
|
-
}, {
|
|
1419
|
-
F: __dxlog_file9,
|
|
1420
|
-
L: 98,
|
|
1421
|
-
S: this,
|
|
1422
|
-
C: (f, a) => f(...a)
|
|
1423
|
-
});
|
|
1424
|
-
} else {
|
|
1425
|
-
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1426
|
-
}
|
|
1427
|
-
}
|
|
1428
|
-
} catch (err) {
|
|
1429
|
-
import_log8.log.catch(err, void 0, {
|
|
1430
|
-
F: __dxlog_file9,
|
|
1431
|
-
L: 104,
|
|
1432
|
-
S: this,
|
|
1433
|
-
C: (f, a) => f(...a)
|
|
1434
|
-
});
|
|
1435
|
-
}
|
|
1436
|
-
}
|
|
1408
|
+
void this._consumePipeline(new import_context5.Context());
|
|
1437
1409
|
});
|
|
1438
1410
|
await this._pipeline.start();
|
|
1439
1411
|
(0, import_log8.log)("started", void 0, {
|
|
1440
1412
|
F: __dxlog_file9,
|
|
1441
|
-
L:
|
|
1413
|
+
L: 99,
|
|
1414
|
+
S: this,
|
|
1415
|
+
C: (f, a) => f(...a)
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
async _consumePipeline(ctx) {
|
|
1419
|
+
for await (const msg of this._pipeline.consume()) {
|
|
1420
|
+
const span = this._usage.beginRecording();
|
|
1421
|
+
this._mutations.inc();
|
|
1422
|
+
try {
|
|
1423
|
+
await this._processMessage(ctx, msg);
|
|
1424
|
+
} catch (err) {
|
|
1425
|
+
import_log8.log.catch(err, void 0, {
|
|
1426
|
+
F: __dxlog_file9,
|
|
1427
|
+
L: 111,
|
|
1428
|
+
S: this,
|
|
1429
|
+
C: (f, a) => f(...a)
|
|
1430
|
+
});
|
|
1431
|
+
}
|
|
1432
|
+
span.end();
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
async _processMessage(ctx, msg) {
|
|
1436
|
+
(0, import_log8.log)("processing", {
|
|
1437
|
+
key: msg.feedKey,
|
|
1438
|
+
seq: msg.seq
|
|
1439
|
+
}, {
|
|
1440
|
+
F: __dxlog_file9,
|
|
1441
|
+
L: 121,
|
|
1442
1442
|
S: this,
|
|
1443
1443
|
C: (f, a) => f(...a)
|
|
1444
1444
|
});
|
|
1445
|
+
if (msg.data.payload.credential) {
|
|
1446
|
+
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1447
|
+
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys4.PublicKey.from(msg.feedKey));
|
|
1448
|
+
timer.end();
|
|
1449
|
+
if (!result) {
|
|
1450
|
+
import_log8.log.warn("processing failed", {
|
|
1451
|
+
msg
|
|
1452
|
+
}, {
|
|
1453
|
+
F: __dxlog_file9,
|
|
1454
|
+
L: 131,
|
|
1455
|
+
S: this,
|
|
1456
|
+
C: (f, a) => f(...a)
|
|
1457
|
+
});
|
|
1458
|
+
} else {
|
|
1459
|
+
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1445
1462
|
}
|
|
1446
1463
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
1447
1464
|
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
|
|
@@ -1452,7 +1469,7 @@ var ControlPipeline = class {
|
|
|
1452
1469
|
async stop() {
|
|
1453
1470
|
(0, import_log8.log)("stopping...", void 0, {
|
|
1454
1471
|
F: __dxlog_file9,
|
|
1455
|
-
L:
|
|
1472
|
+
L: 149,
|
|
1456
1473
|
S: this,
|
|
1457
1474
|
C: (f, a) => f(...a)
|
|
1458
1475
|
});
|
|
@@ -1460,39 +1477,54 @@ var ControlPipeline = class {
|
|
|
1460
1477
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1461
1478
|
(0, import_log8.log)("stopped", void 0, {
|
|
1462
1479
|
F: __dxlog_file9,
|
|
1463
|
-
L:
|
|
1480
|
+
L: 152,
|
|
1464
1481
|
S: this,
|
|
1465
1482
|
C: (f, a) => f(...a)
|
|
1466
1483
|
});
|
|
1467
1484
|
}
|
|
1468
1485
|
async _saveTargetTimeframe(timeframe) {
|
|
1469
|
-
var _a;
|
|
1470
1486
|
try {
|
|
1471
|
-
const newTimeframe = import_timeframe3.Timeframe.merge(
|
|
1487
|
+
const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
|
|
1472
1488
|
await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
|
|
1473
1489
|
this._targetTimeframe = newTimeframe;
|
|
1474
1490
|
} catch (err) {
|
|
1475
1491
|
(0, import_log8.log)(err, void 0, {
|
|
1476
1492
|
F: __dxlog_file9,
|
|
1477
|
-
L:
|
|
1493
|
+
L: 161,
|
|
1478
1494
|
S: this,
|
|
1479
1495
|
C: (f, a) => f(...a)
|
|
1480
1496
|
});
|
|
1481
1497
|
}
|
|
1482
1498
|
}
|
|
1483
1499
|
};
|
|
1500
|
+
_ts_decorate5([
|
|
1501
|
+
import_tracing.trace.metricsCounter()
|
|
1502
|
+
], ControlPipeline.prototype, "_usage", void 0);
|
|
1503
|
+
_ts_decorate5([
|
|
1504
|
+
import_tracing.trace.metricsCounter()
|
|
1505
|
+
], ControlPipeline.prototype, "_mutations", void 0);
|
|
1506
|
+
_ts_decorate5([
|
|
1507
|
+
import_tracing.trace.span()
|
|
1508
|
+
], ControlPipeline.prototype, "_consumePipeline", null);
|
|
1509
|
+
_ts_decorate5([
|
|
1510
|
+
import_tracing.trace.span()
|
|
1511
|
+
], ControlPipeline.prototype, "_processMessage", null);
|
|
1512
|
+
ControlPipeline = _ts_decorate5([
|
|
1513
|
+
import_tracing.trace.resource()
|
|
1514
|
+
], ControlPipeline);
|
|
1484
1515
|
|
|
1485
1516
|
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1486
1517
|
var import_async6 = require("@dxos/async");
|
|
1487
|
-
var
|
|
1518
|
+
var import_context6 = require("@dxos/context");
|
|
1488
1519
|
var import_credentials3 = require("@dxos/credentials");
|
|
1489
1520
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
1490
1521
|
var import_errors2 = require("@dxos/errors");
|
|
1491
1522
|
var import_invariant8 = require("@dxos/invariant");
|
|
1492
1523
|
var import_log9 = require("@dxos/log");
|
|
1493
1524
|
var import_timeframe4 = require("@dxos/timeframe");
|
|
1525
|
+
var import_tracing2 = require("@dxos/tracing");
|
|
1494
1526
|
var import_util6 = require("@dxos/util");
|
|
1495
|
-
function
|
|
1527
|
+
function _ts_decorate6(decorators, target, key, desc) {
|
|
1496
1528
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1497
1529
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1498
1530
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1509,7 +1541,7 @@ var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
|
1509
1541
|
var DataPipeline = class DataPipeline2 {
|
|
1510
1542
|
constructor(_params) {
|
|
1511
1543
|
this._params = _params;
|
|
1512
|
-
this._ctx = new
|
|
1544
|
+
this._ctx = new import_context6.Context();
|
|
1513
1545
|
this._pipeline = void 0;
|
|
1514
1546
|
this._targetTimeframe = void 0;
|
|
1515
1547
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
@@ -1517,6 +1549,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1517
1549
|
this._lastTimeframeSaveTime = 0;
|
|
1518
1550
|
this._lastSnapshotSaveTime = 0;
|
|
1519
1551
|
this._lastProcessedEpoch = -1;
|
|
1552
|
+
this._usage = new import_tracing2.TimeUsageCounter();
|
|
1553
|
+
this._mutations = new import_tracing2.TimeSeriesCounter();
|
|
1520
1554
|
this.currentEpoch = void 0;
|
|
1521
1555
|
this.appliedEpoch = void 0;
|
|
1522
1556
|
this.onNewEpoch = new import_async6.Event();
|
|
@@ -1528,13 +1562,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1528
1562
|
return this._pipeline;
|
|
1529
1563
|
}
|
|
1530
1564
|
get pipelineState() {
|
|
1531
|
-
|
|
1532
|
-
return (_a = this._pipeline) == null ? void 0 : _a.state;
|
|
1565
|
+
return this._pipeline?.state;
|
|
1533
1566
|
}
|
|
1534
1567
|
setTargetTimeframe(timeframe) {
|
|
1535
|
-
var _a;
|
|
1536
1568
|
this._targetTimeframe = timeframe;
|
|
1537
|
-
|
|
1569
|
+
this._pipeline?.state.setTargetTimeframe(timeframe);
|
|
1538
1570
|
}
|
|
1539
1571
|
async processCredential(credential) {
|
|
1540
1572
|
if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
|
|
@@ -1551,8 +1583,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1551
1583
|
}
|
|
1552
1584
|
this._pipeline = new Pipeline();
|
|
1553
1585
|
await this._params.onPipelineCreated(this._pipeline);
|
|
1554
|
-
await this._pipeline.start();
|
|
1555
1586
|
await this._pipeline.pause();
|
|
1587
|
+
await this._pipeline.start();
|
|
1556
1588
|
if (this._targetTimeframe) {
|
|
1557
1589
|
this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
|
|
1558
1590
|
}
|
|
@@ -1560,7 +1592,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1560
1592
|
write: (data, options) => {
|
|
1561
1593
|
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1562
1594
|
F: __dxlog_file10,
|
|
1563
|
-
L:
|
|
1595
|
+
L: 154,
|
|
1564
1596
|
S: this,
|
|
1565
1597
|
A: [
|
|
1566
1598
|
"this._pipeline",
|
|
@@ -1569,7 +1601,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1569
1601
|
});
|
|
1570
1602
|
(0, import_invariant8.invariant)(this.currentEpoch, "Epoch is not initialized.", {
|
|
1571
1603
|
F: __dxlog_file10,
|
|
1572
|
-
L:
|
|
1604
|
+
L: 155,
|
|
1573
1605
|
S: this,
|
|
1574
1606
|
A: [
|
|
1575
1607
|
"this.currentEpoch",
|
|
@@ -1590,19 +1622,18 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1590
1622
|
this._isOpen = true;
|
|
1591
1623
|
}
|
|
1592
1624
|
async close() {
|
|
1593
|
-
var _a, _b, _c;
|
|
1594
1625
|
if (!this._isOpen) {
|
|
1595
1626
|
return;
|
|
1596
1627
|
}
|
|
1597
1628
|
(0, import_log9.log)("close", void 0, {
|
|
1598
1629
|
F: __dxlog_file10,
|
|
1599
|
-
L:
|
|
1630
|
+
L: 179,
|
|
1600
1631
|
S: this,
|
|
1601
1632
|
C: (f, a) => f(...a)
|
|
1602
1633
|
});
|
|
1603
1634
|
this._isOpen = false;
|
|
1604
1635
|
await this._ctx.dispose();
|
|
1605
|
-
await
|
|
1636
|
+
await this._pipeline?.stop();
|
|
1606
1637
|
try {
|
|
1607
1638
|
await this._saveCache();
|
|
1608
1639
|
if (this._pipeline) {
|
|
@@ -1611,14 +1642,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1611
1642
|
} catch (err) {
|
|
1612
1643
|
import_log9.log.catch(err, void 0, {
|
|
1613
1644
|
F: __dxlog_file10,
|
|
1614
|
-
L:
|
|
1645
|
+
L: 192,
|
|
1615
1646
|
S: this,
|
|
1616
1647
|
C: (f, a) => f(...a)
|
|
1617
1648
|
});
|
|
1618
1649
|
}
|
|
1619
|
-
await
|
|
1620
|
-
await
|
|
1621
|
-
this._ctx = new
|
|
1650
|
+
await this.databaseHost?.close();
|
|
1651
|
+
await this.itemManager?.destroy();
|
|
1652
|
+
this._ctx = new import_context6.Context();
|
|
1622
1653
|
this._pipeline = void 0;
|
|
1623
1654
|
this._targetTimeframe = void 0;
|
|
1624
1655
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
@@ -1635,7 +1666,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1635
1666
|
}
|
|
1636
1667
|
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1637
1668
|
F: __dxlog_file10,
|
|
1638
|
-
L:
|
|
1669
|
+
L: 215,
|
|
1639
1670
|
S: this,
|
|
1640
1671
|
A: [
|
|
1641
1672
|
"this._pipeline",
|
|
@@ -1643,13 +1674,15 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1643
1674
|
]
|
|
1644
1675
|
});
|
|
1645
1676
|
for await (const msg of this._pipeline.consume()) {
|
|
1677
|
+
const span = this._usage.beginRecording();
|
|
1678
|
+
this._mutations.inc();
|
|
1646
1679
|
const { feedKey, seq, data } = msg;
|
|
1647
1680
|
(0, import_log9.log)("processing message", {
|
|
1648
1681
|
feedKey,
|
|
1649
1682
|
seq
|
|
1650
1683
|
}, {
|
|
1651
1684
|
F: __dxlog_file10,
|
|
1652
|
-
L:
|
|
1685
|
+
L: 221,
|
|
1653
1686
|
S: this,
|
|
1654
1687
|
C: (f, a) => f(...a)
|
|
1655
1688
|
});
|
|
@@ -1661,7 +1694,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1661
1694
|
feedKey
|
|
1662
1695
|
}, {
|
|
1663
1696
|
F: __dxlog_file10,
|
|
1664
|
-
L:
|
|
1697
|
+
L: 227,
|
|
1665
1698
|
S: this,
|
|
1666
1699
|
C: (f, a) => f(...a)
|
|
1667
1700
|
});
|
|
@@ -1684,7 +1717,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1684
1717
|
spaceKey: this._params.spaceKey.toHex()
|
|
1685
1718
|
}, {
|
|
1686
1719
|
F: __dxlog_file10,
|
|
1687
|
-
L:
|
|
1720
|
+
L: 244,
|
|
1688
1721
|
S: this,
|
|
1689
1722
|
C: (f, a) => f(...a)
|
|
1690
1723
|
});
|
|
@@ -1693,17 +1726,18 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1693
1726
|
} catch (err) {
|
|
1694
1727
|
import_log9.log.catch(err, void 0, {
|
|
1695
1728
|
F: __dxlog_file10,
|
|
1696
|
-
L:
|
|
1729
|
+
L: 254,
|
|
1697
1730
|
S: this,
|
|
1698
1731
|
C: (f, a) => f(...a)
|
|
1699
1732
|
});
|
|
1700
1733
|
}
|
|
1734
|
+
span.end();
|
|
1701
1735
|
}
|
|
1702
1736
|
}
|
|
1703
1737
|
_createSnapshot() {
|
|
1704
1738
|
(0, import_invariant8.invariant)(this.databaseHost, "Database backend is not initialized.", {
|
|
1705
1739
|
F: __dxlog_file10,
|
|
1706
|
-
L:
|
|
1740
|
+
L: 262,
|
|
1707
1741
|
S: this,
|
|
1708
1742
|
A: [
|
|
1709
1743
|
"this.databaseHost",
|
|
@@ -1717,26 +1751,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1717
1751
|
};
|
|
1718
1752
|
}
|
|
1719
1753
|
async _saveTargetTimeframe(timeframe) {
|
|
1720
|
-
|
|
1721
|
-
const newTimeframe = import_timeframe4.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe4.Timeframe(), timeframe);
|
|
1754
|
+
const newTimeframe = import_timeframe4.Timeframe.merge(this._targetTimeframe ?? new import_timeframe4.Timeframe(), timeframe);
|
|
1722
1755
|
await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
|
|
1723
1756
|
this._targetTimeframe = newTimeframe;
|
|
1724
1757
|
}
|
|
1725
1758
|
async _saveCache() {
|
|
1726
|
-
var _a;
|
|
1727
1759
|
const cache = {};
|
|
1728
1760
|
try {
|
|
1729
|
-
const propertiesItem = this.itemManager.items.find((item) =>
|
|
1730
|
-
|
|
1731
|
-
return ((_a2 = item.modelMeta) == null ? void 0 : _a2.type) === "dxos:model/document" && ((_b = (0, import_echo_db3.getStateMachineFromItem)(item)) == null ? void 0 : _b.snapshot()).type === "dxos.sdk.client.Properties";
|
|
1732
|
-
});
|
|
1761
|
+
const propertiesItem = this.itemManager.items.find((item) => item.modelMeta?.type === "dxos:model/document" && // TODO(burdon): Document?
|
|
1762
|
+
((0, import_echo_db3.getStateMachineFromItem)(item)?.snapshot()).type === import_echo_db3.TYPE_PROPERTIES);
|
|
1733
1763
|
if (propertiesItem) {
|
|
1734
|
-
cache.properties = (
|
|
1764
|
+
cache.properties = (0, import_echo_db3.getStateMachineFromItem)(propertiesItem)?.snapshot();
|
|
1735
1765
|
}
|
|
1736
1766
|
} catch (err) {
|
|
1737
1767
|
import_log9.log.warn("Failed to cache properties", err, {
|
|
1738
1768
|
F: __dxlog_file10,
|
|
1739
|
-
L:
|
|
1769
|
+
L: 291,
|
|
1740
1770
|
S: this,
|
|
1741
1771
|
C: (f, a) => f(...a)
|
|
1742
1772
|
});
|
|
@@ -1753,24 +1783,23 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1753
1783
|
}
|
|
1754
1784
|
}
|
|
1755
1785
|
async _processEpochInSeparateTask(epoch) {
|
|
1756
|
-
var _a;
|
|
1757
1786
|
if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
|
|
1758
1787
|
return;
|
|
1759
1788
|
}
|
|
1760
|
-
await
|
|
1761
|
-
const ctx = new
|
|
1789
|
+
await this._epochCtx?.dispose();
|
|
1790
|
+
const ctx = new import_context6.Context({
|
|
1762
1791
|
onError: (err) => {
|
|
1763
1792
|
if (err instanceof import_errors2.CancelledError) {
|
|
1764
1793
|
(0, import_log9.log)("Epoch processing cancelled.", void 0, {
|
|
1765
1794
|
F: __dxlog_file10,
|
|
1766
|
-
L:
|
|
1795
|
+
L: 323,
|
|
1767
1796
|
S: this,
|
|
1768
1797
|
C: (f, a) => f(...a)
|
|
1769
1798
|
});
|
|
1770
1799
|
} else {
|
|
1771
1800
|
import_log9.log.catch(err, void 0, {
|
|
1772
1801
|
F: __dxlog_file10,
|
|
1773
|
-
L:
|
|
1802
|
+
L: 325,
|
|
1774
1803
|
S: this,
|
|
1775
1804
|
C: (f, a) => f(...a)
|
|
1776
1805
|
});
|
|
@@ -1782,14 +1811,6 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1782
1811
|
if (!this._isOpen) {
|
|
1783
1812
|
return;
|
|
1784
1813
|
}
|
|
1785
|
-
(0, import_log9.log)("process epoch", {
|
|
1786
|
-
epoch
|
|
1787
|
-
}, {
|
|
1788
|
-
F: __dxlog_file10,
|
|
1789
|
-
L: 321,
|
|
1790
|
-
S: this,
|
|
1791
|
-
C: (f, a) => f(...a)
|
|
1792
|
-
});
|
|
1793
1814
|
await this._processEpoch(ctx, epoch.subject.assertion);
|
|
1794
1815
|
this.appliedEpoch = epoch;
|
|
1795
1816
|
this.onNewEpoch.emit(epoch);
|
|
@@ -1798,7 +1819,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1798
1819
|
async _processEpoch(ctx, epoch) {
|
|
1799
1820
|
(0, import_invariant8.invariant)(this._isOpen, "Space is closed.", {
|
|
1800
1821
|
F: __dxlog_file10,
|
|
1801
|
-
L:
|
|
1822
|
+
L: 344,
|
|
1802
1823
|
S: this,
|
|
1803
1824
|
A: [
|
|
1804
1825
|
"this._isOpen",
|
|
@@ -1807,7 +1828,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1807
1828
|
});
|
|
1808
1829
|
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1809
1830
|
F: __dxlog_file10,
|
|
1810
|
-
L:
|
|
1831
|
+
L: 345,
|
|
1811
1832
|
S: this,
|
|
1812
1833
|
A: [
|
|
1813
1834
|
"this._pipeline",
|
|
@@ -1815,11 +1836,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1815
1836
|
]
|
|
1816
1837
|
});
|
|
1817
1838
|
this._lastProcessedEpoch = epoch.number;
|
|
1818
|
-
(0, import_log9.log)("
|
|
1819
|
-
epoch
|
|
1839
|
+
(0, import_log9.log)("processing", {
|
|
1840
|
+
epoch: (0, import_log9.omit)(epoch, "proof")
|
|
1820
1841
|
}, {
|
|
1821
1842
|
F: __dxlog_file10,
|
|
1822
|
-
L:
|
|
1843
|
+
L: 348,
|
|
1823
1844
|
S: this,
|
|
1824
1845
|
C: (f, a) => f(...a)
|
|
1825
1846
|
});
|
|
@@ -1827,9 +1848,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1827
1848
|
const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
|
|
1828
1849
|
this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
|
|
1829
1850
|
}
|
|
1830
|
-
(0, import_log9.log)("restarting pipeline
|
|
1851
|
+
(0, import_log9.log)("restarting pipeline from epoch", void 0, {
|
|
1831
1852
|
F: __dxlog_file10,
|
|
1832
|
-
L:
|
|
1853
|
+
L: 354,
|
|
1833
1854
|
S: this,
|
|
1834
1855
|
C: (f, a) => f(...a)
|
|
1835
1856
|
});
|
|
@@ -1840,7 +1861,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1840
1861
|
async waitUntilTimeframe(timeframe) {
|
|
1841
1862
|
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1842
1863
|
F: __dxlog_file10,
|
|
1843
|
-
L:
|
|
1864
|
+
L: 361,
|
|
1844
1865
|
S: this,
|
|
1845
1866
|
A: [
|
|
1846
1867
|
"this._pipeline",
|
|
@@ -1852,7 +1873,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1852
1873
|
async createEpoch() {
|
|
1853
1874
|
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1854
1875
|
F: __dxlog_file10,
|
|
1855
|
-
L:
|
|
1876
|
+
L: 367,
|
|
1856
1877
|
S: this,
|
|
1857
1878
|
A: [
|
|
1858
1879
|
"this._pipeline",
|
|
@@ -1861,7 +1882,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1861
1882
|
});
|
|
1862
1883
|
(0, import_invariant8.invariant)(this.currentEpoch, void 0, {
|
|
1863
1884
|
F: __dxlog_file10,
|
|
1864
|
-
L:
|
|
1885
|
+
L: 368,
|
|
1865
1886
|
S: this,
|
|
1866
1887
|
A: [
|
|
1867
1888
|
"this.currentEpoch",
|
|
@@ -1884,24 +1905,31 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1884
1905
|
await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
|
|
1885
1906
|
}
|
|
1886
1907
|
};
|
|
1887
|
-
|
|
1908
|
+
_ts_decorate6([
|
|
1909
|
+
import_tracing2.trace.metricsCounter()
|
|
1910
|
+
], DataPipeline.prototype, "_usage", void 0);
|
|
1911
|
+
_ts_decorate6([
|
|
1912
|
+
import_tracing2.trace.metricsCounter()
|
|
1913
|
+
], DataPipeline.prototype, "_mutations", void 0);
|
|
1914
|
+
_ts_decorate6([
|
|
1888
1915
|
import_async6.synchronized
|
|
1889
1916
|
], DataPipeline.prototype, "open", null);
|
|
1890
|
-
|
|
1917
|
+
_ts_decorate6([
|
|
1891
1918
|
import_async6.synchronized
|
|
1892
1919
|
], DataPipeline.prototype, "close", null);
|
|
1893
|
-
|
|
1920
|
+
_ts_decorate6([
|
|
1894
1921
|
import_async6.synchronized
|
|
1895
1922
|
], DataPipeline.prototype, "_processEpoch", null);
|
|
1896
|
-
|
|
1923
|
+
_ts_decorate6([
|
|
1897
1924
|
import_async6.synchronized
|
|
1898
1925
|
], DataPipeline.prototype, "createEpoch", null);
|
|
1899
|
-
DataPipeline =
|
|
1900
|
-
(0, import_async6.trackLeaks)("open", "close")
|
|
1926
|
+
DataPipeline = _ts_decorate6([
|
|
1927
|
+
(0, import_async6.trackLeaks)("open", "close"),
|
|
1928
|
+
import_tracing2.trace.resource()
|
|
1901
1929
|
], DataPipeline);
|
|
1902
1930
|
|
|
1903
1931
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
1904
|
-
function
|
|
1932
|
+
function _ts_decorate7(decorators, target, key, desc) {
|
|
1905
1933
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1906
1934
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1907
1935
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2004,12 +2032,10 @@ var Space = class Space2 {
|
|
|
2004
2032
|
return this._genesisFeedKey;
|
|
2005
2033
|
}
|
|
2006
2034
|
get controlFeedKey() {
|
|
2007
|
-
|
|
2008
|
-
return (_a = this._controlFeed) == null ? void 0 : _a.key;
|
|
2035
|
+
return this._controlFeed?.key;
|
|
2009
2036
|
}
|
|
2010
2037
|
get dataFeedKey() {
|
|
2011
|
-
|
|
2012
|
-
return (_a = this._dataFeed) == null ? void 0 : _a.key;
|
|
2038
|
+
return this._dataFeed?.key;
|
|
2013
2039
|
}
|
|
2014
2040
|
get spaceState() {
|
|
2015
2041
|
return this._controlPipeline.spaceState;
|
|
@@ -2029,7 +2055,7 @@ var Space = class Space2 {
|
|
|
2029
2055
|
setControlFeed(feed) {
|
|
2030
2056
|
(0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
|
|
2031
2057
|
F: __dxlog_file11,
|
|
2032
|
-
L:
|
|
2058
|
+
L: 184,
|
|
2033
2059
|
S: this,
|
|
2034
2060
|
A: [
|
|
2035
2061
|
"!this._controlFeed",
|
|
@@ -2041,10 +2067,9 @@ var Space = class Space2 {
|
|
|
2041
2067
|
return this;
|
|
2042
2068
|
}
|
|
2043
2069
|
setDataFeed(feed) {
|
|
2044
|
-
var _a;
|
|
2045
2070
|
(0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
|
|
2046
2071
|
F: __dxlog_file11,
|
|
2047
|
-
L:
|
|
2072
|
+
L: 191,
|
|
2048
2073
|
S: this,
|
|
2049
2074
|
A: [
|
|
2050
2075
|
"!this._dataFeed",
|
|
@@ -2052,7 +2077,7 @@ var Space = class Space2 {
|
|
|
2052
2077
|
]
|
|
2053
2078
|
});
|
|
2054
2079
|
this._dataFeed = feed;
|
|
2055
|
-
|
|
2080
|
+
this._dataPipeline.pipeline?.setWriteFeed(feed);
|
|
2056
2081
|
return this;
|
|
2057
2082
|
}
|
|
2058
2083
|
/**
|
|
@@ -2070,7 +2095,7 @@ var Space = class Space2 {
|
|
|
2070
2095
|
async open(ctx) {
|
|
2071
2096
|
(0, import_log10.log)("opening...", void 0, {
|
|
2072
2097
|
F: __dxlog_file11,
|
|
2073
|
-
L:
|
|
2098
|
+
L: 213,
|
|
2074
2099
|
S: this,
|
|
2075
2100
|
C: (f, a) => f(...a)
|
|
2076
2101
|
});
|
|
@@ -2083,7 +2108,7 @@ var Space = class Space2 {
|
|
|
2083
2108
|
this._isOpen = true;
|
|
2084
2109
|
(0, import_log10.log)("opened", void 0, {
|
|
2085
2110
|
F: __dxlog_file11,
|
|
2086
|
-
L:
|
|
2111
|
+
L: 224,
|
|
2087
2112
|
S: this,
|
|
2088
2113
|
C: (f, a) => f(...a)
|
|
2089
2114
|
});
|
|
@@ -2093,7 +2118,7 @@ var Space = class Space2 {
|
|
|
2093
2118
|
key: this._key
|
|
2094
2119
|
}, {
|
|
2095
2120
|
F: __dxlog_file11,
|
|
2096
|
-
L:
|
|
2121
|
+
L: 229,
|
|
2097
2122
|
S: this,
|
|
2098
2123
|
C: (f, a) => f(...a)
|
|
2099
2124
|
});
|
|
@@ -2107,7 +2132,7 @@ var Space = class Space2 {
|
|
|
2107
2132
|
this._isOpen = false;
|
|
2108
2133
|
(0, import_log10.log)("closed", void 0, {
|
|
2109
2134
|
F: __dxlog_file11,
|
|
2110
|
-
L:
|
|
2135
|
+
L: 242,
|
|
2111
2136
|
S: this,
|
|
2112
2137
|
C: (f, a) => f(...a)
|
|
2113
2138
|
});
|
|
@@ -2115,13 +2140,13 @@ var Space = class Space2 {
|
|
|
2115
2140
|
async initializeDataPipeline() {
|
|
2116
2141
|
(0, import_log10.log)("initializeDataPipeline", void 0, {
|
|
2117
2142
|
F: __dxlog_file11,
|
|
2118
|
-
L:
|
|
2143
|
+
L: 247,
|
|
2119
2144
|
S: this,
|
|
2120
2145
|
C: (f, a) => f(...a)
|
|
2121
2146
|
});
|
|
2122
2147
|
(0, import_invariant9.invariant)(this._isOpen, "Space must be open to initialize data pipeline.", {
|
|
2123
2148
|
F: __dxlog_file11,
|
|
2124
|
-
L:
|
|
2149
|
+
L: 248,
|
|
2125
2150
|
S: this,
|
|
2126
2151
|
A: [
|
|
2127
2152
|
"this._isOpen",
|
|
@@ -2131,22 +2156,23 @@ var Space = class Space2 {
|
|
|
2131
2156
|
await this._dataPipeline.open();
|
|
2132
2157
|
}
|
|
2133
2158
|
};
|
|
2134
|
-
|
|
2135
|
-
import_log10.logInfo
|
|
2159
|
+
_ts_decorate7([
|
|
2160
|
+
import_log10.logInfo,
|
|
2161
|
+
import_tracing3.trace.info()
|
|
2136
2162
|
], Space.prototype, "key", null);
|
|
2137
|
-
|
|
2163
|
+
_ts_decorate7([
|
|
2138
2164
|
import_async7.synchronized,
|
|
2139
|
-
|
|
2165
|
+
import_tracing3.trace.span()
|
|
2140
2166
|
], Space.prototype, "open", null);
|
|
2141
|
-
|
|
2167
|
+
_ts_decorate7([
|
|
2142
2168
|
import_async7.synchronized
|
|
2143
2169
|
], Space.prototype, "close", null);
|
|
2144
|
-
|
|
2170
|
+
_ts_decorate7([
|
|
2145
2171
|
import_async7.synchronized
|
|
2146
2172
|
], Space.prototype, "initializeDataPipeline", null);
|
|
2147
|
-
Space =
|
|
2173
|
+
Space = _ts_decorate7([
|
|
2148
2174
|
(0, import_async7.trackLeaks)("open", "close"),
|
|
2149
|
-
|
|
2175
|
+
import_tracing3.trace.resource()
|
|
2150
2176
|
], Space);
|
|
2151
2177
|
|
|
2152
2178
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
@@ -2166,7 +2192,7 @@ var import_teleport2 = require("@dxos/teleport");
|
|
|
2166
2192
|
var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
|
|
2167
2193
|
var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
|
|
2168
2194
|
var import_util8 = require("@dxos/util");
|
|
2169
|
-
function
|
|
2195
|
+
function _ts_decorate8(decorators, target, key, desc) {
|
|
2170
2196
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2171
2197
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
2172
2198
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2207,7 +2233,7 @@ var SpaceProtocol = class {
|
|
|
2207
2233
|
key: feed.key
|
|
2208
2234
|
}, {
|
|
2209
2235
|
F: __dxlog_file12,
|
|
2210
|
-
L:
|
|
2236
|
+
L: 96,
|
|
2211
2237
|
S: this,
|
|
2212
2238
|
C: (f, a) => f(...a)
|
|
2213
2239
|
});
|
|
@@ -2216,6 +2242,7 @@ var SpaceProtocol = class {
|
|
|
2216
2242
|
session.replicator.addFeed(feed);
|
|
2217
2243
|
}
|
|
2218
2244
|
}
|
|
2245
|
+
// TODO(burdon): Rename open? Common open/close interfaces for all services?
|
|
2219
2246
|
async start() {
|
|
2220
2247
|
if (this._connection) {
|
|
2221
2248
|
return;
|
|
@@ -2229,7 +2256,7 @@ var SpaceProtocol = class {
|
|
|
2229
2256
|
await this.blobSync.open();
|
|
2230
2257
|
(0, import_log11.log)("starting...", void 0, {
|
|
2231
2258
|
F: __dxlog_file12,
|
|
2232
|
-
L:
|
|
2259
|
+
L: 122,
|
|
2233
2260
|
S: this,
|
|
2234
2261
|
C: (f, a) => f(...a)
|
|
2235
2262
|
});
|
|
@@ -2239,11 +2266,11 @@ var SpaceProtocol = class {
|
|
|
2239
2266
|
peerId: this._swarmIdentity.peerKey,
|
|
2240
2267
|
topic,
|
|
2241
2268
|
topology: new import_network_manager.MMSTTopology(topologyConfig),
|
|
2242
|
-
label: `
|
|
2269
|
+
label: `space swarm ${topic.truncate()}`
|
|
2243
2270
|
});
|
|
2244
2271
|
(0, import_log11.log)("started", void 0, {
|
|
2245
2272
|
F: __dxlog_file12,
|
|
2246
|
-
L:
|
|
2273
|
+
L: 132,
|
|
2247
2274
|
S: this,
|
|
2248
2275
|
C: (f, a) => f(...a)
|
|
2249
2276
|
});
|
|
@@ -2253,14 +2280,14 @@ var SpaceProtocol = class {
|
|
|
2253
2280
|
if (this._connection) {
|
|
2254
2281
|
(0, import_log11.log)("stopping...", void 0, {
|
|
2255
2282
|
F: __dxlog_file12,
|
|
2256
|
-
L:
|
|
2283
|
+
L: 139,
|
|
2257
2284
|
S: this,
|
|
2258
2285
|
C: (f, a) => f(...a)
|
|
2259
2286
|
});
|
|
2260
2287
|
await this._connection.close();
|
|
2261
2288
|
(0, import_log11.log)("stopped", void 0, {
|
|
2262
2289
|
F: __dxlog_file12,
|
|
2263
|
-
L:
|
|
2290
|
+
L: 141,
|
|
2264
2291
|
S: this,
|
|
2265
2292
|
C: (f, a) => f(...a)
|
|
2266
2293
|
});
|
|
@@ -2283,10 +2310,10 @@ var SpaceProtocol = class {
|
|
|
2283
2310
|
};
|
|
2284
2311
|
}
|
|
2285
2312
|
};
|
|
2286
|
-
|
|
2313
|
+
_ts_decorate8([
|
|
2287
2314
|
import_log11.logInfo
|
|
2288
2315
|
], SpaceProtocol.prototype, "_topic", void 0);
|
|
2289
|
-
|
|
2316
|
+
_ts_decorate8([
|
|
2290
2317
|
import_log11.logInfo
|
|
2291
2318
|
], SpaceProtocol.prototype, "_ownPeerKey", null);
|
|
2292
2319
|
var AuthStatus;
|
|
@@ -2325,20 +2352,18 @@ var SpaceProtocolSession = class {
|
|
|
2325
2352
|
provider: this._swarmIdentity.credentialProvider,
|
|
2326
2353
|
verifier: this._swarmIdentity.credentialAuthenticator,
|
|
2327
2354
|
onAuthSuccess: () => {
|
|
2328
|
-
var _a;
|
|
2329
2355
|
(0, import_log11.log)("Peer authenticated", void 0, {
|
|
2330
2356
|
F: __dxlog_file12,
|
|
2331
|
-
L:
|
|
2357
|
+
L: 238,
|
|
2332
2358
|
S: this,
|
|
2333
2359
|
C: (f, a) => f(...a)
|
|
2334
2360
|
});
|
|
2335
2361
|
this._authStatus = AuthStatus.SUCCESS;
|
|
2336
|
-
|
|
2362
|
+
this._onSessionAuth?.(this._teleport);
|
|
2337
2363
|
},
|
|
2338
2364
|
onAuthFailure: () => {
|
|
2339
|
-
var _a;
|
|
2340
2365
|
this._authStatus = AuthStatus.FAILURE;
|
|
2341
|
-
|
|
2366
|
+
this._onAuthFailure?.(this._teleport);
|
|
2342
2367
|
}
|
|
2343
2368
|
}));
|
|
2344
2369
|
this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
|
|
@@ -2348,15 +2373,15 @@ var SpaceProtocolSession = class {
|
|
|
2348
2373
|
await this._teleport.close();
|
|
2349
2374
|
}
|
|
2350
2375
|
};
|
|
2351
|
-
|
|
2376
|
+
_ts_decorate8([
|
|
2352
2377
|
import_log11.logInfo
|
|
2353
2378
|
], SpaceProtocolSession.prototype, "_wireParams", void 0);
|
|
2354
|
-
|
|
2379
|
+
_ts_decorate8([
|
|
2355
2380
|
import_log11.logInfo
|
|
2356
2381
|
], SpaceProtocolSession.prototype, "authStatus", null);
|
|
2357
2382
|
|
|
2358
2383
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
2359
|
-
function
|
|
2384
|
+
function _ts_decorate9(decorators, target, key, desc) {
|
|
2360
2385
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2361
2386
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
2362
2387
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2390,7 +2415,6 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2390
2415
|
].map((space) => space.close()));
|
|
2391
2416
|
}
|
|
2392
2417
|
async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
|
|
2393
|
-
var _a;
|
|
2394
2418
|
import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.begin({
|
|
2395
2419
|
id: this._instanceId
|
|
2396
2420
|
}), {
|
|
@@ -2407,7 +2431,7 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2407
2431
|
S: this,
|
|
2408
2432
|
C: (f, a) => f(...a)
|
|
2409
2433
|
});
|
|
2410
|
-
const genesisFeed = await this._feedStore.openFeed(
|
|
2434
|
+
const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? (0, import_debug4.failUndefined)());
|
|
2411
2435
|
const spaceKey = metadata.key;
|
|
2412
2436
|
const protocol = new SpaceProtocol({
|
|
2413
2437
|
topic: spaceKey,
|
|
@@ -2433,20 +2457,20 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2433
2457
|
id: this._instanceId
|
|
2434
2458
|
}), {
|
|
2435
2459
|
F: __dxlog_file13,
|
|
2436
|
-
L:
|
|
2460
|
+
L: 126,
|
|
2437
2461
|
S: this,
|
|
2438
2462
|
C: (f, a) => f(...a)
|
|
2439
2463
|
});
|
|
2440
2464
|
return space;
|
|
2441
2465
|
}
|
|
2442
2466
|
};
|
|
2443
|
-
|
|
2467
|
+
_ts_decorate9([
|
|
2444
2468
|
import_async8.synchronized
|
|
2445
2469
|
], SpaceManager.prototype, "open", null);
|
|
2446
|
-
|
|
2470
|
+
_ts_decorate9([
|
|
2447
2471
|
import_async8.synchronized
|
|
2448
2472
|
], SpaceManager.prototype, "close", null);
|
|
2449
|
-
SpaceManager =
|
|
2473
|
+
SpaceManager = _ts_decorate9([
|
|
2450
2474
|
(0, import_async8.trackLeaks)("open", "close")
|
|
2451
2475
|
], SpaceManager);
|
|
2452
2476
|
|
|
@@ -2476,10 +2500,10 @@ var WebsocketNetworkManagerProvider = (signalUrl) => () => new import_network_ma
|
|
|
2476
2500
|
var TestAgentBuilder = class {
|
|
2477
2501
|
constructor({ storage, networkManagerProvider } = {}) {
|
|
2478
2502
|
this._agents = new import_util10.ComplexMap(import_keys7.PublicKey.hash);
|
|
2479
|
-
this._storage = storage
|
|
2503
|
+
this._storage = storage ?? (0, import_random_access_storage.createStorage)({
|
|
2480
2504
|
type: import_random_access_storage.StorageType.RAM
|
|
2481
2505
|
});
|
|
2482
|
-
this._networkManagerProvider = networkManagerProvider
|
|
2506
|
+
this._networkManagerProvider = networkManagerProvider ?? MemoryNetworkManagerProvider(new import_messaging.MemorySignalManagerContext());
|
|
2483
2507
|
}
|
|
2484
2508
|
async close() {
|
|
2485
2509
|
return Promise.all(this.agents.map((agent) => agent.close()));
|
|
@@ -2501,16 +2525,13 @@ var TestAgentBuilder = class {
|
|
|
2501
2525
|
};
|
|
2502
2526
|
var TestAgent = class {
|
|
2503
2527
|
get metadataStore() {
|
|
2504
|
-
|
|
2505
|
-
return (_a = this._metadataStore) != null ? _a : this._metadataStore = new MetadataStore(this.storage.createDirectory("metadata"));
|
|
2528
|
+
return this._metadataStore ??= new MetadataStore(this.storage.createDirectory("metadata"));
|
|
2506
2529
|
}
|
|
2507
2530
|
get snapshotStore() {
|
|
2508
|
-
|
|
2509
|
-
return (_a = this._snapshotStore) != null ? _a : this._snapshotStore = new SnapshotStore(this.storage.createDirectory("snapshots"));
|
|
2531
|
+
return this._snapshotStore ??= new SnapshotStore(this.storage.createDirectory("snapshots"));
|
|
2510
2532
|
}
|
|
2511
2533
|
get blobStore() {
|
|
2512
|
-
|
|
2513
|
-
return (_a = this._blobStore) != null ? _a : this._blobStore = new import_teleport_extension_object_sync2.BlobStore(this.storage.createDirectory("blobs"));
|
|
2534
|
+
return this._blobStore ??= new import_teleport_extension_object_sync2.BlobStore(this.storage.createDirectory("blobs"));
|
|
2514
2535
|
}
|
|
2515
2536
|
constructor(_networkManagerProvider, _feedBuilder, identityKey, deviceKey) {
|
|
2516
2537
|
this._networkManagerProvider = _networkManagerProvider;
|
|
@@ -2535,8 +2556,7 @@ var TestAgent = class {
|
|
|
2535
2556
|
return this._spaces.get(spaceKey);
|
|
2536
2557
|
}
|
|
2537
2558
|
get spaceManager() {
|
|
2538
|
-
|
|
2539
|
-
return (_a = this._spaceManager) != null ? _a : this._spaceManager = new SpaceManager({
|
|
2559
|
+
return this._spaceManager ??= new SpaceManager({
|
|
2540
2560
|
feedStore: this.feedStore,
|
|
2541
2561
|
networkManager: this._networkManagerProvider(),
|
|
2542
2562
|
modelFactory: this.modelFactory,
|
|
@@ -2556,7 +2576,7 @@ var TestAgent = class {
|
|
|
2556
2576
|
const controlFeed = await this.feedStore.openFeed(genesisKey, {
|
|
2557
2577
|
writable: true
|
|
2558
2578
|
});
|
|
2559
|
-
const dataFeed = await this.feedStore.openFeed(dataKey
|
|
2579
|
+
const dataFeed = await this.feedStore.openFeed(dataKey ?? await this.keyring.createKey(), {
|
|
2560
2580
|
writable: true,
|
|
2561
2581
|
sparse: true
|
|
2562
2582
|
});
|
|
@@ -2586,7 +2606,7 @@ var TestAgent = class {
|
|
|
2586
2606
|
});
|
|
2587
2607
|
space.setControlFeed(controlFeed);
|
|
2588
2608
|
space.setDataFeed(dataFeed);
|
|
2589
|
-
await space.open(new
|
|
2609
|
+
await space.open(new import_context7.Context());
|
|
2590
2610
|
this._spaces.set(spaceKey, space);
|
|
2591
2611
|
return space;
|
|
2592
2612
|
}
|
|
@@ -2601,7 +2621,7 @@ var TestAgent = class {
|
|
|
2601
2621
|
networkManager: this._networkManagerProvider(),
|
|
2602
2622
|
blobStore: this.blobStore,
|
|
2603
2623
|
onSessionAuth: (session) => {
|
|
2604
|
-
session.addExtension("dxos.mesh.teleport.gossip", (gossip
|
|
2624
|
+
session.addExtension("dxos.mesh.teleport.gossip", (gossip ?? this.createGossip()).createExtension({
|
|
2605
2625
|
remotePeerId: session.remotePeerId
|
|
2606
2626
|
}));
|
|
2607
2627
|
}
|
|
@@ -2617,7 +2637,7 @@ var TestAgent = class {
|
|
|
2617
2637
|
announceInterval: 30,
|
|
2618
2638
|
offlineTimeout: 200,
|
|
2619
2639
|
identityKey: this.identityKey,
|
|
2620
|
-
gossip: gossip
|
|
2640
|
+
gossip: gossip ?? this.createGossip()
|
|
2621
2641
|
});
|
|
2622
2642
|
}
|
|
2623
2643
|
async spaceGenesis(space) {
|
|
@@ -2682,9 +2702,8 @@ var createRemoteDatabaseFromDataServiceHost = async (modelFactory, dataServiceHo
|
|
|
2682
2702
|
};
|
|
2683
2703
|
};
|
|
2684
2704
|
var testLocalDatabase = async (create, check = create) => {
|
|
2685
|
-
var _a;
|
|
2686
2705
|
const objectId = import_keys8.PublicKey.random().toHex();
|
|
2687
|
-
await
|
|
2706
|
+
await create.databaseHost.getWriteStream()?.write({
|
|
2688
2707
|
batch: {
|
|
2689
2708
|
objects: [
|
|
2690
2709
|
{
|
|
@@ -2695,7 +2714,7 @@ var testLocalDatabase = async (create, check = create) => {
|
|
|
2695
2714
|
}
|
|
2696
2715
|
]
|
|
2697
2716
|
}
|
|
2698
|
-
})
|
|
2717
|
+
});
|
|
2699
2718
|
await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 2e3);
|
|
2700
2719
|
};
|
|
2701
2720
|
// Annotate the CommonJS export names for ESM import in node:
|