@dxos/echo-pipeline 0.1.57-main.e87098f → 0.1.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-GELWTWOW.mjs → chunk-V6J3XM4Y.mjs} +371 -145
- package/dist/lib/browser/chunk-V6J3XM4Y.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +5 -4
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/index.cjs +385 -159
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +397 -170
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/db-host/data-service-host.d.ts +3 -1
- package/dist/types/src/db-host/data-service-host.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts +2 -1
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/database-host.d.ts +4 -2
- package/dist/types/src/db-host/database-host.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts +12 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline.d.ts +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/auth.d.ts +1 -0
- package/dist/types/src/space/auth.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +4 -0
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +1 -0
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts +3 -2
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/dist/types/src/testing/util.d.ts.map +1 -1
- package/package.json +32 -32
- package/src/db-host/data-service-host.ts +5 -0
- package/src/db-host/data-service.ts +8 -0
- package/src/db-host/database-host.ts +7 -3
- package/src/metadata/metadata-store.ts +125 -28
- package/src/pipeline/pipeline.ts +11 -0
- package/src/space/auth.ts +5 -0
- package/src/space/control-pipeline.ts +63 -10
- package/src/space/data-pipeline.ts +29 -4
- package/src/space/space-protocol.ts +7 -2
- package/src/space/space.ts +4 -2
- package/src/testing/database-test-rig.ts +26 -21
- package/src/testing/test-agent-builder.ts +2 -2
- package/src/testing/util.ts +3 -1
- package/src/tests/database-unit.test.ts +26 -1
- package/src/tests/database.test.ts +3 -1
- package/dist/lib/browser/chunk-GELWTWOW.mjs.map +0 -7
package/dist/lib/node/index.cjs
CHANGED
|
@@ -72,7 +72,7 @@ var valueEncoding = (0, import_hypercore.createCodecEncoding)(codec);
|
|
|
72
72
|
|
|
73
73
|
// packages/core/echo/echo-pipeline/src/common/feeds.ts
|
|
74
74
|
var import_invariant = require("@dxos/invariant");
|
|
75
|
-
var __dxlog_file = "/
|
|
75
|
+
var __dxlog_file = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/common/feeds.ts";
|
|
76
76
|
var createMappedFeedWriter = (mapper, writer) => {
|
|
77
77
|
(0, import_invariant.invariant)(mapper, void 0, {
|
|
78
78
|
F: __dxlog_file,
|
|
@@ -105,11 +105,12 @@ var import_invariant2 = require("@dxos/invariant");
|
|
|
105
105
|
var import_log = require("@dxos/log");
|
|
106
106
|
var import_service = require("@dxos/protocols/proto/dxos/echo/service");
|
|
107
107
|
var import_util = require("@dxos/util");
|
|
108
|
-
var __dxlog_file2 = "/
|
|
108
|
+
var __dxlog_file2 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
|
|
109
109
|
var DataServiceHost = class {
|
|
110
|
-
constructor(_itemManager, _itemDemuxer, _writeStream) {
|
|
110
|
+
constructor(_itemManager, _itemDemuxer, _flush, _writeStream) {
|
|
111
111
|
this._itemManager = _itemManager;
|
|
112
112
|
this._itemDemuxer = _itemDemuxer;
|
|
113
|
+
this._flush = _flush;
|
|
113
114
|
this._writeStream = _writeStream;
|
|
114
115
|
this._ctx = new import_context.Context();
|
|
115
116
|
this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
|
|
@@ -143,7 +144,7 @@ var DataServiceHost = class {
|
|
|
143
144
|
const { batch, meta } = message;
|
|
144
145
|
(0, import_invariant2.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
|
|
145
146
|
F: __dxlog_file2,
|
|
146
|
-
L:
|
|
147
|
+
L: 69,
|
|
147
148
|
S: this,
|
|
148
149
|
A: [
|
|
149
150
|
"!(meta as any).clientTag",
|
|
@@ -155,7 +156,7 @@ var DataServiceHost = class {
|
|
|
155
156
|
meta
|
|
156
157
|
}, {
|
|
157
158
|
F: __dxlog_file2,
|
|
158
|
-
L:
|
|
159
|
+
L: 70,
|
|
159
160
|
S: this,
|
|
160
161
|
C: (f, a) => f(...a)
|
|
161
162
|
});
|
|
@@ -183,7 +184,7 @@ var DataServiceHost = class {
|
|
|
183
184
|
async write(request) {
|
|
184
185
|
(0, import_invariant2.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
|
|
185
186
|
F: __dxlog_file2,
|
|
186
|
-
L:
|
|
187
|
+
L: 96,
|
|
187
188
|
S: this,
|
|
188
189
|
A: [
|
|
189
190
|
"!this._ctx.disposed",
|
|
@@ -192,7 +193,7 @@ var DataServiceHost = class {
|
|
|
192
193
|
});
|
|
193
194
|
(0, import_invariant2.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
|
|
194
195
|
F: __dxlog_file2,
|
|
195
|
-
L:
|
|
196
|
+
L: 97,
|
|
196
197
|
S: this,
|
|
197
198
|
A: [
|
|
198
199
|
"this._writeStream",
|
|
@@ -204,7 +205,7 @@ var DataServiceHost = class {
|
|
|
204
205
|
objectCount: request.batch.objects?.length ?? 0
|
|
205
206
|
}, {
|
|
206
207
|
F: __dxlog_file2,
|
|
207
|
-
L:
|
|
208
|
+
L: 99,
|
|
208
209
|
S: this,
|
|
209
210
|
C: (f, a) => f(...a)
|
|
210
211
|
});
|
|
@@ -218,7 +219,7 @@ var DataServiceHost = class {
|
|
|
218
219
|
seq: receipt2.seq
|
|
219
220
|
}, {
|
|
220
221
|
F: __dxlog_file2,
|
|
221
|
-
L:
|
|
222
|
+
L: 108,
|
|
222
223
|
S: this,
|
|
223
224
|
C: (f, a) => f(...a)
|
|
224
225
|
});
|
|
@@ -231,6 +232,9 @@ var DataServiceHost = class {
|
|
|
231
232
|
});
|
|
232
233
|
return receipt;
|
|
233
234
|
}
|
|
235
|
+
async flush() {
|
|
236
|
+
await this._flush();
|
|
237
|
+
}
|
|
234
238
|
};
|
|
235
239
|
var createDataMessage = (batch) => ({
|
|
236
240
|
batch: {
|
|
@@ -248,8 +252,9 @@ var createDataMessage = (batch) => ({
|
|
|
248
252
|
// packages/core/echo/echo-pipeline/src/db-host/database-host.ts
|
|
249
253
|
var import_echo_db2 = require("@dxos/echo-db");
|
|
250
254
|
var DatabaseHost = class {
|
|
251
|
-
constructor(_outboundStream) {
|
|
255
|
+
constructor(_outboundStream, _flush) {
|
|
252
256
|
this._outboundStream = _outboundStream;
|
|
257
|
+
this._flush = _flush;
|
|
253
258
|
}
|
|
254
259
|
get isReadOnly() {
|
|
255
260
|
return !!this._outboundStream;
|
|
@@ -272,7 +277,7 @@ var DatabaseHost = class {
|
|
|
272
277
|
return this._itemDemuxer.createSnapshot();
|
|
273
278
|
}
|
|
274
279
|
createDataServiceHost() {
|
|
275
|
-
return new DataServiceHost(this._itemManager, this._itemDemuxer, this._outboundStream ?? void 0);
|
|
280
|
+
return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0);
|
|
276
281
|
}
|
|
277
282
|
};
|
|
278
283
|
|
|
@@ -379,7 +384,7 @@ var import_invariant3 = require("@dxos/invariant");
|
|
|
379
384
|
var import_keys2 = require("@dxos/keys");
|
|
380
385
|
var import_log2 = require("@dxos/log");
|
|
381
386
|
var import_util2 = require("@dxos/util");
|
|
382
|
-
var __dxlog_file3 = "/
|
|
387
|
+
var __dxlog_file3 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
|
|
383
388
|
var DataServiceSubscriptions = class {
|
|
384
389
|
constructor() {
|
|
385
390
|
this._spaces = new import_util2.ComplexMap(import_keys2.PublicKey.hash);
|
|
@@ -392,13 +397,13 @@ var DataServiceSubscriptions = class {
|
|
|
392
397
|
spaceKey
|
|
393
398
|
}, {
|
|
394
399
|
F: __dxlog_file3,
|
|
395
|
-
L:
|
|
400
|
+
L: 31,
|
|
396
401
|
S: this,
|
|
397
402
|
C: (f, a) => f(...a)
|
|
398
403
|
});
|
|
399
404
|
(0, import_invariant3.invariant)(!this._spaces.has(spaceKey), void 0, {
|
|
400
405
|
F: __dxlog_file3,
|
|
401
|
-
L:
|
|
406
|
+
L: 32,
|
|
402
407
|
S: this,
|
|
403
408
|
A: [
|
|
404
409
|
"!this._spaces.has(spaceKey)",
|
|
@@ -413,7 +418,7 @@ var DataServiceSubscriptions = class {
|
|
|
413
418
|
spaceKey
|
|
414
419
|
}, {
|
|
415
420
|
F: __dxlog_file3,
|
|
416
|
-
L:
|
|
421
|
+
L: 38,
|
|
417
422
|
S: this,
|
|
418
423
|
C: (f, a) => f(...a)
|
|
419
424
|
});
|
|
@@ -432,7 +437,7 @@ var DataServiceImpl = class {
|
|
|
432
437
|
subscribe(request) {
|
|
433
438
|
(0, import_invariant3.invariant)(request.spaceKey, void 0, {
|
|
434
439
|
F: __dxlog_file3,
|
|
435
|
-
L:
|
|
440
|
+
L: 57,
|
|
436
441
|
S: this,
|
|
437
442
|
A: [
|
|
438
443
|
"request.spaceKey",
|
|
@@ -445,7 +450,7 @@ var DataServiceImpl = class {
|
|
|
445
450
|
write(request) {
|
|
446
451
|
(0, import_invariant3.invariant)(request.spaceKey, void 0, {
|
|
447
452
|
F: __dxlog_file3,
|
|
448
|
-
L:
|
|
453
|
+
L: 64,
|
|
449
454
|
S: this,
|
|
450
455
|
A: [
|
|
451
456
|
"request.spaceKey",
|
|
@@ -454,7 +459,7 @@ var DataServiceImpl = class {
|
|
|
454
459
|
});
|
|
455
460
|
(0, import_invariant3.invariant)(request.batch, void 0, {
|
|
456
461
|
F: __dxlog_file3,
|
|
457
|
-
L:
|
|
462
|
+
L: 65,
|
|
458
463
|
S: this,
|
|
459
464
|
A: [
|
|
460
465
|
"request.batch",
|
|
@@ -464,6 +469,19 @@ var DataServiceImpl = class {
|
|
|
464
469
|
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
465
470
|
return host.write(request);
|
|
466
471
|
}
|
|
472
|
+
flush(request) {
|
|
473
|
+
(0, import_invariant3.invariant)(request.spaceKey, void 0, {
|
|
474
|
+
F: __dxlog_file3,
|
|
475
|
+
L: 72,
|
|
476
|
+
S: this,
|
|
477
|
+
A: [
|
|
478
|
+
"request.spaceKey",
|
|
479
|
+
""
|
|
480
|
+
]
|
|
481
|
+
});
|
|
482
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
483
|
+
return host.flush();
|
|
484
|
+
}
|
|
467
485
|
};
|
|
468
486
|
|
|
469
487
|
// packages/core/echo/echo-pipeline/src/errors/errors.ts
|
|
@@ -520,6 +538,7 @@ var import_crc_32 = __toESM(require("crc-32"));
|
|
|
520
538
|
var import_async2 = require("@dxos/async");
|
|
521
539
|
var import_errors = require("@dxos/errors");
|
|
522
540
|
var import_invariant4 = require("@dxos/invariant");
|
|
541
|
+
var import_keys3 = require("@dxos/keys");
|
|
523
542
|
var import_log3 = require("@dxos/log");
|
|
524
543
|
var import_protocols4 = require("@dxos/protocols");
|
|
525
544
|
var import_services = require("@dxos/protocols/proto/dxos/client/services");
|
|
@@ -534,19 +553,23 @@ function _ts_decorate2(decorators, target, key, desc) {
|
|
|
534
553
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
535
554
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
536
555
|
}
|
|
537
|
-
var __dxlog_file4 = "/
|
|
556
|
+
var __dxlog_file4 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
|
|
538
557
|
var emptyEchoMetadata = () => ({
|
|
539
558
|
version: import_protocols4.STORAGE_VERSION,
|
|
540
559
|
spaces: [],
|
|
541
560
|
created: /* @__PURE__ */ new Date(),
|
|
542
561
|
updated: /* @__PURE__ */ new Date()
|
|
543
562
|
});
|
|
563
|
+
var emptyLargeSpaceMetadata = () => ({});
|
|
544
564
|
var EchoMetadata = import_protocols4.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
565
|
+
var LargeSpaceMetadata = import_protocols4.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
|
|
545
566
|
var MetadataStore = class {
|
|
546
567
|
// prettier-ignore
|
|
547
568
|
constructor(_directory) {
|
|
548
569
|
this._directory = _directory;
|
|
549
570
|
this._metadata = emptyEchoMetadata();
|
|
571
|
+
this._spaceLargeMetadata = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
|
|
572
|
+
this._metadataFile = void 0;
|
|
550
573
|
this.update = new import_async2.Event();
|
|
551
574
|
}
|
|
552
575
|
get metadata() {
|
|
@@ -562,11 +585,7 @@ var MetadataStore = class {
|
|
|
562
585
|
get spaces() {
|
|
563
586
|
return this._metadata.spaces ?? [];
|
|
564
587
|
}
|
|
565
|
-
|
|
566
|
-
* Loads metadata from persistent storage.
|
|
567
|
-
*/
|
|
568
|
-
async load() {
|
|
569
|
-
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
588
|
+
async _readFile(file, codec2) {
|
|
570
589
|
try {
|
|
571
590
|
const { size: fileLength } = await file.stat();
|
|
572
591
|
if (fileLength < 8) {
|
|
@@ -576,10 +595,11 @@ var MetadataStore = class {
|
|
|
576
595
|
const checksum = fromBytesInt32(await file.read(4, 4));
|
|
577
596
|
(0, import_log3.log)("loaded", {
|
|
578
597
|
size: dataSize,
|
|
579
|
-
checksum
|
|
598
|
+
checksum,
|
|
599
|
+
name: file.filename
|
|
580
600
|
}, {
|
|
581
601
|
F: __dxlog_file4,
|
|
582
|
-
L:
|
|
602
|
+
L: 82,
|
|
583
603
|
S: this,
|
|
584
604
|
C: (f, a) => f(...a)
|
|
585
605
|
});
|
|
@@ -594,7 +614,48 @@ var MetadataStore = class {
|
|
|
594
614
|
if (calculatedChecksum !== checksum) {
|
|
595
615
|
throw new import_errors.DataCorruptionError("Metadata checksum is invalid.");
|
|
596
616
|
}
|
|
597
|
-
|
|
617
|
+
return codec2.decode(data);
|
|
618
|
+
} finally {
|
|
619
|
+
await file.close();
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
async _writeFile(file, codec2, data) {
|
|
623
|
+
const encoded = (0, import_util3.arrayToBuffer)(codec2.encode(data));
|
|
624
|
+
const checksum = import_crc_32.default.buf(encoded);
|
|
625
|
+
const result = Buffer.alloc(8 + encoded.length);
|
|
626
|
+
result.writeInt32LE(encoded.length, 0);
|
|
627
|
+
result.writeInt32LE(checksum, 4);
|
|
628
|
+
encoded.copy(result, 8);
|
|
629
|
+
await file.write(0, result);
|
|
630
|
+
(0, import_log3.log)("saved", {
|
|
631
|
+
size: encoded.length,
|
|
632
|
+
checksum
|
|
633
|
+
}, {
|
|
634
|
+
F: __dxlog_file4,
|
|
635
|
+
L: 114,
|
|
636
|
+
S: this,
|
|
637
|
+
C: (f, a) => f(...a)
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
async close() {
|
|
641
|
+
await this.flush();
|
|
642
|
+
await this._metadataFile?.close();
|
|
643
|
+
this._metadataFile = void 0;
|
|
644
|
+
this._metadata = emptyEchoMetadata();
|
|
645
|
+
this._spaceLargeMetadata.clear();
|
|
646
|
+
}
|
|
647
|
+
/**
|
|
648
|
+
* Loads metadata from persistent storage.
|
|
649
|
+
*/
|
|
650
|
+
async load() {
|
|
651
|
+
if (!this._metadataFile || this._metadataFile.closed) {
|
|
652
|
+
this._metadataFile = this._directory.getOrCreateFile("EchoMetadata");
|
|
653
|
+
}
|
|
654
|
+
try {
|
|
655
|
+
const metadata = await this._readFile(this._metadataFile, EchoMetadata);
|
|
656
|
+
if (metadata) {
|
|
657
|
+
this._metadata = metadata;
|
|
658
|
+
}
|
|
598
659
|
this._metadata.spaces?.forEach((space) => {
|
|
599
660
|
space.state ??= import_services.SpaceState.ACTIVE;
|
|
600
661
|
});
|
|
@@ -603,14 +664,29 @@ var MetadataStore = class {
|
|
|
603
664
|
err
|
|
604
665
|
}, {
|
|
605
666
|
F: __dxlog_file4,
|
|
606
|
-
L:
|
|
667
|
+
L: 145,
|
|
607
668
|
S: this,
|
|
608
669
|
C: (f, a) => f(...a)
|
|
609
670
|
});
|
|
610
671
|
this._metadata = emptyEchoMetadata();
|
|
611
|
-
} finally {
|
|
612
|
-
await file.close();
|
|
613
672
|
}
|
|
673
|
+
await (0, import_util3.forEachAsync)([
|
|
674
|
+
this._metadata.identity?.haloSpace.key,
|
|
675
|
+
...this._metadata.spaces?.map((space) => space.key) ?? []
|
|
676
|
+
].filter(import_util3.isNotNullOrUndefined), async (key) => {
|
|
677
|
+
try {
|
|
678
|
+
await this._loadSpaceLargeMetadata(key);
|
|
679
|
+
} catch (err) {
|
|
680
|
+
import_log3.log.error("failed to load space large metadata", {
|
|
681
|
+
err
|
|
682
|
+
}, {
|
|
683
|
+
F: __dxlog_file4,
|
|
684
|
+
L: 157,
|
|
685
|
+
S: this,
|
|
686
|
+
C: (f, a) => f(...a)
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
});
|
|
614
690
|
}
|
|
615
691
|
async _save() {
|
|
616
692
|
const data = {
|
|
@@ -621,27 +697,34 @@ var MetadataStore = class {
|
|
|
621
697
|
};
|
|
622
698
|
this.update.emit(data);
|
|
623
699
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
700
|
+
await this._writeFile(file, EchoMetadata, data);
|
|
701
|
+
}
|
|
702
|
+
async _loadSpaceLargeMetadata(key) {
|
|
703
|
+
const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
|
|
624
704
|
try {
|
|
625
|
-
const
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
(0, import_log3.log)("saved", {
|
|
633
|
-
size: encoded.length,
|
|
634
|
-
checksum
|
|
705
|
+
const metadata = await this._readFile(file, LargeSpaceMetadata);
|
|
706
|
+
if (metadata) {
|
|
707
|
+
this._spaceLargeMetadata.set(key, metadata);
|
|
708
|
+
}
|
|
709
|
+
} catch (err) {
|
|
710
|
+
import_log3.log.error("failed to load space large metadata", {
|
|
711
|
+
err
|
|
635
712
|
}, {
|
|
636
713
|
F: __dxlog_file4,
|
|
637
|
-
L:
|
|
714
|
+
L: 186,
|
|
638
715
|
S: this,
|
|
639
716
|
C: (f, a) => f(...a)
|
|
640
717
|
});
|
|
641
|
-
} finally {
|
|
642
|
-
await file.close();
|
|
643
718
|
}
|
|
644
719
|
}
|
|
720
|
+
async _saveSpaceLargeMetadata(key) {
|
|
721
|
+
const data = this._getLargeSpaceMetadata(key);
|
|
722
|
+
const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
|
|
723
|
+
await this._writeFile(file, LargeSpaceMetadata, data);
|
|
724
|
+
}
|
|
725
|
+
async flush() {
|
|
726
|
+
await this._directory.flush();
|
|
727
|
+
}
|
|
645
728
|
_getSpace(spaceKey) {
|
|
646
729
|
if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
|
|
647
730
|
return this._metadata.identity.haloSpace;
|
|
@@ -649,7 +732,7 @@ var MetadataStore = class {
|
|
|
649
732
|
const space = this.spaces.find((space2) => space2.key === spaceKey);
|
|
650
733
|
(0, import_invariant4.invariant)(space, "Space not found", {
|
|
651
734
|
F: __dxlog_file4,
|
|
652
|
-
L:
|
|
735
|
+
L: 208,
|
|
653
736
|
S: this,
|
|
654
737
|
A: [
|
|
655
738
|
"space",
|
|
@@ -658,13 +741,22 @@ var MetadataStore = class {
|
|
|
658
741
|
});
|
|
659
742
|
return space;
|
|
660
743
|
}
|
|
744
|
+
_getLargeSpaceMetadata(key) {
|
|
745
|
+
let entry = this._spaceLargeMetadata.get(key);
|
|
746
|
+
if (entry) {
|
|
747
|
+
return entry;
|
|
748
|
+
}
|
|
749
|
+
entry = emptyLargeSpaceMetadata();
|
|
750
|
+
this._spaceLargeMetadata.set(key, entry);
|
|
751
|
+
return entry;
|
|
752
|
+
}
|
|
661
753
|
/**
|
|
662
754
|
* Clears storage - doesn't work for now.
|
|
663
755
|
*/
|
|
664
756
|
async clear() {
|
|
665
757
|
(0, import_log3.log)("clearing all metadata", void 0, {
|
|
666
758
|
F: __dxlog_file4,
|
|
667
|
-
L:
|
|
759
|
+
L: 227,
|
|
668
760
|
S: this,
|
|
669
761
|
C: (f, a) => f(...a)
|
|
670
762
|
});
|
|
@@ -677,7 +769,7 @@ var MetadataStore = class {
|
|
|
677
769
|
async setIdentityRecord(record) {
|
|
678
770
|
(0, import_invariant4.invariant)(!this._metadata.identity, "Cannot overwrite existing identity in metadata", {
|
|
679
771
|
F: __dxlog_file4,
|
|
680
|
-
L:
|
|
772
|
+
L: 237,
|
|
681
773
|
S: this,
|
|
682
774
|
A: [
|
|
683
775
|
"!this._metadata.identity",
|
|
@@ -686,11 +778,12 @@ var MetadataStore = class {
|
|
|
686
778
|
});
|
|
687
779
|
this._metadata.identity = record;
|
|
688
780
|
await this._save();
|
|
781
|
+
await this.flush();
|
|
689
782
|
}
|
|
690
783
|
async addSpace(record) {
|
|
691
784
|
(0, import_invariant4.invariant)(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
|
|
692
785
|
F: __dxlog_file4,
|
|
693
|
-
L:
|
|
786
|
+
L: 245,
|
|
694
787
|
S: this,
|
|
695
788
|
A: [
|
|
696
789
|
"!(this._metadata.spaces ?? []).find((space) => space.key === record.key)",
|
|
@@ -699,6 +792,7 @@ var MetadataStore = class {
|
|
|
699
792
|
});
|
|
700
793
|
(this._metadata.spaces ??= []).push(record);
|
|
701
794
|
await this._save();
|
|
795
|
+
await this.flush();
|
|
702
796
|
}
|
|
703
797
|
async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
|
|
704
798
|
this._getSpace(spaceKey).dataTimeframe = timeframe;
|
|
@@ -707,6 +801,7 @@ var MetadataStore = class {
|
|
|
707
801
|
async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
|
|
708
802
|
this._getSpace(spaceKey).controlTimeframe = timeframe;
|
|
709
803
|
await this._save();
|
|
804
|
+
await this.flush();
|
|
710
805
|
}
|
|
711
806
|
async setCache(spaceKey, cache) {
|
|
712
807
|
this._getSpace(spaceKey).cache = cache;
|
|
@@ -717,10 +812,20 @@ var MetadataStore = class {
|
|
|
717
812
|
space.controlFeedKey = controlFeedKey;
|
|
718
813
|
space.dataFeedKey = dataFeedKey;
|
|
719
814
|
await this._save();
|
|
815
|
+
await this.flush();
|
|
720
816
|
}
|
|
721
817
|
async setSpaceState(spaceKey, state) {
|
|
722
818
|
this._getSpace(spaceKey).state = state;
|
|
723
819
|
await this._save();
|
|
820
|
+
await this.flush();
|
|
821
|
+
}
|
|
822
|
+
getSpaceControlPipelineSnapshot(spaceKey) {
|
|
823
|
+
return this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot;
|
|
824
|
+
}
|
|
825
|
+
async setSpaceControlPipelineSnapshot(spaceKey, snapshot) {
|
|
826
|
+
this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot = snapshot;
|
|
827
|
+
await this._saveSpaceLargeMetadata(spaceKey);
|
|
828
|
+
await this.flush();
|
|
724
829
|
}
|
|
725
830
|
};
|
|
726
831
|
_ts_decorate2([
|
|
@@ -729,6 +834,9 @@ _ts_decorate2([
|
|
|
729
834
|
_ts_decorate2([
|
|
730
835
|
import_async2.synchronized
|
|
731
836
|
], MetadataStore.prototype, "_save", null);
|
|
837
|
+
_ts_decorate2([
|
|
838
|
+
import_async2.synchronized
|
|
839
|
+
], MetadataStore.prototype, "_saveSpaceLargeMetadata", null);
|
|
732
840
|
var fromBytesInt32 = (buf) => buf.readInt32LE(0);
|
|
733
841
|
|
|
734
842
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
@@ -737,7 +845,7 @@ var import_context3 = require("@dxos/context");
|
|
|
737
845
|
var import_debug3 = require("@dxos/debug");
|
|
738
846
|
var import_feed_store = require("@dxos/feed-store");
|
|
739
847
|
var import_invariant6 = require("@dxos/invariant");
|
|
740
|
-
var
|
|
848
|
+
var import_keys4 = require("@dxos/keys");
|
|
741
849
|
var import_log6 = require("@dxos/log");
|
|
742
850
|
var import_timeframe2 = require("@dxos/timeframe");
|
|
743
851
|
var import_util4 = require("@dxos/util");
|
|
@@ -745,7 +853,7 @@ var import_util4 = require("@dxos/util");
|
|
|
745
853
|
// packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
|
|
746
854
|
var import_invariant5 = require("@dxos/invariant");
|
|
747
855
|
var import_log4 = require("@dxos/log");
|
|
748
|
-
var __dxlog_file5 = "/
|
|
856
|
+
var __dxlog_file5 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
|
|
749
857
|
var createMessageSelector = (timeframeClock) => {
|
|
750
858
|
return (messages) => {
|
|
751
859
|
for (let i = 0; i < messages.length; i++) {
|
|
@@ -787,7 +895,7 @@ function _ts_decorate3(decorators, target, key, desc) {
|
|
|
787
895
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
788
896
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
789
897
|
}
|
|
790
|
-
var __dxlog_file6 = "/
|
|
898
|
+
var __dxlog_file6 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
|
|
791
899
|
var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
|
|
792
900
|
feedKey,
|
|
793
901
|
index
|
|
@@ -881,7 +989,7 @@ function _ts_decorate4(decorators, target, key, desc) {
|
|
|
881
989
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
882
990
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
883
991
|
}
|
|
884
|
-
var __dxlog_file7 = "/
|
|
992
|
+
var __dxlog_file7 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
|
|
885
993
|
var PipelineState = class {
|
|
886
994
|
// prettier-ignore
|
|
887
995
|
constructor(_feeds, _timeframeClock) {
|
|
@@ -891,6 +999,7 @@ var PipelineState = class {
|
|
|
891
999
|
this.timeframeUpdate = this._timeframeClock.update;
|
|
892
1000
|
this.stalled = new import_async4.Event();
|
|
893
1001
|
this._startTimeframe = new import_timeframe2.Timeframe();
|
|
1002
|
+
this._reachedTarget = false;
|
|
894
1003
|
}
|
|
895
1004
|
/**
|
|
896
1005
|
* Latest theoretical timeframe based on the last mutation in each feed.
|
|
@@ -915,6 +1024,9 @@ var PipelineState = class {
|
|
|
915
1024
|
get targetTimeframe() {
|
|
916
1025
|
return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
|
|
917
1026
|
}
|
|
1027
|
+
get reachedTarget() {
|
|
1028
|
+
return this._reachedTarget;
|
|
1029
|
+
}
|
|
918
1030
|
get feeds() {
|
|
919
1031
|
return Array.from(this._feeds.values());
|
|
920
1032
|
}
|
|
@@ -938,7 +1050,7 @@ var PipelineState = class {
|
|
|
938
1050
|
target: this.targetTimeframe
|
|
939
1051
|
}, {
|
|
940
1052
|
F: __dxlog_file7,
|
|
941
|
-
L:
|
|
1053
|
+
L: 134,
|
|
942
1054
|
S: this,
|
|
943
1055
|
C: (f, a) => f(...a)
|
|
944
1056
|
});
|
|
@@ -957,6 +1069,7 @@ var PipelineState = class {
|
|
|
957
1069
|
(0, import_context3.rejectOnDispose)(this._ctx),
|
|
958
1070
|
this._reachedTargetPromise.then(() => {
|
|
959
1071
|
done = true;
|
|
1072
|
+
this._reachedTarget = true;
|
|
960
1073
|
}),
|
|
961
1074
|
(0, import_async4.sleep)(timeout).then(() => {
|
|
962
1075
|
if (done) {
|
|
@@ -969,7 +1082,7 @@ var PipelineState = class {
|
|
|
969
1082
|
dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
|
|
970
1083
|
}, {
|
|
971
1084
|
F: __dxlog_file7,
|
|
972
|
-
L:
|
|
1085
|
+
L: 162,
|
|
973
1086
|
S: this,
|
|
974
1087
|
C: (f, a) => f(...a)
|
|
975
1088
|
});
|
|
@@ -983,14 +1096,14 @@ var PipelineState = class {
|
|
|
983
1096
|
var Pipeline = class {
|
|
984
1097
|
constructor() {
|
|
985
1098
|
this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
|
|
986
|
-
this._feeds = new import_util4.ComplexMap(
|
|
1099
|
+
this._feeds = new import_util4.ComplexMap(import_keys4.PublicKey.hash);
|
|
987
1100
|
// External state accessor.
|
|
988
1101
|
this._state = new PipelineState(this._feeds, this._timeframeClock);
|
|
989
1102
|
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
990
1103
|
this._processingTrigger = new import_async4.Trigger().wake();
|
|
991
1104
|
this._pauseTrigger = new import_async4.Trigger().wake();
|
|
992
1105
|
// Pending downloads.
|
|
993
|
-
this._downloads = new import_util4.ComplexMap((value) =>
|
|
1106
|
+
this._downloads = new import_util4.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
|
|
994
1107
|
this._isStopping = false;
|
|
995
1108
|
this._isStarted = false;
|
|
996
1109
|
this._isBeingConsumed = false;
|
|
@@ -1002,7 +1115,7 @@ var Pipeline = class {
|
|
|
1002
1115
|
get writer() {
|
|
1003
1116
|
(0, import_invariant6.invariant)(this._writer, "Writer not set.", {
|
|
1004
1117
|
F: __dxlog_file7,
|
|
1005
|
-
L:
|
|
1118
|
+
L: 243,
|
|
1006
1119
|
S: this,
|
|
1007
1120
|
A: [
|
|
1008
1121
|
"this._writer",
|
|
@@ -1031,7 +1144,7 @@ var Pipeline = class {
|
|
|
1031
1144
|
setWriteFeed(feed) {
|
|
1032
1145
|
(0, import_invariant6.invariant)(!this._writer, "Writer already set.", {
|
|
1033
1146
|
F: __dxlog_file7,
|
|
1034
|
-
L:
|
|
1147
|
+
L: 270,
|
|
1035
1148
|
S: this,
|
|
1036
1149
|
A: [
|
|
1037
1150
|
"!this._writer",
|
|
@@ -1040,7 +1153,7 @@ var Pipeline = class {
|
|
|
1040
1153
|
});
|
|
1041
1154
|
(0, import_invariant6.invariant)(feed.properties.writable, "Feed must be writable.", {
|
|
1042
1155
|
F: __dxlog_file7,
|
|
1043
|
-
L:
|
|
1156
|
+
L: 271,
|
|
1044
1157
|
S: this,
|
|
1045
1158
|
A: [
|
|
1046
1159
|
"feed.properties.writable",
|
|
@@ -1055,7 +1168,7 @@ var Pipeline = class {
|
|
|
1055
1168
|
async start() {
|
|
1056
1169
|
(0, import_invariant6.invariant)(!this._isStarted, "Pipeline is already started.", {
|
|
1057
1170
|
F: __dxlog_file7,
|
|
1058
|
-
L:
|
|
1171
|
+
L: 284,
|
|
1059
1172
|
S: this,
|
|
1060
1173
|
A: [
|
|
1061
1174
|
"!this._isStarted",
|
|
@@ -1064,7 +1177,7 @@ var Pipeline = class {
|
|
|
1064
1177
|
});
|
|
1065
1178
|
(0, import_log6.log)("starting...", void 0, {
|
|
1066
1179
|
F: __dxlog_file7,
|
|
1067
|
-
L:
|
|
1180
|
+
L: 285,
|
|
1068
1181
|
S: this,
|
|
1069
1182
|
C: (f, a) => f(...a)
|
|
1070
1183
|
});
|
|
@@ -1073,7 +1186,7 @@ var Pipeline = class {
|
|
|
1073
1186
|
this._isStarted = true;
|
|
1074
1187
|
(0, import_log6.log)("started", void 0, {
|
|
1075
1188
|
F: __dxlog_file7,
|
|
1076
|
-
L:
|
|
1189
|
+
L: 289,
|
|
1077
1190
|
S: this,
|
|
1078
1191
|
C: (f, a) => f(...a)
|
|
1079
1192
|
});
|
|
@@ -1086,7 +1199,7 @@ var Pipeline = class {
|
|
|
1086
1199
|
async stop() {
|
|
1087
1200
|
(0, import_log6.log)("stopping...", void 0, {
|
|
1088
1201
|
F: __dxlog_file7,
|
|
1089
|
-
L:
|
|
1202
|
+
L: 300,
|
|
1090
1203
|
S: this,
|
|
1091
1204
|
C: (f, a) => f(...a)
|
|
1092
1205
|
});
|
|
@@ -1100,10 +1213,11 @@ var Pipeline = class {
|
|
|
1100
1213
|
await this._state._ctx.dispose();
|
|
1101
1214
|
this._state._ctx = new import_context3.Context();
|
|
1102
1215
|
this._state._reachedTargetPromise = void 0;
|
|
1216
|
+
this._state._reachedTarget = false;
|
|
1103
1217
|
this._isStarted = false;
|
|
1104
1218
|
(0, import_log6.log)("stopped", void 0, {
|
|
1105
1219
|
F: __dxlog_file7,
|
|
1106
|
-
L:
|
|
1220
|
+
L: 313,
|
|
1107
1221
|
S: this,
|
|
1108
1222
|
C: (f, a) => f(...a)
|
|
1109
1223
|
});
|
|
@@ -1115,7 +1229,7 @@ var Pipeline = class {
|
|
|
1115
1229
|
async setCursor(timeframe) {
|
|
1116
1230
|
(0, import_invariant6.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
|
|
1117
1231
|
F: __dxlog_file7,
|
|
1118
|
-
L:
|
|
1232
|
+
L: 322,
|
|
1119
1233
|
S: this,
|
|
1120
1234
|
A: [
|
|
1121
1235
|
"!this._isStarted || this._isPaused",
|
|
@@ -1144,7 +1258,7 @@ var Pipeline = class {
|
|
|
1144
1258
|
async unpause() {
|
|
1145
1259
|
(0, import_invariant6.invariant)(this._isPaused, "Pipeline is not paused.", {
|
|
1146
1260
|
F: __dxlog_file7,
|
|
1147
|
-
L:
|
|
1261
|
+
L: 351,
|
|
1148
1262
|
S: this,
|
|
1149
1263
|
A: [
|
|
1150
1264
|
"this._isPaused",
|
|
@@ -1164,7 +1278,7 @@ var Pipeline = class {
|
|
|
1164
1278
|
async *consume() {
|
|
1165
1279
|
(0, import_invariant6.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
|
|
1166
1280
|
F: __dxlog_file7,
|
|
1167
|
-
L:
|
|
1281
|
+
L: 366,
|
|
1168
1282
|
S: this,
|
|
1169
1283
|
A: [
|
|
1170
1284
|
"!this._isBeingConsumed",
|
|
@@ -1174,7 +1288,7 @@ var Pipeline = class {
|
|
|
1174
1288
|
this._isBeingConsumed = true;
|
|
1175
1289
|
(0, import_invariant6.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1176
1290
|
F: __dxlog_file7,
|
|
1177
|
-
L:
|
|
1291
|
+
L: 369,
|
|
1178
1292
|
S: this,
|
|
1179
1293
|
A: [
|
|
1180
1294
|
"this._feedSetIterator",
|
|
@@ -1188,7 +1302,7 @@ var Pipeline = class {
|
|
|
1188
1302
|
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
1189
1303
|
(0, import_invariant6.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1190
1304
|
F: __dxlog_file7,
|
|
1191
|
-
L:
|
|
1305
|
+
L: 378,
|
|
1192
1306
|
S: this,
|
|
1193
1307
|
A: [
|
|
1194
1308
|
"this._feedSetIterator",
|
|
@@ -1202,7 +1316,7 @@ var Pipeline = class {
|
|
|
1202
1316
|
if (!done) {
|
|
1203
1317
|
const block = value ?? (0, import_debug3.failUndefined)();
|
|
1204
1318
|
this._processingTrigger.reset();
|
|
1205
|
-
this._timeframeClock.updatePendingTimeframe(
|
|
1319
|
+
this._timeframeClock.updatePendingTimeframe(import_keys4.PublicKey.from(block.feedKey), block.seq);
|
|
1206
1320
|
yield block;
|
|
1207
1321
|
this._processingTrigger.wake();
|
|
1208
1322
|
this._timeframeClock.updateTimeframe();
|
|
@@ -1223,7 +1337,7 @@ var Pipeline = class {
|
|
|
1223
1337
|
length: feed.length
|
|
1224
1338
|
}, {
|
|
1225
1339
|
F: __dxlog_file7,
|
|
1226
|
-
L:
|
|
1340
|
+
L: 407,
|
|
1227
1341
|
S: this,
|
|
1228
1342
|
C: (f, a) => f(...a)
|
|
1229
1343
|
});
|
|
@@ -1237,7 +1351,7 @@ var Pipeline = class {
|
|
|
1237
1351
|
data
|
|
1238
1352
|
}, {
|
|
1239
1353
|
F: __dxlog_file7,
|
|
1240
|
-
L:
|
|
1354
|
+
L: 412,
|
|
1241
1355
|
S: this,
|
|
1242
1356
|
C: (f, a) => f(...a)
|
|
1243
1357
|
});
|
|
@@ -1253,7 +1367,7 @@ var Pipeline = class {
|
|
|
1253
1367
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
1254
1368
|
import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
1255
1369
|
F: __dxlog_file7,
|
|
1256
|
-
L:
|
|
1370
|
+
L: 426,
|
|
1257
1371
|
S: this,
|
|
1258
1372
|
C: (f, a) => f(...a)
|
|
1259
1373
|
});
|
|
@@ -1288,7 +1402,7 @@ var import_invariant7 = require("@dxos/invariant");
|
|
|
1288
1402
|
var import_log7 = require("@dxos/log");
|
|
1289
1403
|
var import_protocols5 = require("@dxos/protocols");
|
|
1290
1404
|
var import_teleport = require("@dxos/teleport");
|
|
1291
|
-
var __dxlog_file8 = "/
|
|
1405
|
+
var __dxlog_file8 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/auth.ts";
|
|
1292
1406
|
var AuthExtension = class extends import_teleport.RpcExtension {
|
|
1293
1407
|
constructor(_authParams) {
|
|
1294
1408
|
super({
|
|
@@ -1381,10 +1495,14 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
1381
1495
|
await this._ctx.dispose();
|
|
1382
1496
|
await super.onClose();
|
|
1383
1497
|
}
|
|
1498
|
+
async onAbort() {
|
|
1499
|
+
await this._ctx.dispose();
|
|
1500
|
+
await super.onAbort();
|
|
1501
|
+
}
|
|
1384
1502
|
};
|
|
1385
1503
|
|
|
1386
1504
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
1387
|
-
var
|
|
1505
|
+
var import_async8 = require("@dxos/async");
|
|
1388
1506
|
var import_invariant9 = require("@dxos/invariant");
|
|
1389
1507
|
var import_log10 = require("@dxos/log");
|
|
1390
1508
|
var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
@@ -1392,9 +1510,10 @@ var import_tracing3 = require("@dxos/tracing");
|
|
|
1392
1510
|
var import_util7 = require("@dxos/util");
|
|
1393
1511
|
|
|
1394
1512
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
1513
|
+
var import_async6 = require("@dxos/async");
|
|
1395
1514
|
var import_context5 = require("@dxos/context");
|
|
1396
1515
|
var import_credentials = require("@dxos/credentials");
|
|
1397
|
-
var
|
|
1516
|
+
var import_keys5 = require("@dxos/keys");
|
|
1398
1517
|
var import_log8 = require("@dxos/log");
|
|
1399
1518
|
var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
1400
1519
|
var import_timeframe3 = require("@dxos/timeframe");
|
|
@@ -1410,14 +1529,21 @@ function _ts_decorate5(decorators, target, key, desc) {
|
|
|
1410
1529
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1411
1530
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1412
1531
|
}
|
|
1413
|
-
var __dxlog_file9 = "/
|
|
1532
|
+
var __dxlog_file9 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
|
|
1414
1533
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
1534
|
+
var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
|
|
1535
|
+
var USE_SNAPSHOTS = true;
|
|
1415
1536
|
var ControlPipeline = class ControlPipeline2 {
|
|
1416
1537
|
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
1538
|
+
this._ctx = new import_context5.Context();
|
|
1417
1539
|
this._lastTimeframeSaveTime = Date.now();
|
|
1418
1540
|
this.onFeedAdmitted = new import_util5.Callback();
|
|
1419
1541
|
this._usage = new import_tracing.TimeUsageCounter();
|
|
1420
1542
|
this._mutations = new import_tracing.TimeSeriesCounter();
|
|
1543
|
+
this._snapshotTask = new import_async6.DeferredTask(this._ctx, async () => {
|
|
1544
|
+
await (0, import_async6.sleep)(CONTROL_PIPELINE_SNAPSHOT_DELAY);
|
|
1545
|
+
await this._saveSnapshot();
|
|
1546
|
+
});
|
|
1421
1547
|
this._spaceKey = spaceKey;
|
|
1422
1548
|
this._metadata = metadataStore;
|
|
1423
1549
|
this._pipeline = new Pipeline();
|
|
@@ -1428,22 +1554,24 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1428
1554
|
key: info.key
|
|
1429
1555
|
}, {
|
|
1430
1556
|
F: __dxlog_file9,
|
|
1431
|
-
L:
|
|
1557
|
+
L: 73,
|
|
1432
1558
|
S: this,
|
|
1433
1559
|
C: (f, a) => f(...a)
|
|
1434
1560
|
});
|
|
1435
1561
|
if (info.assertion.designation === import_credentials2.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1562
|
+
queueMicrotask(async () => {
|
|
1563
|
+
try {
|
|
1564
|
+
const feed = await feedProvider(info.key);
|
|
1565
|
+
await this._pipeline.addFeed(feed);
|
|
1566
|
+
} catch (err) {
|
|
1567
|
+
import_log8.log.catch(err, void 0, {
|
|
1568
|
+
F: __dxlog_file9,
|
|
1569
|
+
L: 82,
|
|
1570
|
+
S: this,
|
|
1571
|
+
C: (f, a) => f(...a)
|
|
1572
|
+
});
|
|
1573
|
+
}
|
|
1574
|
+
});
|
|
1447
1575
|
}
|
|
1448
1576
|
await this.onFeedAdmitted.callIfSet(info);
|
|
1449
1577
|
});
|
|
@@ -1460,9 +1588,23 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1460
1588
|
this._pipeline.setWriteFeed(feed);
|
|
1461
1589
|
}
|
|
1462
1590
|
async start() {
|
|
1591
|
+
const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
|
|
1592
|
+
(0, import_log8.log)("load snapshot", {
|
|
1593
|
+
key: this._spaceKey,
|
|
1594
|
+
present: !!snapshot,
|
|
1595
|
+
tf: snapshot?.timeframe
|
|
1596
|
+
}, {
|
|
1597
|
+
F: __dxlog_file9,
|
|
1598
|
+
L: 109,
|
|
1599
|
+
S: this,
|
|
1600
|
+
C: (f, a) => f(...a)
|
|
1601
|
+
});
|
|
1602
|
+
if (USE_SNAPSHOTS && snapshot) {
|
|
1603
|
+
await this._processSnapshot(snapshot);
|
|
1604
|
+
}
|
|
1463
1605
|
(0, import_log8.log)("starting...", void 0, {
|
|
1464
1606
|
F: __dxlog_file9,
|
|
1465
|
-
L:
|
|
1607
|
+
L: 114,
|
|
1466
1608
|
S: this,
|
|
1467
1609
|
C: (f, a) => f(...a)
|
|
1468
1610
|
});
|
|
@@ -1472,11 +1614,51 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1472
1614
|
await this._pipeline.start();
|
|
1473
1615
|
(0, import_log8.log)("started", void 0, {
|
|
1474
1616
|
F: __dxlog_file9,
|
|
1475
|
-
L:
|
|
1617
|
+
L: 120,
|
|
1476
1618
|
S: this,
|
|
1477
1619
|
C: (f, a) => f(...a)
|
|
1478
1620
|
});
|
|
1479
1621
|
}
|
|
1622
|
+
async _processSnapshot(snapshot) {
|
|
1623
|
+
await this._pipeline.setCursor(snapshot.timeframe);
|
|
1624
|
+
for (const message of snapshot.messages ?? []) {
|
|
1625
|
+
const result = await this._spaceStateMachine.process(message.credential, {
|
|
1626
|
+
sourceFeed: message.feedKey,
|
|
1627
|
+
skipVerification: true
|
|
1628
|
+
});
|
|
1629
|
+
if (!result) {
|
|
1630
|
+
import_log8.log.warn("credential processing failed from snapshot", {
|
|
1631
|
+
message
|
|
1632
|
+
}, {
|
|
1633
|
+
F: __dxlog_file9,
|
|
1634
|
+
L: 133,
|
|
1635
|
+
S: this,
|
|
1636
|
+
C: (f, a) => f(...a)
|
|
1637
|
+
});
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
async _saveSnapshot() {
|
|
1642
|
+
await this._pipeline.pause();
|
|
1643
|
+
const snapshot = {
|
|
1644
|
+
timeframe: this._pipeline.state.timeframe,
|
|
1645
|
+
messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
|
|
1646
|
+
feedKey: entry.sourceFeed,
|
|
1647
|
+
credential: entry.credential
|
|
1648
|
+
}))
|
|
1649
|
+
};
|
|
1650
|
+
await this._pipeline.unpause();
|
|
1651
|
+
(0, import_log8.log)("save snapshot", {
|
|
1652
|
+
key: this._spaceKey,
|
|
1653
|
+
snapshot
|
|
1654
|
+
}, {
|
|
1655
|
+
F: __dxlog_file9,
|
|
1656
|
+
L: 149,
|
|
1657
|
+
S: this,
|
|
1658
|
+
C: (f, a) => f(...a)
|
|
1659
|
+
});
|
|
1660
|
+
await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
|
|
1661
|
+
}
|
|
1480
1662
|
async _consumePipeline(ctx) {
|
|
1481
1663
|
for await (const msg of this._pipeline.consume()) {
|
|
1482
1664
|
const span = this._usage.beginRecording();
|
|
@@ -1486,7 +1668,7 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1486
1668
|
} catch (err) {
|
|
1487
1669
|
import_log8.log.catch(err, void 0, {
|
|
1488
1670
|
F: __dxlog_file9,
|
|
1489
|
-
L:
|
|
1671
|
+
L: 162,
|
|
1490
1672
|
S: this,
|
|
1491
1673
|
C: (f, a) => f(...a)
|
|
1492
1674
|
});
|
|
@@ -1500,26 +1682,29 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1500
1682
|
seq: msg.seq
|
|
1501
1683
|
}, {
|
|
1502
1684
|
F: __dxlog_file9,
|
|
1503
|
-
L:
|
|
1685
|
+
L: 172,
|
|
1504
1686
|
S: this,
|
|
1505
1687
|
C: (f, a) => f(...a)
|
|
1506
1688
|
});
|
|
1507
1689
|
if (msg.data.payload.credential) {
|
|
1508
1690
|
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1509
|
-
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential,
|
|
1691
|
+
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
|
|
1692
|
+
sourceFeed: import_keys5.PublicKey.from(msg.feedKey)
|
|
1693
|
+
});
|
|
1510
1694
|
timer.end();
|
|
1511
1695
|
if (!result) {
|
|
1512
1696
|
import_log8.log.warn("processing failed", {
|
|
1513
1697
|
msg
|
|
1514
1698
|
}, {
|
|
1515
1699
|
F: __dxlog_file9,
|
|
1516
|
-
L:
|
|
1700
|
+
L: 181,
|
|
1517
1701
|
S: this,
|
|
1518
1702
|
C: (f, a) => f(...a)
|
|
1519
1703
|
});
|
|
1520
1704
|
} else {
|
|
1521
1705
|
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1522
1706
|
}
|
|
1707
|
+
this._snapshotTask.schedule();
|
|
1523
1708
|
}
|
|
1524
1709
|
}
|
|
1525
1710
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
@@ -1531,15 +1716,16 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1531
1716
|
async stop() {
|
|
1532
1717
|
(0, import_log8.log)("stopping...", void 0, {
|
|
1533
1718
|
F: __dxlog_file9,
|
|
1534
|
-
L:
|
|
1719
|
+
L: 201,
|
|
1535
1720
|
S: this,
|
|
1536
1721
|
C: (f, a) => f(...a)
|
|
1537
1722
|
});
|
|
1723
|
+
await this._ctx.dispose();
|
|
1538
1724
|
await this._pipeline.stop();
|
|
1539
1725
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1540
1726
|
(0, import_log8.log)("stopped", void 0, {
|
|
1541
1727
|
F: __dxlog_file9,
|
|
1542
|
-
L:
|
|
1728
|
+
L: 205,
|
|
1543
1729
|
S: this,
|
|
1544
1730
|
C: (f, a) => f(...a)
|
|
1545
1731
|
});
|
|
@@ -1552,7 +1738,7 @@ var ControlPipeline = class ControlPipeline2 {
|
|
|
1552
1738
|
} catch (err) {
|
|
1553
1739
|
(0, import_log8.log)(err, void 0, {
|
|
1554
1740
|
F: __dxlog_file9,
|
|
1555
|
-
L:
|
|
1741
|
+
L: 214,
|
|
1556
1742
|
S: this,
|
|
1557
1743
|
C: (f, a) => f(...a)
|
|
1558
1744
|
});
|
|
@@ -1565,6 +1751,11 @@ _ts_decorate5([
|
|
|
1565
1751
|
_ts_decorate5([
|
|
1566
1752
|
import_tracing.trace.metricsCounter()
|
|
1567
1753
|
], ControlPipeline.prototype, "_mutations", void 0);
|
|
1754
|
+
_ts_decorate5([
|
|
1755
|
+
import_tracing.trace.span({
|
|
1756
|
+
showInBrowserTimeline: true
|
|
1757
|
+
})
|
|
1758
|
+
], ControlPipeline.prototype, "start", null);
|
|
1568
1759
|
_ts_decorate5([
|
|
1569
1760
|
import_tracing.trace.span()
|
|
1570
1761
|
], ControlPipeline.prototype, "_consumePipeline", null);
|
|
@@ -1576,7 +1767,7 @@ ControlPipeline = _ts_decorate5([
|
|
|
1576
1767
|
], ControlPipeline);
|
|
1577
1768
|
|
|
1578
1769
|
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1579
|
-
var
|
|
1770
|
+
var import_async7 = require("@dxos/async");
|
|
1580
1771
|
var import_context6 = require("@dxos/context");
|
|
1581
1772
|
var import_credentials3 = require("@dxos/credentials");
|
|
1582
1773
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
@@ -1596,10 +1787,10 @@ function _ts_decorate6(decorators, target, key, desc) {
|
|
|
1596
1787
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1597
1788
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1598
1789
|
}
|
|
1599
|
-
var __dxlog_file10 = "/
|
|
1790
|
+
var __dxlog_file10 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
|
|
1600
1791
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1601
1792
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1602
|
-
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 =
|
|
1793
|
+
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 5e3;
|
|
1603
1794
|
var DataPipeline = class DataPipeline2 {
|
|
1604
1795
|
constructor(_params) {
|
|
1605
1796
|
this._params = _params;
|
|
@@ -1615,7 +1806,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1615
1806
|
this._mutations = new import_tracing2.TimeSeriesCounter();
|
|
1616
1807
|
this.currentEpoch = void 0;
|
|
1617
1808
|
this.appliedEpoch = void 0;
|
|
1618
|
-
this.onNewEpoch = new
|
|
1809
|
+
this.onNewEpoch = new import_async7.Event();
|
|
1619
1810
|
}
|
|
1620
1811
|
get isOpen() {
|
|
1621
1812
|
return this._isOpen;
|
|
@@ -1675,10 +1866,10 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1675
1866
|
}, options);
|
|
1676
1867
|
}
|
|
1677
1868
|
};
|
|
1678
|
-
this.databaseHost = new DatabaseHost(feedWriter);
|
|
1869
|
+
this.databaseHost = new DatabaseHost(feedWriter, () => this._flush());
|
|
1679
1870
|
this.itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
|
|
1680
1871
|
await this.databaseHost.open(this.itemManager, this._params.modelFactory);
|
|
1681
|
-
(0,
|
|
1872
|
+
(0, import_async7.scheduleTask)(this._ctx, async () => {
|
|
1682
1873
|
await this._consumePipeline();
|
|
1683
1874
|
});
|
|
1684
1875
|
this._isOpen = true;
|
|
@@ -1726,9 +1917,10 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1726
1917
|
await this._processEpochInSeparateTask(this.currentEpoch);
|
|
1727
1918
|
await waitForOneEpoch;
|
|
1728
1919
|
}
|
|
1920
|
+
let messageCounter = 0;
|
|
1729
1921
|
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1730
1922
|
F: __dxlog_file10,
|
|
1731
|
-
L:
|
|
1923
|
+
L: 217,
|
|
1732
1924
|
S: this,
|
|
1733
1925
|
A: [
|
|
1734
1926
|
"this._pipeline",
|
|
@@ -1744,7 +1936,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1744
1936
|
seq
|
|
1745
1937
|
}, {
|
|
1746
1938
|
F: __dxlog_file10,
|
|
1747
|
-
L:
|
|
1939
|
+
L: 223,
|
|
1748
1940
|
S: this,
|
|
1749
1941
|
C: (f, a) => f(...a)
|
|
1750
1942
|
});
|
|
@@ -1756,7 +1948,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1756
1948
|
feedKey
|
|
1757
1949
|
}, {
|
|
1758
1950
|
F: __dxlog_file10,
|
|
1759
|
-
L:
|
|
1951
|
+
L: 229,
|
|
1760
1952
|
S: this,
|
|
1761
1953
|
C: (f, a) => f(...a)
|
|
1762
1954
|
});
|
|
@@ -1779,7 +1971,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1779
1971
|
spaceKey: this._params.spaceKey.toHex()
|
|
1780
1972
|
}, {
|
|
1781
1973
|
F: __dxlog_file10,
|
|
1782
|
-
L:
|
|
1974
|
+
L: 246,
|
|
1783
1975
|
S: this,
|
|
1784
1976
|
C: (f, a) => f(...a)
|
|
1785
1977
|
});
|
|
@@ -1788,18 +1980,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1788
1980
|
} catch (err) {
|
|
1789
1981
|
import_log9.log.catch(err, void 0, {
|
|
1790
1982
|
F: __dxlog_file10,
|
|
1791
|
-
L:
|
|
1983
|
+
L: 256,
|
|
1792
1984
|
S: this,
|
|
1793
1985
|
C: (f, a) => f(...a)
|
|
1794
1986
|
});
|
|
1795
1987
|
}
|
|
1796
1988
|
span.end();
|
|
1989
|
+
if (++messageCounter > 1e3) {
|
|
1990
|
+
messageCounter = 0;
|
|
1991
|
+
await (0, import_async7.sleep)(1);
|
|
1992
|
+
}
|
|
1797
1993
|
}
|
|
1798
1994
|
}
|
|
1799
1995
|
_createSnapshot() {
|
|
1800
1996
|
(0, import_invariant8.invariant)(this.databaseHost, "Database backend is not initialized.", {
|
|
1801
1997
|
F: __dxlog_file10,
|
|
1802
|
-
L:
|
|
1998
|
+
L: 270,
|
|
1803
1999
|
S: this,
|
|
1804
2000
|
A: [
|
|
1805
2001
|
"this.databaseHost",
|
|
@@ -1828,7 +2024,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1828
2024
|
} catch (err) {
|
|
1829
2025
|
import_log9.log.warn("Failed to cache properties", err, {
|
|
1830
2026
|
F: __dxlog_file10,
|
|
1831
|
-
L:
|
|
2027
|
+
L: 299,
|
|
1832
2028
|
S: this,
|
|
1833
2029
|
C: (f, a) => f(...a)
|
|
1834
2030
|
});
|
|
@@ -1836,6 +2032,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1836
2032
|
await this._params.metadataStore.setCache(this._params.spaceKey, cache);
|
|
1837
2033
|
}
|
|
1838
2034
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
2035
|
+
if (!this._pipeline?.state.reachedTarget) {
|
|
2036
|
+
return;
|
|
2037
|
+
}
|
|
1839
2038
|
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
|
|
1840
2039
|
this._lastTimeframeSaveTime = Date.now();
|
|
1841
2040
|
await this._saveTargetTimeframe(timeframe);
|
|
@@ -1854,14 +2053,14 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1854
2053
|
if (err instanceof import_errors2.CancelledError) {
|
|
1855
2054
|
(0, import_log9.log)("Epoch processing cancelled.", void 0, {
|
|
1856
2055
|
F: __dxlog_file10,
|
|
1857
|
-
L:
|
|
2056
|
+
L: 335,
|
|
1858
2057
|
S: this,
|
|
1859
2058
|
C: (f, a) => f(...a)
|
|
1860
2059
|
});
|
|
1861
2060
|
} else {
|
|
1862
2061
|
import_log9.log.catch(err, void 0, {
|
|
1863
2062
|
F: __dxlog_file10,
|
|
1864
|
-
L:
|
|
2063
|
+
L: 337,
|
|
1865
2064
|
S: this,
|
|
1866
2065
|
C: (f, a) => f(...a)
|
|
1867
2066
|
});
|
|
@@ -1869,7 +2068,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1869
2068
|
}
|
|
1870
2069
|
});
|
|
1871
2070
|
this._epochCtx = ctx;
|
|
1872
|
-
(0,
|
|
2071
|
+
(0, import_async7.scheduleTask)(ctx, async () => {
|
|
1873
2072
|
if (!this._isOpen) {
|
|
1874
2073
|
return;
|
|
1875
2074
|
}
|
|
@@ -1881,7 +2080,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1881
2080
|
async _processEpoch(ctx, epoch) {
|
|
1882
2081
|
(0, import_invariant8.invariant)(this._isOpen, "Space is closed.", {
|
|
1883
2082
|
F: __dxlog_file10,
|
|
1884
|
-
L:
|
|
2083
|
+
L: 356,
|
|
1885
2084
|
S: this,
|
|
1886
2085
|
A: [
|
|
1887
2086
|
"this._isOpen",
|
|
@@ -1890,7 +2089,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1890
2089
|
});
|
|
1891
2090
|
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1892
2091
|
F: __dxlog_file10,
|
|
1893
|
-
L:
|
|
2092
|
+
L: 357,
|
|
1894
2093
|
S: this,
|
|
1895
2094
|
A: [
|
|
1896
2095
|
"this._pipeline",
|
|
@@ -1902,7 +2101,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1902
2101
|
epoch: (0, import_log9.omit)(epoch, "proof")
|
|
1903
2102
|
}, {
|
|
1904
2103
|
F: __dxlog_file10,
|
|
1905
|
-
L:
|
|
2104
|
+
L: 360,
|
|
1906
2105
|
S: this,
|
|
1907
2106
|
C: (f, a) => f(...a)
|
|
1908
2107
|
});
|
|
@@ -1912,7 +2111,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1912
2111
|
}
|
|
1913
2112
|
(0, import_log9.log)("restarting pipeline from epoch", void 0, {
|
|
1914
2113
|
F: __dxlog_file10,
|
|
1915
|
-
L:
|
|
2114
|
+
L: 366,
|
|
1916
2115
|
S: this,
|
|
1917
2116
|
C: (f, a) => f(...a)
|
|
1918
2117
|
});
|
|
@@ -1923,7 +2122,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1923
2122
|
async waitUntilTimeframe(timeframe) {
|
|
1924
2123
|
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1925
2124
|
F: __dxlog_file10,
|
|
1926
|
-
L:
|
|
2125
|
+
L: 373,
|
|
1927
2126
|
S: this,
|
|
1928
2127
|
A: [
|
|
1929
2128
|
"this._pipeline",
|
|
@@ -1935,7 +2134,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1935
2134
|
async createEpoch() {
|
|
1936
2135
|
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1937
2136
|
F: __dxlog_file10,
|
|
1938
|
-
L:
|
|
2137
|
+
L: 379,
|
|
1939
2138
|
S: this,
|
|
1940
2139
|
A: [
|
|
1941
2140
|
"this._pipeline",
|
|
@@ -1944,7 +2143,7 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1944
2143
|
});
|
|
1945
2144
|
(0, import_invariant8.invariant)(this.currentEpoch, void 0, {
|
|
1946
2145
|
F: __dxlog_file10,
|
|
1947
|
-
L:
|
|
2146
|
+
L: 380,
|
|
1948
2147
|
S: this,
|
|
1949
2148
|
A: [
|
|
1950
2149
|
"this.currentEpoch",
|
|
@@ -1966,6 +2165,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1966
2165
|
async ensureEpochInitialized() {
|
|
1967
2166
|
await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
|
|
1968
2167
|
}
|
|
2168
|
+
async _flush() {
|
|
2169
|
+
try {
|
|
2170
|
+
await this._saveCache();
|
|
2171
|
+
if (this._pipeline) {
|
|
2172
|
+
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
2173
|
+
}
|
|
2174
|
+
} catch (err) {
|
|
2175
|
+
import_log9.log.catch(err, void 0, {
|
|
2176
|
+
F: __dxlog_file10,
|
|
2177
|
+
L: 410,
|
|
2178
|
+
S: this,
|
|
2179
|
+
C: (f, a) => f(...a)
|
|
2180
|
+
});
|
|
2181
|
+
}
|
|
2182
|
+
await this._params.metadataStore.flush();
|
|
2183
|
+
}
|
|
1969
2184
|
};
|
|
1970
2185
|
_ts_decorate6([
|
|
1971
2186
|
import_tracing2.trace.metricsCounter()
|
|
@@ -1974,19 +2189,19 @@ _ts_decorate6([
|
|
|
1974
2189
|
import_tracing2.trace.metricsCounter()
|
|
1975
2190
|
], DataPipeline.prototype, "_mutations", void 0);
|
|
1976
2191
|
_ts_decorate6([
|
|
1977
|
-
|
|
2192
|
+
import_async7.synchronized
|
|
1978
2193
|
], DataPipeline.prototype, "open", null);
|
|
1979
2194
|
_ts_decorate6([
|
|
1980
|
-
|
|
2195
|
+
import_async7.synchronized
|
|
1981
2196
|
], DataPipeline.prototype, "close", null);
|
|
1982
2197
|
_ts_decorate6([
|
|
1983
|
-
|
|
2198
|
+
import_async7.synchronized
|
|
1984
2199
|
], DataPipeline.prototype, "_processEpoch", null);
|
|
1985
2200
|
_ts_decorate6([
|
|
1986
|
-
|
|
2201
|
+
import_async7.synchronized
|
|
1987
2202
|
], DataPipeline.prototype, "createEpoch", null);
|
|
1988
2203
|
DataPipeline = _ts_decorate6([
|
|
1989
|
-
(0,
|
|
2204
|
+
(0, import_async7.trackLeaks)("open", "close"),
|
|
1990
2205
|
import_tracing2.trace.resource()
|
|
1991
2206
|
], DataPipeline);
|
|
1992
2207
|
|
|
@@ -2001,12 +2216,12 @@ function _ts_decorate7(decorators, target, key, desc) {
|
|
|
2001
2216
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2002
2217
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2003
2218
|
}
|
|
2004
|
-
var __dxlog_file11 = "/
|
|
2219
|
+
var __dxlog_file11 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space.ts";
|
|
2005
2220
|
var Space = class Space2 {
|
|
2006
2221
|
constructor(params) {
|
|
2007
|
-
this._addFeedLock = new
|
|
2222
|
+
this._addFeedLock = new import_async8.Lock();
|
|
2008
2223
|
this.onCredentialProcessed = new import_util7.Callback();
|
|
2009
|
-
this.stateUpdate = new
|
|
2224
|
+
this.stateUpdate = new import_async8.Event();
|
|
2010
2225
|
this._isOpen = false;
|
|
2011
2226
|
(0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
|
|
2012
2227
|
F: __dxlog_file11,
|
|
@@ -2030,7 +2245,7 @@ var Space = class Space2 {
|
|
|
2030
2245
|
this._controlPipeline.onFeedAdmitted.set(async (info) => {
|
|
2031
2246
|
const sparse = info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA;
|
|
2032
2247
|
if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA) {
|
|
2033
|
-
|
|
2248
|
+
queueMicrotask(async () => {
|
|
2034
2249
|
if (this._dataPipeline.pipeline) {
|
|
2035
2250
|
if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
|
|
2036
2251
|
return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
|
|
@@ -2041,9 +2256,11 @@ var Space = class Space2 {
|
|
|
2041
2256
|
});
|
|
2042
2257
|
}
|
|
2043
2258
|
if (!info.key.equals(params.genesisFeed.key)) {
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2259
|
+
queueMicrotask(async () => {
|
|
2260
|
+
this.protocol.addFeed(await params.feedProvider(info.key, {
|
|
2261
|
+
sparse
|
|
2262
|
+
}));
|
|
2263
|
+
});
|
|
2047
2264
|
}
|
|
2048
2265
|
});
|
|
2049
2266
|
this._controlPipeline.onCredentialProcessed.set(async (credential) => {
|
|
@@ -2052,7 +2269,7 @@ var Space = class Space2 {
|
|
|
2052
2269
|
credential
|
|
2053
2270
|
}, {
|
|
2054
2271
|
F: __dxlog_file11,
|
|
2055
|
-
L:
|
|
2272
|
+
L: 111,
|
|
2056
2273
|
S: this,
|
|
2057
2274
|
C: (f, a) => f(...a)
|
|
2058
2275
|
});
|
|
@@ -2117,7 +2334,7 @@ var Space = class Space2 {
|
|
|
2117
2334
|
setControlFeed(feed) {
|
|
2118
2335
|
(0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
|
|
2119
2336
|
F: __dxlog_file11,
|
|
2120
|
-
L:
|
|
2337
|
+
L: 186,
|
|
2121
2338
|
S: this,
|
|
2122
2339
|
A: [
|
|
2123
2340
|
"!this._controlFeed",
|
|
@@ -2131,7 +2348,7 @@ var Space = class Space2 {
|
|
|
2131
2348
|
setDataFeed(feed) {
|
|
2132
2349
|
(0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
|
|
2133
2350
|
F: __dxlog_file11,
|
|
2134
|
-
L:
|
|
2351
|
+
L: 193,
|
|
2135
2352
|
S: this,
|
|
2136
2353
|
A: [
|
|
2137
2354
|
"!this._dataFeed",
|
|
@@ -2157,7 +2374,7 @@ var Space = class Space2 {
|
|
|
2157
2374
|
async open(ctx) {
|
|
2158
2375
|
(0, import_log10.log)("opening...", void 0, {
|
|
2159
2376
|
F: __dxlog_file11,
|
|
2160
|
-
L:
|
|
2377
|
+
L: 215,
|
|
2161
2378
|
S: this,
|
|
2162
2379
|
C: (f, a) => f(...a)
|
|
2163
2380
|
});
|
|
@@ -2170,7 +2387,7 @@ var Space = class Space2 {
|
|
|
2170
2387
|
this._isOpen = true;
|
|
2171
2388
|
(0, import_log10.log)("opened", void 0, {
|
|
2172
2389
|
F: __dxlog_file11,
|
|
2173
|
-
L:
|
|
2390
|
+
L: 226,
|
|
2174
2391
|
S: this,
|
|
2175
2392
|
C: (f, a) => f(...a)
|
|
2176
2393
|
});
|
|
@@ -2180,7 +2397,7 @@ var Space = class Space2 {
|
|
|
2180
2397
|
key: this._key
|
|
2181
2398
|
}, {
|
|
2182
2399
|
F: __dxlog_file11,
|
|
2183
|
-
L:
|
|
2400
|
+
L: 231,
|
|
2184
2401
|
S: this,
|
|
2185
2402
|
C: (f, a) => f(...a)
|
|
2186
2403
|
});
|
|
@@ -2194,7 +2411,7 @@ var Space = class Space2 {
|
|
|
2194
2411
|
this._isOpen = false;
|
|
2195
2412
|
(0, import_log10.log)("closed", void 0, {
|
|
2196
2413
|
F: __dxlog_file11,
|
|
2197
|
-
L:
|
|
2414
|
+
L: 244,
|
|
2198
2415
|
S: this,
|
|
2199
2416
|
C: (f, a) => f(...a)
|
|
2200
2417
|
});
|
|
@@ -2202,13 +2419,13 @@ var Space = class Space2 {
|
|
|
2202
2419
|
async initializeDataPipeline() {
|
|
2203
2420
|
(0, import_log10.log)("initializeDataPipeline", void 0, {
|
|
2204
2421
|
F: __dxlog_file11,
|
|
2205
|
-
L:
|
|
2422
|
+
L: 249,
|
|
2206
2423
|
S: this,
|
|
2207
2424
|
C: (f, a) => f(...a)
|
|
2208
2425
|
});
|
|
2209
2426
|
(0, import_invariant9.invariant)(this._isOpen, "Space must be open to initialize data pipeline.", {
|
|
2210
2427
|
F: __dxlog_file11,
|
|
2211
|
-
L:
|
|
2428
|
+
L: 250,
|
|
2212
2429
|
S: this,
|
|
2213
2430
|
A: [
|
|
2214
2431
|
"this._isOpen",
|
|
@@ -2223,31 +2440,31 @@ _ts_decorate7([
|
|
|
2223
2440
|
import_tracing3.trace.info()
|
|
2224
2441
|
], Space.prototype, "key", null);
|
|
2225
2442
|
_ts_decorate7([
|
|
2226
|
-
|
|
2443
|
+
import_async8.synchronized,
|
|
2227
2444
|
import_tracing3.trace.span()
|
|
2228
2445
|
], Space.prototype, "open", null);
|
|
2229
2446
|
_ts_decorate7([
|
|
2230
|
-
|
|
2447
|
+
import_async8.synchronized
|
|
2231
2448
|
], Space.prototype, "close", null);
|
|
2232
2449
|
_ts_decorate7([
|
|
2233
|
-
|
|
2450
|
+
import_async8.synchronized
|
|
2234
2451
|
], Space.prototype, "initializeDataPipeline", null);
|
|
2235
2452
|
Space = _ts_decorate7([
|
|
2236
|
-
(0,
|
|
2453
|
+
(0, import_async8.trackLeaks)("open", "close"),
|
|
2237
2454
|
import_tracing3.trace.resource()
|
|
2238
2455
|
], Space);
|
|
2239
2456
|
|
|
2240
2457
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
2241
|
-
var
|
|
2458
|
+
var import_async9 = require("@dxos/async");
|
|
2242
2459
|
var import_debug4 = require("@dxos/debug");
|
|
2243
|
-
var
|
|
2460
|
+
var import_keys7 = require("@dxos/keys");
|
|
2244
2461
|
var import_log12 = require("@dxos/log");
|
|
2245
2462
|
var import_protocols6 = require("@dxos/protocols");
|
|
2246
2463
|
var import_util9 = require("@dxos/util");
|
|
2247
2464
|
|
|
2248
2465
|
// packages/core/echo/echo-pipeline/src/space/space-protocol.ts
|
|
2249
2466
|
var import_crypto3 = require("@dxos/crypto");
|
|
2250
|
-
var
|
|
2467
|
+
var import_keys6 = require("@dxos/keys");
|
|
2251
2468
|
var import_log11 = require("@dxos/log");
|
|
2252
2469
|
var import_network_manager = require("@dxos/network-manager");
|
|
2253
2470
|
var import_teleport2 = require("@dxos/teleport");
|
|
@@ -2264,13 +2481,13 @@ function _ts_decorate8(decorators, target, key, desc) {
|
|
|
2264
2481
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2265
2482
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2266
2483
|
}
|
|
2267
|
-
var __dxlog_file12 = "/
|
|
2484
|
+
var __dxlog_file12 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
|
|
2268
2485
|
var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
|
|
2269
2486
|
var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
|
|
2270
2487
|
var SpaceProtocol = class {
|
|
2271
2488
|
constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
|
|
2272
2489
|
this._feeds = /* @__PURE__ */ new Set();
|
|
2273
|
-
this._sessions = new import_util8.ComplexMap(
|
|
2490
|
+
this._sessions = new import_util8.ComplexMap(import_keys6.PublicKey.hash);
|
|
2274
2491
|
this._networkManager = networkManager;
|
|
2275
2492
|
this._swarmIdentity = swarmIdentity;
|
|
2276
2493
|
this._onSessionAuth = onSessionAuth;
|
|
@@ -2278,7 +2495,7 @@ var SpaceProtocol = class {
|
|
|
2278
2495
|
this.blobSync = new import_teleport_extension_object_sync.BlobSync({
|
|
2279
2496
|
blobStore
|
|
2280
2497
|
});
|
|
2281
|
-
this._topic = import_crypto3.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto3.discoveryKey).then(
|
|
2498
|
+
this._topic = import_crypto3.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto3.discoveryKey).then(import_keys6.PublicKey.from);
|
|
2282
2499
|
}
|
|
2283
2500
|
get sessions() {
|
|
2284
2501
|
return this._sessions;
|
|
@@ -2408,7 +2625,7 @@ var SpaceProtocolSession = class {
|
|
|
2408
2625
|
get stream() {
|
|
2409
2626
|
return this._teleport.stream;
|
|
2410
2627
|
}
|
|
2411
|
-
async
|
|
2628
|
+
async open() {
|
|
2412
2629
|
await this._teleport.open();
|
|
2413
2630
|
this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
|
|
2414
2631
|
provider: this._swarmIdentity.credentialProvider,
|
|
@@ -2431,9 +2648,18 @@ var SpaceProtocolSession = class {
|
|
|
2431
2648
|
this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
|
|
2432
2649
|
this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
|
|
2433
2650
|
}
|
|
2434
|
-
async
|
|
2651
|
+
async close() {
|
|
2652
|
+
import_log11.log.info("close", void 0, {
|
|
2653
|
+
F: __dxlog_file12,
|
|
2654
|
+
L: 254,
|
|
2655
|
+
S: this,
|
|
2656
|
+
C: (f, a) => f(...a)
|
|
2657
|
+
});
|
|
2435
2658
|
await this._teleport.close();
|
|
2436
2659
|
}
|
|
2660
|
+
async abort() {
|
|
2661
|
+
await this._teleport.abort();
|
|
2662
|
+
}
|
|
2437
2663
|
};
|
|
2438
2664
|
_ts_decorate8([
|
|
2439
2665
|
import_log11.logInfo
|
|
@@ -2453,11 +2679,11 @@ function _ts_decorate9(decorators, target, key, desc) {
|
|
|
2453
2679
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2454
2680
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2455
2681
|
}
|
|
2456
|
-
var __dxlog_file13 = "/
|
|
2682
|
+
var __dxlog_file13 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
|
|
2457
2683
|
var SpaceManager = class SpaceManager2 {
|
|
2458
2684
|
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
|
|
2459
|
-
this._spaces = new import_util9.ComplexMap(
|
|
2460
|
-
this._instanceId =
|
|
2685
|
+
this._spaces = new import_util9.ComplexMap(import_keys7.PublicKey.hash);
|
|
2686
|
+
this._instanceId = import_keys7.PublicKey.random().toHex();
|
|
2461
2687
|
this._feedStore = feedStore;
|
|
2462
2688
|
this._networkManager = networkManager;
|
|
2463
2689
|
this._modelFactory = modelFactory;
|
|
@@ -2527,13 +2753,13 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2527
2753
|
}
|
|
2528
2754
|
};
|
|
2529
2755
|
_ts_decorate9([
|
|
2530
|
-
|
|
2756
|
+
import_async9.synchronized
|
|
2531
2757
|
], SpaceManager.prototype, "open", null);
|
|
2532
2758
|
_ts_decorate9([
|
|
2533
|
-
|
|
2759
|
+
import_async9.synchronized
|
|
2534
2760
|
], SpaceManager.prototype, "close", null);
|
|
2535
2761
|
SpaceManager = _ts_decorate9([
|
|
2536
|
-
(0,
|
|
2762
|
+
(0, import_async9.trackLeaks)("open", "close")
|
|
2537
2763
|
], SpaceManager);
|
|
2538
2764
|
// Annotate the CommonJS export names for ESM import in node:
|
|
2539
2765
|
0 && (module.exports = {
|