@dxos/echo-pipeline 0.1.56-main.e47dfd1 → 0.1.56-main.ec576e1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-7U2NXI2P.mjs → chunk-GELWTWOW.mjs} +694 -355
- package/dist/lib/browser/chunk-GELWTWOW.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/index.mjs.map +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +14 -18
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/index.cjs +701 -362
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +698 -363
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/common/feeds.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service-host.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/snapshot-manager.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts +2 -0
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts.map +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts +4 -3
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/auth.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +4 -0
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +2 -0
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +2 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts +4 -1
- package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/package.json +32 -31
- package/src/common/feeds.ts +1 -2
- package/src/db-host/data-service-host.ts +10 -4
- package/src/db-host/data-service.ts +1 -2
- package/src/db-host/snapshot-manager.ts +0 -2
- package/src/metadata/metadata-store.ts +2 -2
- package/src/pipeline/message-selector.ts +1 -2
- package/src/pipeline/pipeline-stress.test.ts +226 -0
- package/src/pipeline/pipeline.test.ts +30 -206
- package/src/pipeline/pipeline.ts +50 -24
- package/src/space/auth.ts +1 -2
- package/src/space/control-pipeline.ts +47 -22
- package/src/space/data-pipeline.ts +21 -12
- package/src/space/space-manager.ts +1 -0
- package/src/space/space-protocol.ts +3 -1
- package/src/space/space.test.ts +18 -17
- package/src/space/space.ts +7 -3
- package/src/testing/database-test-rig.ts +27 -2
- package/src/testing/test-agent-builder.ts +2 -1
- package/src/tests/database-unit.test.ts +39 -0
- package/dist/lib/browser/chunk-7U2NXI2P.mjs.map +0 -7
|
@@ -42,6 +42,7 @@ __export(testing_exports, {
|
|
|
42
42
|
module.exports = __toCommonJS(testing_exports);
|
|
43
43
|
|
|
44
44
|
// packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
|
|
45
|
+
var import_context7 = require("@dxos/context");
|
|
45
46
|
var import_credentials5 = require("@dxos/credentials");
|
|
46
47
|
var import_document_model = require("@dxos/document-model");
|
|
47
48
|
var import_keys7 = require("@dxos/keys");
|
|
@@ -55,11 +56,12 @@ var import_teleport_extension_object_sync2 = require("@dxos/teleport-extension-o
|
|
|
55
56
|
var import_util10 = require("@dxos/util");
|
|
56
57
|
|
|
57
58
|
// packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
|
|
58
|
-
var import_tiny_invariant = __toESM(require("tiny-invariant"));
|
|
59
59
|
var import_codec_protobuf = require("@dxos/codec-protobuf");
|
|
60
60
|
var import_context = require("@dxos/context");
|
|
61
61
|
var import_echo_db = require("@dxos/echo-db");
|
|
62
|
+
var import_invariant = require("@dxos/invariant");
|
|
62
63
|
var import_log = require("@dxos/log");
|
|
64
|
+
var import_service = require("@dxos/protocols/proto/dxos/echo/service");
|
|
63
65
|
var import_util = require("@dxos/util");
|
|
64
66
|
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
|
|
65
67
|
var DataServiceHost = class {
|
|
@@ -87,16 +89,31 @@ var DataServiceHost = class {
|
|
|
87
89
|
objects
|
|
88
90
|
}
|
|
89
91
|
});
|
|
92
|
+
this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
|
|
93
|
+
next({
|
|
94
|
+
action: import_service.EchoEvent.DatabaseAction.RESET,
|
|
95
|
+
batch: {
|
|
96
|
+
objects: snapshot.items
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
});
|
|
90
100
|
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
91
|
-
var _a;
|
|
92
101
|
const { batch, meta } = message;
|
|
93
|
-
(0,
|
|
102
|
+
(0, import_invariant.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
|
|
103
|
+
F: __dxlog_file,
|
|
104
|
+
L: 68,
|
|
105
|
+
S: this,
|
|
106
|
+
A: [
|
|
107
|
+
"!(meta as any).clientTag",
|
|
108
|
+
"'Unexpected client tag in mutation message'"
|
|
109
|
+
]
|
|
110
|
+
});
|
|
94
111
|
(0, import_log.log)("message", {
|
|
95
|
-
batch,
|
|
112
|
+
batch: batch.objects?.length,
|
|
96
113
|
meta
|
|
97
114
|
}, {
|
|
98
115
|
F: __dxlog_file,
|
|
99
|
-
L:
|
|
116
|
+
L: 69,
|
|
100
117
|
S: this,
|
|
101
118
|
C: (f, a) => f(...a)
|
|
102
119
|
});
|
|
@@ -104,7 +121,7 @@ var DataServiceHost = class {
|
|
|
104
121
|
message.meta.feedKey,
|
|
105
122
|
message.meta.seq
|
|
106
123
|
]);
|
|
107
|
-
|
|
124
|
+
batch.objects?.forEach((object) => {
|
|
108
125
|
(0, import_echo_db.setMetadataOnObject)(object, {
|
|
109
126
|
...meta
|
|
110
127
|
});
|
|
@@ -122,15 +139,30 @@ var DataServiceHost = class {
|
|
|
122
139
|
});
|
|
123
140
|
}
|
|
124
141
|
async write(request) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
142
|
+
(0, import_invariant.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
|
|
143
|
+
F: __dxlog_file,
|
|
144
|
+
L: 95,
|
|
145
|
+
S: this,
|
|
146
|
+
A: [
|
|
147
|
+
"!this._ctx.disposed",
|
|
148
|
+
"'Cannot write to closed DataServiceHost'"
|
|
149
|
+
]
|
|
150
|
+
});
|
|
151
|
+
(0, import_invariant.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
|
|
152
|
+
F: __dxlog_file,
|
|
153
|
+
L: 96,
|
|
154
|
+
S: this,
|
|
155
|
+
A: [
|
|
156
|
+
"this._writeStream",
|
|
157
|
+
"'Cannot write mutations in readonly mode'"
|
|
158
|
+
]
|
|
159
|
+
});
|
|
128
160
|
(0, import_log.log)("write", {
|
|
129
161
|
clientTag: request.clientTag,
|
|
130
|
-
objectCount:
|
|
162
|
+
objectCount: request.batch.objects?.length ?? 0
|
|
131
163
|
}, {
|
|
132
164
|
F: __dxlog_file,
|
|
133
|
-
L:
|
|
165
|
+
L: 98,
|
|
134
166
|
S: this,
|
|
135
167
|
C: (f, a) => f(...a)
|
|
136
168
|
});
|
|
@@ -144,7 +176,7 @@ var DataServiceHost = class {
|
|
|
144
176
|
seq: receipt2.seq
|
|
145
177
|
}, {
|
|
146
178
|
F: __dxlog_file,
|
|
147
|
-
L:
|
|
179
|
+
L: 107,
|
|
148
180
|
S: this,
|
|
149
181
|
C: (f, a) => f(...a)
|
|
150
182
|
});
|
|
@@ -158,24 +190,18 @@ var DataServiceHost = class {
|
|
|
158
190
|
return receipt;
|
|
159
191
|
}
|
|
160
192
|
};
|
|
161
|
-
var createDataMessage = (batch) => {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
meta: void 0
|
|
174
|
-
};
|
|
175
|
-
})
|
|
176
|
-
}
|
|
177
|
-
};
|
|
178
|
-
};
|
|
193
|
+
var createDataMessage = (batch) => ({
|
|
194
|
+
batch: {
|
|
195
|
+
objects: batch.objects?.map((object) => ({
|
|
196
|
+
...object,
|
|
197
|
+
mutations: object.mutations?.map((mutation) => ({
|
|
198
|
+
...mutation,
|
|
199
|
+
meta: void 0
|
|
200
|
+
})),
|
|
201
|
+
meta: void 0
|
|
202
|
+
}))
|
|
203
|
+
}
|
|
204
|
+
});
|
|
179
205
|
|
|
180
206
|
// packages/core/echo/echo-pipeline/src/db-host/database-host.ts
|
|
181
207
|
var import_echo_db2 = require("@dxos/echo-db");
|
|
@@ -204,15 +230,13 @@ var DatabaseHost = class {
|
|
|
204
230
|
return this._itemDemuxer.createSnapshot();
|
|
205
231
|
}
|
|
206
232
|
createDataServiceHost() {
|
|
207
|
-
|
|
208
|
-
return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
|
|
233
|
+
return new DataServiceHost(this._itemManager, this._itemDemuxer, this._outboundStream ?? void 0);
|
|
209
234
|
}
|
|
210
235
|
};
|
|
211
236
|
|
|
212
237
|
// packages/core/echo/echo-pipeline/src/db-host/snapshot-manager.ts
|
|
213
238
|
var import_async = require("@dxos/async");
|
|
214
239
|
var import_context2 = require("@dxos/context");
|
|
215
|
-
var import_debug = require("@dxos/debug");
|
|
216
240
|
var import_keys = require("@dxos/keys");
|
|
217
241
|
var import_protocols = require("@dxos/protocols");
|
|
218
242
|
var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
@@ -257,9 +281,6 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
257
281
|
return import_keys.PublicKey.from(id).toHex();
|
|
258
282
|
}
|
|
259
283
|
};
|
|
260
|
-
_ts_decorate([
|
|
261
|
-
(0, import_debug.timed)(1e4)
|
|
262
|
-
], SnapshotManager.prototype, "load", null);
|
|
263
284
|
SnapshotManager = _ts_decorate([
|
|
264
285
|
(0, import_async.trackLeaks)("open", "close")
|
|
265
286
|
], SnapshotManager);
|
|
@@ -311,8 +332,8 @@ var SnapshotStore = class {
|
|
|
311
332
|
};
|
|
312
333
|
|
|
313
334
|
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
314
|
-
var
|
|
315
|
-
var
|
|
335
|
+
var import_debug = require("@dxos/debug");
|
|
336
|
+
var import_invariant2 = require("@dxos/invariant");
|
|
316
337
|
var import_keys2 = require("@dxos/keys");
|
|
317
338
|
var import_log2 = require("@dxos/log");
|
|
318
339
|
var import_util2 = require("@dxos/util");
|
|
@@ -329,11 +350,19 @@ var DataServiceSubscriptions = class {
|
|
|
329
350
|
spaceKey
|
|
330
351
|
}, {
|
|
331
352
|
F: __dxlog_file2,
|
|
332
|
-
L:
|
|
353
|
+
L: 30,
|
|
333
354
|
S: this,
|
|
334
355
|
C: (f, a) => f(...a)
|
|
335
356
|
});
|
|
336
|
-
(0,
|
|
357
|
+
(0, import_invariant2.invariant)(!this._spaces.has(spaceKey), void 0, {
|
|
358
|
+
F: __dxlog_file2,
|
|
359
|
+
L: 31,
|
|
360
|
+
S: this,
|
|
361
|
+
A: [
|
|
362
|
+
"!this._spaces.has(spaceKey)",
|
|
363
|
+
""
|
|
364
|
+
]
|
|
365
|
+
});
|
|
337
366
|
await host.open();
|
|
338
367
|
this._spaces.set(spaceKey, host);
|
|
339
368
|
}
|
|
@@ -342,12 +371,12 @@ var DataServiceSubscriptions = class {
|
|
|
342
371
|
spaceKey
|
|
343
372
|
}, {
|
|
344
373
|
F: __dxlog_file2,
|
|
345
|
-
L:
|
|
374
|
+
L: 37,
|
|
346
375
|
S: this,
|
|
347
376
|
C: (f, a) => f(...a)
|
|
348
377
|
});
|
|
349
378
|
const host = this._spaces.get(spaceKey);
|
|
350
|
-
await
|
|
379
|
+
await host?.close();
|
|
351
380
|
this._spaces.delete(spaceKey);
|
|
352
381
|
}
|
|
353
382
|
getDataService(spaceKey) {
|
|
@@ -359,25 +388,47 @@ var DataServiceImpl = class {
|
|
|
359
388
|
this._subscriptions = _subscriptions;
|
|
360
389
|
}
|
|
361
390
|
subscribe(request) {
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
391
|
+
(0, import_invariant2.invariant)(request.spaceKey, void 0, {
|
|
392
|
+
F: __dxlog_file2,
|
|
393
|
+
L: 56,
|
|
394
|
+
S: this,
|
|
395
|
+
A: [
|
|
396
|
+
"request.spaceKey",
|
|
397
|
+
""
|
|
398
|
+
]
|
|
399
|
+
});
|
|
400
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
365
401
|
return host.subscribe();
|
|
366
402
|
}
|
|
367
403
|
write(request) {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
404
|
+
(0, import_invariant2.invariant)(request.spaceKey, void 0, {
|
|
405
|
+
F: __dxlog_file2,
|
|
406
|
+
L: 63,
|
|
407
|
+
S: this,
|
|
408
|
+
A: [
|
|
409
|
+
"request.spaceKey",
|
|
410
|
+
""
|
|
411
|
+
]
|
|
412
|
+
});
|
|
413
|
+
(0, import_invariant2.invariant)(request.batch, void 0, {
|
|
414
|
+
F: __dxlog_file2,
|
|
415
|
+
L: 64,
|
|
416
|
+
S: this,
|
|
417
|
+
A: [
|
|
418
|
+
"request.batch",
|
|
419
|
+
""
|
|
420
|
+
]
|
|
421
|
+
});
|
|
422
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
372
423
|
return host.write(request);
|
|
373
424
|
}
|
|
374
425
|
};
|
|
375
426
|
|
|
376
427
|
// packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
|
|
377
428
|
var import_crc_32 = __toESM(require("crc-32"));
|
|
378
|
-
var import_tiny_invariant3 = __toESM(require("tiny-invariant"));
|
|
379
429
|
var import_async2 = require("@dxos/async");
|
|
380
430
|
var import_errors = require("@dxos/errors");
|
|
431
|
+
var import_invariant3 = require("@dxos/invariant");
|
|
381
432
|
var import_log3 = require("@dxos/log");
|
|
382
433
|
var import_protocols3 = require("@dxos/protocols");
|
|
383
434
|
var import_services = require("@dxos/protocols/proto/dxos/client/services");
|
|
@@ -396,8 +447,8 @@ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipelin
|
|
|
396
447
|
var emptyEchoMetadata = () => ({
|
|
397
448
|
version: import_protocols3.STORAGE_VERSION,
|
|
398
449
|
spaces: [],
|
|
399
|
-
created: new Date(),
|
|
400
|
-
updated: new Date()
|
|
450
|
+
created: /* @__PURE__ */ new Date(),
|
|
451
|
+
updated: /* @__PURE__ */ new Date()
|
|
401
452
|
});
|
|
402
453
|
var EchoMetadata = import_protocols3.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
403
454
|
var MetadataStore = class {
|
|
@@ -411,22 +462,19 @@ var MetadataStore = class {
|
|
|
411
462
|
return this._metadata;
|
|
412
463
|
}
|
|
413
464
|
get version() {
|
|
414
|
-
|
|
415
|
-
return (_a = this._metadata.version) != null ? _a : 0;
|
|
465
|
+
return this._metadata.version ?? 0;
|
|
416
466
|
}
|
|
417
467
|
/**
|
|
418
468
|
* Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
|
|
419
469
|
* addSpaceFeed functions.
|
|
420
470
|
*/
|
|
421
471
|
get spaces() {
|
|
422
|
-
|
|
423
|
-
return (_a = this._metadata.spaces) != null ? _a : [];
|
|
472
|
+
return this._metadata.spaces ?? [];
|
|
424
473
|
}
|
|
425
474
|
/**
|
|
426
475
|
* Loads metadata from persistent storage.
|
|
427
476
|
*/
|
|
428
477
|
async load() {
|
|
429
|
-
var _a;
|
|
430
478
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
431
479
|
try {
|
|
432
480
|
const { size: fileLength } = await file.stat();
|
|
@@ -445,7 +493,10 @@ var MetadataStore = class {
|
|
|
445
493
|
C: (f, a) => f(...a)
|
|
446
494
|
});
|
|
447
495
|
if (fileLength < dataSize + 8) {
|
|
448
|
-
throw new import_errors.DataCorruptionError("Metadata size is smaller than expected."
|
|
496
|
+
throw new import_errors.DataCorruptionError("Metadata size is smaller than expected.", {
|
|
497
|
+
fileLength,
|
|
498
|
+
dataSize
|
|
499
|
+
});
|
|
449
500
|
}
|
|
450
501
|
const data = await file.read(8, dataSize);
|
|
451
502
|
const calculatedChecksum = import_crc_32.default.buf(data);
|
|
@@ -453,9 +504,8 @@ var MetadataStore = class {
|
|
|
453
504
|
throw new import_errors.DataCorruptionError("Metadata checksum is invalid.");
|
|
454
505
|
}
|
|
455
506
|
this._metadata = EchoMetadata.decode(data);
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
(_a2 = space.state) != null ? _a2 : space.state = import_services.SpaceState.ACTIVE;
|
|
507
|
+
this._metadata.spaces?.forEach((space) => {
|
|
508
|
+
space.state ??= import_services.SpaceState.ACTIVE;
|
|
459
509
|
});
|
|
460
510
|
} catch (err) {
|
|
461
511
|
import_log3.log.error("failed to load metadata", {
|
|
@@ -472,12 +522,11 @@ var MetadataStore = class {
|
|
|
472
522
|
}
|
|
473
523
|
}
|
|
474
524
|
async _save() {
|
|
475
|
-
var _a;
|
|
476
525
|
const data = {
|
|
477
526
|
...this._metadata,
|
|
478
527
|
version: import_protocols3.STORAGE_VERSION,
|
|
479
|
-
created:
|
|
480
|
-
updated: new Date()
|
|
528
|
+
created: this._metadata.created ?? /* @__PURE__ */ new Date(),
|
|
529
|
+
updated: /* @__PURE__ */ new Date()
|
|
481
530
|
};
|
|
482
531
|
this.update.emit(data);
|
|
483
532
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
@@ -503,12 +552,19 @@ var MetadataStore = class {
|
|
|
503
552
|
}
|
|
504
553
|
}
|
|
505
554
|
_getSpace(spaceKey) {
|
|
506
|
-
|
|
507
|
-
if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
|
|
555
|
+
if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
|
|
508
556
|
return this._metadata.identity.haloSpace;
|
|
509
557
|
}
|
|
510
558
|
const space = this.spaces.find((space2) => space2.key === spaceKey);
|
|
511
|
-
(0,
|
|
559
|
+
(0, import_invariant3.invariant)(space, "Space not found", {
|
|
560
|
+
F: __dxlog_file3,
|
|
561
|
+
L: 137,
|
|
562
|
+
S: this,
|
|
563
|
+
A: [
|
|
564
|
+
"space",
|
|
565
|
+
"'Space not found'"
|
|
566
|
+
]
|
|
567
|
+
});
|
|
512
568
|
return space;
|
|
513
569
|
}
|
|
514
570
|
/**
|
|
@@ -528,14 +584,29 @@ var MetadataStore = class {
|
|
|
528
584
|
return this._metadata.identity;
|
|
529
585
|
}
|
|
530
586
|
async setIdentityRecord(record) {
|
|
531
|
-
(0,
|
|
587
|
+
(0, import_invariant3.invariant)(!this._metadata.identity, "Cannot overwrite existing identity in metadata", {
|
|
588
|
+
F: __dxlog_file3,
|
|
589
|
+
L: 155,
|
|
590
|
+
S: this,
|
|
591
|
+
A: [
|
|
592
|
+
"!this._metadata.identity",
|
|
593
|
+
"'Cannot overwrite existing identity in metadata'"
|
|
594
|
+
]
|
|
595
|
+
});
|
|
532
596
|
this._metadata.identity = record;
|
|
533
597
|
await this._save();
|
|
534
598
|
}
|
|
535
599
|
async addSpace(record) {
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
600
|
+
(0, import_invariant3.invariant)(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
|
|
601
|
+
F: __dxlog_file3,
|
|
602
|
+
L: 162,
|
|
603
|
+
S: this,
|
|
604
|
+
A: [
|
|
605
|
+
"!(this._metadata.spaces ?? []).find((space) => space.key === record.key)",
|
|
606
|
+
"'Cannot overwrite existing space in metadata'"
|
|
607
|
+
]
|
|
608
|
+
});
|
|
609
|
+
(this._metadata.spaces ??= []).push(record);
|
|
539
610
|
await this._save();
|
|
540
611
|
}
|
|
541
612
|
async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
|
|
@@ -570,10 +641,10 @@ _ts_decorate2([
|
|
|
570
641
|
var fromBytesInt32 = (buf) => buf.readInt32LE(0);
|
|
571
642
|
|
|
572
643
|
// packages/core/echo/echo-pipeline/src/space/auth.ts
|
|
573
|
-
var import_tiny_invariant4 = __toESM(require("tiny-invariant"));
|
|
574
644
|
var import_async3 = require("@dxos/async");
|
|
575
645
|
var import_context3 = require("@dxos/context");
|
|
576
646
|
var import_crypto2 = require("@dxos/crypto");
|
|
647
|
+
var import_invariant4 = require("@dxos/invariant");
|
|
577
648
|
var import_log4 = require("@dxos/log");
|
|
578
649
|
var import_protocols4 = require("@dxos/protocols");
|
|
579
650
|
var import_teleport = require("@dxos/teleport");
|
|
@@ -594,7 +665,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
594
665
|
onError: (err) => {
|
|
595
666
|
import_log4.log.catch(err, void 0, {
|
|
596
667
|
F: __dxlog_file4,
|
|
597
|
-
L:
|
|
668
|
+
L: 28,
|
|
598
669
|
S: this,
|
|
599
670
|
C: (f, a) => f(...a)
|
|
600
671
|
});
|
|
@@ -616,7 +687,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
616
687
|
} catch (err) {
|
|
617
688
|
import_log4.log.error("failed to generate auth credentials", err, {
|
|
618
689
|
F: __dxlog_file4,
|
|
619
|
-
L:
|
|
690
|
+
L: 55,
|
|
620
691
|
S: this,
|
|
621
692
|
C: (f, a) => f(...a)
|
|
622
693
|
});
|
|
@@ -634,14 +705,30 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
634
705
|
const { credential } = await this.rpc.AuthService.authenticate({
|
|
635
706
|
challenge
|
|
636
707
|
});
|
|
637
|
-
(0,
|
|
708
|
+
(0, import_invariant4.invariant)(credential?.length > 0, "invalid credential", {
|
|
709
|
+
F: __dxlog_file4,
|
|
710
|
+
L: 69,
|
|
711
|
+
S: this,
|
|
712
|
+
A: [
|
|
713
|
+
"credential?.length > 0",
|
|
714
|
+
"'invalid credential'"
|
|
715
|
+
]
|
|
716
|
+
});
|
|
638
717
|
const success = await this._authParams.verifier(challenge, credential);
|
|
639
|
-
(0,
|
|
718
|
+
(0, import_invariant4.invariant)(success, "credential not verified", {
|
|
719
|
+
F: __dxlog_file4,
|
|
720
|
+
L: 71,
|
|
721
|
+
S: this,
|
|
722
|
+
A: [
|
|
723
|
+
"success",
|
|
724
|
+
"'credential not verified'"
|
|
725
|
+
]
|
|
726
|
+
});
|
|
640
727
|
(0, import_async3.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
|
|
641
728
|
} catch (err) {
|
|
642
729
|
(0, import_log4.log)("auth failed", err, {
|
|
643
730
|
F: __dxlog_file4,
|
|
644
|
-
L:
|
|
731
|
+
L: 74,
|
|
645
732
|
S: this,
|
|
646
733
|
C: (f, a) => f(...a)
|
|
647
734
|
});
|
|
@@ -657,26 +744,29 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
657
744
|
};
|
|
658
745
|
|
|
659
746
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
660
|
-
var import_tiny_invariant9 = __toESM(require("tiny-invariant"));
|
|
661
747
|
var import_async7 = require("@dxos/async");
|
|
748
|
+
var import_invariant9 = require("@dxos/invariant");
|
|
662
749
|
var import_log10 = require("@dxos/log");
|
|
663
750
|
var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
751
|
+
var import_tracing3 = require("@dxos/tracing");
|
|
664
752
|
var import_util7 = require("@dxos/util");
|
|
665
753
|
|
|
666
754
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
755
|
+
var import_context5 = require("@dxos/context");
|
|
667
756
|
var import_credentials = require("@dxos/credentials");
|
|
668
757
|
var import_keys4 = require("@dxos/keys");
|
|
669
758
|
var import_log8 = require("@dxos/log");
|
|
670
759
|
var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
671
760
|
var import_timeframe3 = require("@dxos/timeframe");
|
|
761
|
+
var import_tracing = require("@dxos/tracing");
|
|
672
762
|
var import_util5 = require("@dxos/util");
|
|
673
763
|
|
|
674
764
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
675
|
-
var import_tiny_invariant7 = __toESM(require("tiny-invariant"));
|
|
676
765
|
var import_async5 = require("@dxos/async");
|
|
677
766
|
var import_context4 = require("@dxos/context");
|
|
678
|
-
var
|
|
767
|
+
var import_debug3 = require("@dxos/debug");
|
|
679
768
|
var import_feed_store = require("@dxos/feed-store");
|
|
769
|
+
var import_invariant7 = require("@dxos/invariant");
|
|
680
770
|
var import_keys3 = require("@dxos/keys");
|
|
681
771
|
var import_log7 = require("@dxos/log");
|
|
682
772
|
var import_timeframe2 = require("@dxos/timeframe");
|
|
@@ -689,31 +779,56 @@ var codec = import_protocols5.schema.getCodecForType("dxos.echo.feed.FeedMessage
|
|
|
689
779
|
var valueEncoding = (0, import_hypercore.createCodecEncoding)(codec);
|
|
690
780
|
|
|
691
781
|
// packages/core/echo/echo-pipeline/src/common/feeds.ts
|
|
692
|
-
var
|
|
782
|
+
var import_invariant5 = require("@dxos/invariant");
|
|
783
|
+
var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/common/feeds.ts";
|
|
693
784
|
var createMappedFeedWriter = (mapper, writer) => {
|
|
694
|
-
(0,
|
|
695
|
-
|
|
785
|
+
(0, import_invariant5.invariant)(mapper, void 0, {
|
|
786
|
+
F: __dxlog_file5,
|
|
787
|
+
L: 16,
|
|
788
|
+
S: void 0,
|
|
789
|
+
A: [
|
|
790
|
+
"mapper",
|
|
791
|
+
""
|
|
792
|
+
]
|
|
793
|
+
});
|
|
794
|
+
(0, import_invariant5.invariant)(writer, void 0, {
|
|
795
|
+
F: __dxlog_file5,
|
|
796
|
+
L: 17,
|
|
797
|
+
S: void 0,
|
|
798
|
+
A: [
|
|
799
|
+
"writer",
|
|
800
|
+
""
|
|
801
|
+
]
|
|
802
|
+
});
|
|
696
803
|
return {
|
|
697
804
|
write: async (data, options) => await writer.write(await mapper(data), options)
|
|
698
805
|
};
|
|
699
806
|
};
|
|
700
807
|
|
|
701
808
|
// packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
|
|
702
|
-
var
|
|
809
|
+
var import_invariant6 = require("@dxos/invariant");
|
|
703
810
|
var import_log5 = require("@dxos/log");
|
|
704
|
-
var
|
|
811
|
+
var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
|
|
705
812
|
var createMessageSelector = (timeframeClock) => {
|
|
706
813
|
return (messages) => {
|
|
707
814
|
for (let i = 0; i < messages.length; i++) {
|
|
708
815
|
const { data: { timeframe } } = messages[i];
|
|
709
|
-
(0,
|
|
816
|
+
(0, import_invariant6.invariant)(timeframe, void 0, {
|
|
817
|
+
F: __dxlog_file6,
|
|
818
|
+
L: 25,
|
|
819
|
+
S: void 0,
|
|
820
|
+
A: [
|
|
821
|
+
"timeframe",
|
|
822
|
+
""
|
|
823
|
+
]
|
|
824
|
+
});
|
|
710
825
|
if (!timeframeClock.hasGaps(timeframe)) {
|
|
711
826
|
return i;
|
|
712
827
|
}
|
|
713
828
|
}
|
|
714
829
|
(0, import_log5.log)("Skipping...", void 0, {
|
|
715
|
-
F:
|
|
716
|
-
L:
|
|
830
|
+
F: __dxlog_file6,
|
|
831
|
+
L: 33,
|
|
717
832
|
S: void 0,
|
|
718
833
|
C: (f, a) => f(...a)
|
|
719
834
|
});
|
|
@@ -722,7 +837,7 @@ var createMessageSelector = (timeframeClock) => {
|
|
|
722
837
|
|
|
723
838
|
// packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
|
|
724
839
|
var import_async4 = require("@dxos/async");
|
|
725
|
-
var
|
|
840
|
+
var import_debug2 = require("@dxos/debug");
|
|
726
841
|
var import_log6 = require("@dxos/log");
|
|
727
842
|
var import_timeframe = require("@dxos/timeframe");
|
|
728
843
|
function _ts_decorate3(decorators, target, key, desc) {
|
|
@@ -735,7 +850,7 @@ function _ts_decorate3(decorators, target, key, desc) {
|
|
|
735
850
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
736
851
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
737
852
|
}
|
|
738
|
-
var
|
|
853
|
+
var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
|
|
739
854
|
var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
|
|
740
855
|
feedKey,
|
|
741
856
|
index
|
|
@@ -790,7 +905,7 @@ var TimeframeClock = class {
|
|
|
790
905
|
target,
|
|
791
906
|
current: this._timeframe
|
|
792
907
|
}, {
|
|
793
|
-
F:
|
|
908
|
+
F: __dxlog_file7,
|
|
794
909
|
L: 73,
|
|
795
910
|
S: this,
|
|
796
911
|
C: (f, a) => f(...a)
|
|
@@ -801,7 +916,7 @@ var TimeframeClock = class {
|
|
|
801
916
|
current: this._timeframe,
|
|
802
917
|
deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
|
|
803
918
|
}, {
|
|
804
|
-
F:
|
|
919
|
+
F: __dxlog_file7,
|
|
805
920
|
L: 75,
|
|
806
921
|
S: this,
|
|
807
922
|
C: (f, a) => f(...a)
|
|
@@ -811,7 +926,7 @@ var TimeframeClock = class {
|
|
|
811
926
|
}
|
|
812
927
|
};
|
|
813
928
|
_ts_decorate3([
|
|
814
|
-
(0,
|
|
929
|
+
(0, import_debug2.timed)(5e3)
|
|
815
930
|
], TimeframeClock.prototype, "waitUntilReached", null);
|
|
816
931
|
|
|
817
932
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
@@ -825,7 +940,7 @@ function _ts_decorate4(decorators, target, key, desc) {
|
|
|
825
940
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
826
941
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
827
942
|
}
|
|
828
|
-
var
|
|
943
|
+
var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
|
|
829
944
|
var PipelineState = class {
|
|
830
945
|
// prettier-ignore
|
|
831
946
|
constructor(_feeds, _timeframeClock) {
|
|
@@ -876,18 +991,17 @@ var PipelineState = class {
|
|
|
876
991
|
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
877
992
|
*/
|
|
878
993
|
async waitUntilReachedTargetTimeframe({ ctx = new import_context4.Context(), timeout, breakOnStall = true } = {}) {
|
|
879
|
-
var _a;
|
|
880
994
|
(0, import_log7.log)("waitUntilReachedTargetTimeframe", {
|
|
881
995
|
timeout,
|
|
882
996
|
current: this.timeframe,
|
|
883
997
|
target: this.targetTimeframe
|
|
884
998
|
}, {
|
|
885
|
-
F:
|
|
886
|
-
L:
|
|
999
|
+
F: __dxlog_file8,
|
|
1000
|
+
L: 125,
|
|
887
1001
|
S: this,
|
|
888
1002
|
C: (f, a) => f(...a)
|
|
889
1003
|
});
|
|
890
|
-
|
|
1004
|
+
this._reachedTargetPromise ??= Promise.race([
|
|
891
1005
|
this._timeframeClock.update.waitForCondition(() => {
|
|
892
1006
|
return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
|
|
893
1007
|
}),
|
|
@@ -913,8 +1027,8 @@ var PipelineState = class {
|
|
|
913
1027
|
target: this.targetTimeframe,
|
|
914
1028
|
dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
|
|
915
1029
|
}, {
|
|
916
|
-
F:
|
|
917
|
-
L:
|
|
1030
|
+
F: __dxlog_file8,
|
|
1031
|
+
L: 152,
|
|
918
1032
|
S: this,
|
|
919
1033
|
C: (f, a) => f(...a)
|
|
920
1034
|
});
|
|
@@ -934,16 +1048,26 @@ var Pipeline = class {
|
|
|
934
1048
|
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
935
1049
|
this._processingTrigger = new import_async5.Trigger().wake();
|
|
936
1050
|
this._pauseTrigger = new import_async5.Trigger().wake();
|
|
1051
|
+
// Pending downloads.
|
|
1052
|
+
this._downloads = new import_util4.ComplexMap((value) => import_keys3.PublicKey.hash(value.key));
|
|
937
1053
|
this._isStopping = false;
|
|
938
1054
|
this._isStarted = false;
|
|
939
|
-
this.
|
|
1055
|
+
this._isBeingConsumed = false;
|
|
940
1056
|
this._isPaused = false;
|
|
941
1057
|
}
|
|
942
1058
|
get state() {
|
|
943
1059
|
return this._state;
|
|
944
1060
|
}
|
|
945
1061
|
get writer() {
|
|
946
|
-
(0,
|
|
1062
|
+
(0, import_invariant7.invariant)(this._writer, "Writer not set.", {
|
|
1063
|
+
F: __dxlog_file8,
|
|
1064
|
+
L: 233,
|
|
1065
|
+
S: this,
|
|
1066
|
+
A: [
|
|
1067
|
+
"this._writer",
|
|
1068
|
+
"'Writer not set.'"
|
|
1069
|
+
]
|
|
1070
|
+
});
|
|
947
1071
|
return this._writer;
|
|
948
1072
|
}
|
|
949
1073
|
hasFeed(feedKey) {
|
|
@@ -959,20 +1083,47 @@ var Pipeline = class {
|
|
|
959
1083
|
if (this._feedSetIterator) {
|
|
960
1084
|
await this._feedSetIterator.addFeed(feed);
|
|
961
1085
|
}
|
|
962
|
-
this.
|
|
1086
|
+
if (this._isStarted && !this._isPaused) {
|
|
1087
|
+
this._setFeedDownloadState(feed);
|
|
1088
|
+
}
|
|
963
1089
|
}
|
|
964
1090
|
setWriteFeed(feed) {
|
|
965
|
-
(0,
|
|
966
|
-
|
|
1091
|
+
(0, import_invariant7.invariant)(!this._writer, "Writer already set.", {
|
|
1092
|
+
F: __dxlog_file8,
|
|
1093
|
+
L: 260,
|
|
1094
|
+
S: this,
|
|
1095
|
+
A: [
|
|
1096
|
+
"!this._writer",
|
|
1097
|
+
"'Writer already set.'"
|
|
1098
|
+
]
|
|
1099
|
+
});
|
|
1100
|
+
(0, import_invariant7.invariant)(feed.properties.writable, "Feed must be writable.", {
|
|
1101
|
+
F: __dxlog_file8,
|
|
1102
|
+
L: 261,
|
|
1103
|
+
S: this,
|
|
1104
|
+
A: [
|
|
1105
|
+
"feed.properties.writable",
|
|
1106
|
+
"'Feed must be writable.'"
|
|
1107
|
+
]
|
|
1108
|
+
});
|
|
967
1109
|
this._writer = createMappedFeedWriter((payload) => ({
|
|
968
1110
|
timeframe: this._timeframeClock.timeframe,
|
|
969
1111
|
payload
|
|
970
1112
|
}), feed.createFeedWriter());
|
|
971
1113
|
}
|
|
972
1114
|
async start() {
|
|
1115
|
+
(0, import_invariant7.invariant)(!this._isStarted, "Pipeline is already started.", {
|
|
1116
|
+
F: __dxlog_file8,
|
|
1117
|
+
L: 274,
|
|
1118
|
+
S: this,
|
|
1119
|
+
A: [
|
|
1120
|
+
"!this._isStarted",
|
|
1121
|
+
"'Pipeline is already started.'"
|
|
1122
|
+
]
|
|
1123
|
+
});
|
|
973
1124
|
(0, import_log7.log)("starting...", void 0, {
|
|
974
|
-
F:
|
|
975
|
-
L:
|
|
1125
|
+
F: __dxlog_file8,
|
|
1126
|
+
L: 275,
|
|
976
1127
|
S: this,
|
|
977
1128
|
C: (f, a) => f(...a)
|
|
978
1129
|
});
|
|
@@ -980,30 +1131,38 @@ var Pipeline = class {
|
|
|
980
1131
|
await this._feedSetIterator.open();
|
|
981
1132
|
this._isStarted = true;
|
|
982
1133
|
(0, import_log7.log)("started", void 0, {
|
|
983
|
-
F:
|
|
984
|
-
L:
|
|
1134
|
+
F: __dxlog_file8,
|
|
1135
|
+
L: 279,
|
|
985
1136
|
S: this,
|
|
986
1137
|
C: (f, a) => f(...a)
|
|
987
1138
|
});
|
|
1139
|
+
if (!this._isPaused) {
|
|
1140
|
+
for (const feed of this._feeds.values()) {
|
|
1141
|
+
this._setFeedDownloadState(feed);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
988
1144
|
}
|
|
989
1145
|
async stop() {
|
|
990
|
-
var _a;
|
|
991
1146
|
(0, import_log7.log)("stopping...", void 0, {
|
|
992
|
-
F:
|
|
993
|
-
L:
|
|
1147
|
+
F: __dxlog_file8,
|
|
1148
|
+
L: 290,
|
|
994
1149
|
S: this,
|
|
995
1150
|
C: (f, a) => f(...a)
|
|
996
1151
|
});
|
|
997
1152
|
this._isStopping = true;
|
|
998
|
-
|
|
1153
|
+
for (const [feed, handle] of this._downloads.entries()) {
|
|
1154
|
+
feed.undownload(handle);
|
|
1155
|
+
}
|
|
1156
|
+
this._downloads.clear();
|
|
1157
|
+
await this._feedSetIterator?.close();
|
|
999
1158
|
await this._processingTrigger.wait();
|
|
1000
1159
|
await this._state._ctx.dispose();
|
|
1001
1160
|
this._state._ctx = new import_context4.Context();
|
|
1002
1161
|
this._state._reachedTargetPromise = void 0;
|
|
1003
1162
|
this._isStarted = false;
|
|
1004
1163
|
(0, import_log7.log)("stopped", void 0, {
|
|
1005
|
-
F:
|
|
1006
|
-
L:
|
|
1164
|
+
F: __dxlog_file8,
|
|
1165
|
+
L: 302,
|
|
1007
1166
|
S: this,
|
|
1008
1167
|
C: (f, a) => f(...a)
|
|
1009
1168
|
});
|
|
@@ -1013,12 +1172,17 @@ var Pipeline = class {
|
|
|
1013
1172
|
* The pipeline will start processing messages AFTER this timeframe.
|
|
1014
1173
|
*/
|
|
1015
1174
|
async setCursor(timeframe) {
|
|
1016
|
-
(0,
|
|
1175
|
+
(0, import_invariant7.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
|
|
1176
|
+
F: __dxlog_file8,
|
|
1177
|
+
L: 311,
|
|
1178
|
+
S: this,
|
|
1179
|
+
A: [
|
|
1180
|
+
"!this._isStarted || this._isPaused",
|
|
1181
|
+
"'Invalid state.'"
|
|
1182
|
+
]
|
|
1183
|
+
});
|
|
1017
1184
|
this._state._startTimeframe = timeframe;
|
|
1018
1185
|
this._timeframeClock.setTimeframe(timeframe);
|
|
1019
|
-
for (const feed of this._feeds.values()) {
|
|
1020
|
-
this._setFeedDownloadState(feed);
|
|
1021
|
-
}
|
|
1022
1186
|
if (this._feedSetIterator) {
|
|
1023
1187
|
await this._feedSetIterator.close();
|
|
1024
1188
|
await this._initIterator();
|
|
@@ -1029,7 +1193,6 @@ var Pipeline = class {
|
|
|
1029
1193
|
* Calling pause while processing will cause a deadlock.
|
|
1030
1194
|
*/
|
|
1031
1195
|
async pause() {
|
|
1032
|
-
(0, import_tiny_invariant7.default)(this._isStarted, "Pipeline is not open.");
|
|
1033
1196
|
if (this._isPaused) {
|
|
1034
1197
|
return;
|
|
1035
1198
|
}
|
|
@@ -1038,31 +1201,65 @@ var Pipeline = class {
|
|
|
1038
1201
|
this._isPaused = true;
|
|
1039
1202
|
}
|
|
1040
1203
|
async unpause() {
|
|
1041
|
-
(0,
|
|
1042
|
-
|
|
1204
|
+
(0, import_invariant7.invariant)(this._isPaused, "Pipeline is not paused.", {
|
|
1205
|
+
F: __dxlog_file8,
|
|
1206
|
+
L: 340,
|
|
1207
|
+
S: this,
|
|
1208
|
+
A: [
|
|
1209
|
+
"this._isPaused",
|
|
1210
|
+
"'Pipeline is not paused.'"
|
|
1211
|
+
]
|
|
1212
|
+
});
|
|
1043
1213
|
this._pauseTrigger.wake();
|
|
1044
1214
|
this._isPaused = false;
|
|
1215
|
+
for (const feed of this._feeds.values()) {
|
|
1216
|
+
this._setFeedDownloadState(feed);
|
|
1217
|
+
}
|
|
1045
1218
|
}
|
|
1046
1219
|
/**
|
|
1047
1220
|
* Starts to iterate over the ordered messages from the added feeds.
|
|
1048
1221
|
* Updates the timeframe clock after the message has bee processed.
|
|
1049
1222
|
*/
|
|
1050
1223
|
async *consume() {
|
|
1051
|
-
(0,
|
|
1052
|
-
|
|
1053
|
-
|
|
1224
|
+
(0, import_invariant7.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
|
|
1225
|
+
F: __dxlog_file8,
|
|
1226
|
+
L: 355,
|
|
1227
|
+
S: this,
|
|
1228
|
+
A: [
|
|
1229
|
+
"!this._isBeingConsumed",
|
|
1230
|
+
"'Pipeline is already being consumed.'"
|
|
1231
|
+
]
|
|
1232
|
+
});
|
|
1233
|
+
this._isBeingConsumed = true;
|
|
1234
|
+
(0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1235
|
+
F: __dxlog_file8,
|
|
1236
|
+
L: 358,
|
|
1237
|
+
S: this,
|
|
1238
|
+
A: [
|
|
1239
|
+
"this._feedSetIterator",
|
|
1240
|
+
"'Iterator not initialized.'"
|
|
1241
|
+
]
|
|
1242
|
+
});
|
|
1054
1243
|
let lastFeedSetIterator = this._feedSetIterator;
|
|
1055
1244
|
let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
1056
1245
|
while (!this._isStopping) {
|
|
1057
1246
|
await this._pauseTrigger.wait();
|
|
1058
1247
|
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
1059
|
-
(0,
|
|
1248
|
+
(0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1249
|
+
F: __dxlog_file8,
|
|
1250
|
+
L: 367,
|
|
1251
|
+
S: this,
|
|
1252
|
+
A: [
|
|
1253
|
+
"this._feedSetIterator",
|
|
1254
|
+
"'Iterator not initialized.'"
|
|
1255
|
+
]
|
|
1256
|
+
});
|
|
1060
1257
|
lastFeedSetIterator = this._feedSetIterator;
|
|
1061
1258
|
iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
1062
1259
|
}
|
|
1063
1260
|
const { done, value } = await iterable.next();
|
|
1064
1261
|
if (!done) {
|
|
1065
|
-
const block = value
|
|
1262
|
+
const block = value ?? (0, import_debug3.failUndefined)();
|
|
1066
1263
|
this._processingTrigger.reset();
|
|
1067
1264
|
this._timeframeClock.updatePendingTimeframe(import_keys3.PublicKey.from(block.feedKey), block.seq);
|
|
1068
1265
|
yield block;
|
|
@@ -1070,33 +1267,42 @@ var Pipeline = class {
|
|
|
1070
1267
|
this._timeframeClock.updateTimeframe();
|
|
1071
1268
|
}
|
|
1072
1269
|
}
|
|
1073
|
-
this.
|
|
1270
|
+
this._isBeingConsumed = false;
|
|
1074
1271
|
}
|
|
1075
1272
|
_setFeedDownloadState(feed) {
|
|
1076
|
-
|
|
1273
|
+
let handle = this._downloads.get(feed);
|
|
1274
|
+
if (handle) {
|
|
1275
|
+
feed.undownload(handle);
|
|
1276
|
+
}
|
|
1077
1277
|
const timeframe = this._state._startTimeframe;
|
|
1078
|
-
const seq =
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1278
|
+
const seq = timeframe.get(feed.key) ?? -1;
|
|
1279
|
+
(0, import_log7.log)("download", {
|
|
1280
|
+
feed: feed.key.truncate(),
|
|
1281
|
+
seq,
|
|
1282
|
+
length: feed.length
|
|
1283
|
+
}, {
|
|
1284
|
+
F: __dxlog_file8,
|
|
1285
|
+
L: 396,
|
|
1286
|
+
S: this,
|
|
1287
|
+
C: (f, a) => f(...a)
|
|
1086
1288
|
});
|
|
1087
|
-
feed.download({
|
|
1289
|
+
handle = feed.download({
|
|
1088
1290
|
start: seq + 1,
|
|
1089
1291
|
linear: true
|
|
1090
|
-
}
|
|
1091
|
-
(
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1292
|
+
}, (err, data) => {
|
|
1293
|
+
if (err) {
|
|
1294
|
+
} else {
|
|
1295
|
+
import_log7.log.info("downloaded", {
|
|
1296
|
+
data
|
|
1297
|
+
}, {
|
|
1298
|
+
F: __dxlog_file8,
|
|
1299
|
+
L: 401,
|
|
1300
|
+
S: this,
|
|
1301
|
+
C: (f, a) => f(...a)
|
|
1302
|
+
});
|
|
1303
|
+
}
|
|
1099
1304
|
});
|
|
1305
|
+
this._downloads.set(feed, handle);
|
|
1100
1306
|
}
|
|
1101
1307
|
async _initIterator() {
|
|
1102
1308
|
this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
|
|
@@ -1105,8 +1311,8 @@ var Pipeline = class {
|
|
|
1105
1311
|
});
|
|
1106
1312
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
1107
1313
|
import_log7.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
1108
|
-
F:
|
|
1109
|
-
L:
|
|
1314
|
+
F: __dxlog_file8,
|
|
1315
|
+
L: 415,
|
|
1110
1316
|
S: this,
|
|
1111
1317
|
C: (f, a) => f(...a)
|
|
1112
1318
|
});
|
|
@@ -1134,12 +1340,24 @@ _ts_decorate4([
|
|
|
1134
1340
|
], Pipeline.prototype, "unpause", null);
|
|
1135
1341
|
|
|
1136
1342
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
1137
|
-
|
|
1343
|
+
function _ts_decorate5(decorators, target, key, desc) {
|
|
1344
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1345
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1346
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
1347
|
+
else
|
|
1348
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
1349
|
+
if (d = decorators[i])
|
|
1350
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1351
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1352
|
+
}
|
|
1353
|
+
var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
|
|
1138
1354
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
1139
|
-
var ControlPipeline = class {
|
|
1355
|
+
var ControlPipeline = class ControlPipeline2 {
|
|
1140
1356
|
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
1141
1357
|
this._lastTimeframeSaveTime = Date.now();
|
|
1142
1358
|
this.onFeedAdmitted = new import_util5.Callback();
|
|
1359
|
+
this._usage = new import_tracing.TimeUsageCounter();
|
|
1360
|
+
this._mutations = new import_tracing.TimeSeriesCounter();
|
|
1143
1361
|
this._spaceKey = spaceKey;
|
|
1144
1362
|
this._metadata = metadataStore;
|
|
1145
1363
|
this._pipeline = new Pipeline();
|
|
@@ -1149,8 +1367,8 @@ var ControlPipeline = class {
|
|
|
1149
1367
|
(0, import_log8.log)("feed admitted", {
|
|
1150
1368
|
key: info.key
|
|
1151
1369
|
}, {
|
|
1152
|
-
F:
|
|
1153
|
-
L:
|
|
1370
|
+
F: __dxlog_file9,
|
|
1371
|
+
L: 61,
|
|
1154
1372
|
S: this,
|
|
1155
1373
|
C: (f, a) => f(...a)
|
|
1156
1374
|
});
|
|
@@ -1160,8 +1378,8 @@ var ControlPipeline = class {
|
|
|
1160
1378
|
await this._pipeline.addFeed(feed);
|
|
1161
1379
|
} catch (err) {
|
|
1162
1380
|
import_log8.log.catch(err, void 0, {
|
|
1163
|
-
F:
|
|
1164
|
-
L:
|
|
1381
|
+
F: __dxlog_file9,
|
|
1382
|
+
L: 69,
|
|
1165
1383
|
S: this,
|
|
1166
1384
|
C: (f, a) => f(...a)
|
|
1167
1385
|
});
|
|
@@ -1183,58 +1401,67 @@ var ControlPipeline = class {
|
|
|
1183
1401
|
}
|
|
1184
1402
|
async start() {
|
|
1185
1403
|
(0, import_log8.log)("starting...", void 0, {
|
|
1186
|
-
F:
|
|
1187
|
-
L:
|
|
1404
|
+
F: __dxlog_file9,
|
|
1405
|
+
L: 93,
|
|
1188
1406
|
S: this,
|
|
1189
1407
|
C: (f, a) => f(...a)
|
|
1190
1408
|
});
|
|
1191
1409
|
setTimeout(async () => {
|
|
1192
|
-
|
|
1193
|
-
try {
|
|
1194
|
-
(0, import_log8.log)("processing", {
|
|
1195
|
-
key: msg.feedKey,
|
|
1196
|
-
seq: msg.seq
|
|
1197
|
-
}, {
|
|
1198
|
-
F: __dxlog_file8,
|
|
1199
|
-
L: 88,
|
|
1200
|
-
S: this,
|
|
1201
|
-
C: (f, a) => f(...a)
|
|
1202
|
-
});
|
|
1203
|
-
if (msg.data.payload.credential) {
|
|
1204
|
-
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1205
|
-
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys4.PublicKey.from(msg.feedKey));
|
|
1206
|
-
timer.end();
|
|
1207
|
-
if (!result) {
|
|
1208
|
-
import_log8.log.warn("processing failed", {
|
|
1209
|
-
msg
|
|
1210
|
-
}, {
|
|
1211
|
-
F: __dxlog_file8,
|
|
1212
|
-
L: 98,
|
|
1213
|
-
S: this,
|
|
1214
|
-
C: (f, a) => f(...a)
|
|
1215
|
-
});
|
|
1216
|
-
} else {
|
|
1217
|
-
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1218
|
-
}
|
|
1219
|
-
}
|
|
1220
|
-
} catch (err) {
|
|
1221
|
-
import_log8.log.catch(err, void 0, {
|
|
1222
|
-
F: __dxlog_file8,
|
|
1223
|
-
L: 104,
|
|
1224
|
-
S: this,
|
|
1225
|
-
C: (f, a) => f(...a)
|
|
1226
|
-
});
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1410
|
+
void this._consumePipeline(new import_context5.Context());
|
|
1229
1411
|
});
|
|
1230
1412
|
await this._pipeline.start();
|
|
1231
1413
|
(0, import_log8.log)("started", void 0, {
|
|
1232
|
-
F:
|
|
1233
|
-
L:
|
|
1414
|
+
F: __dxlog_file9,
|
|
1415
|
+
L: 99,
|
|
1234
1416
|
S: this,
|
|
1235
1417
|
C: (f, a) => f(...a)
|
|
1236
1418
|
});
|
|
1237
1419
|
}
|
|
1420
|
+
async _consumePipeline(ctx) {
|
|
1421
|
+
for await (const msg of this._pipeline.consume()) {
|
|
1422
|
+
const span = this._usage.beginRecording();
|
|
1423
|
+
this._mutations.inc();
|
|
1424
|
+
try {
|
|
1425
|
+
await this._processMessage(ctx, msg);
|
|
1426
|
+
} catch (err) {
|
|
1427
|
+
import_log8.log.catch(err, void 0, {
|
|
1428
|
+
F: __dxlog_file9,
|
|
1429
|
+
L: 111,
|
|
1430
|
+
S: this,
|
|
1431
|
+
C: (f, a) => f(...a)
|
|
1432
|
+
});
|
|
1433
|
+
}
|
|
1434
|
+
span.end();
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
async _processMessage(ctx, msg) {
|
|
1438
|
+
(0, import_log8.log)("processing", {
|
|
1439
|
+
key: msg.feedKey,
|
|
1440
|
+
seq: msg.seq
|
|
1441
|
+
}, {
|
|
1442
|
+
F: __dxlog_file9,
|
|
1443
|
+
L: 121,
|
|
1444
|
+
S: this,
|
|
1445
|
+
C: (f, a) => f(...a)
|
|
1446
|
+
});
|
|
1447
|
+
if (msg.data.payload.credential) {
|
|
1448
|
+
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1449
|
+
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys4.PublicKey.from(msg.feedKey));
|
|
1450
|
+
timer.end();
|
|
1451
|
+
if (!result) {
|
|
1452
|
+
import_log8.log.warn("processing failed", {
|
|
1453
|
+
msg
|
|
1454
|
+
}, {
|
|
1455
|
+
F: __dxlog_file9,
|
|
1456
|
+
L: 131,
|
|
1457
|
+
S: this,
|
|
1458
|
+
C: (f, a) => f(...a)
|
|
1459
|
+
});
|
|
1460
|
+
} else {
|
|
1461
|
+
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1238
1465
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
1239
1466
|
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
|
|
1240
1467
|
this._lastTimeframeSaveTime = Date.now();
|
|
@@ -1243,48 +1470,63 @@ var ControlPipeline = class {
|
|
|
1243
1470
|
}
|
|
1244
1471
|
async stop() {
|
|
1245
1472
|
(0, import_log8.log)("stopping...", void 0, {
|
|
1246
|
-
F:
|
|
1247
|
-
L:
|
|
1473
|
+
F: __dxlog_file9,
|
|
1474
|
+
L: 149,
|
|
1248
1475
|
S: this,
|
|
1249
1476
|
C: (f, a) => f(...a)
|
|
1250
1477
|
});
|
|
1251
1478
|
await this._pipeline.stop();
|
|
1252
1479
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1253
1480
|
(0, import_log8.log)("stopped", void 0, {
|
|
1254
|
-
F:
|
|
1255
|
-
L:
|
|
1481
|
+
F: __dxlog_file9,
|
|
1482
|
+
L: 152,
|
|
1256
1483
|
S: this,
|
|
1257
1484
|
C: (f, a) => f(...a)
|
|
1258
1485
|
});
|
|
1259
1486
|
}
|
|
1260
1487
|
async _saveTargetTimeframe(timeframe) {
|
|
1261
|
-
var _a;
|
|
1262
1488
|
try {
|
|
1263
|
-
const newTimeframe = import_timeframe3.Timeframe.merge(
|
|
1489
|
+
const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
|
|
1264
1490
|
await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
|
|
1265
1491
|
this._targetTimeframe = newTimeframe;
|
|
1266
1492
|
} catch (err) {
|
|
1267
1493
|
(0, import_log8.log)(err, void 0, {
|
|
1268
|
-
F:
|
|
1269
|
-
L:
|
|
1494
|
+
F: __dxlog_file9,
|
|
1495
|
+
L: 161,
|
|
1270
1496
|
S: this,
|
|
1271
1497
|
C: (f, a) => f(...a)
|
|
1272
1498
|
});
|
|
1273
1499
|
}
|
|
1274
1500
|
}
|
|
1275
1501
|
};
|
|
1502
|
+
_ts_decorate5([
|
|
1503
|
+
import_tracing.trace.metricsCounter()
|
|
1504
|
+
], ControlPipeline.prototype, "_usage", void 0);
|
|
1505
|
+
_ts_decorate5([
|
|
1506
|
+
import_tracing.trace.metricsCounter()
|
|
1507
|
+
], ControlPipeline.prototype, "_mutations", void 0);
|
|
1508
|
+
_ts_decorate5([
|
|
1509
|
+
import_tracing.trace.span()
|
|
1510
|
+
], ControlPipeline.prototype, "_consumePipeline", null);
|
|
1511
|
+
_ts_decorate5([
|
|
1512
|
+
import_tracing.trace.span()
|
|
1513
|
+
], ControlPipeline.prototype, "_processMessage", null);
|
|
1514
|
+
ControlPipeline = _ts_decorate5([
|
|
1515
|
+
import_tracing.trace.resource()
|
|
1516
|
+
], ControlPipeline);
|
|
1276
1517
|
|
|
1277
1518
|
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1278
|
-
var import_tiny_invariant8 = __toESM(require("tiny-invariant"));
|
|
1279
1519
|
var import_async6 = require("@dxos/async");
|
|
1280
|
-
var
|
|
1520
|
+
var import_context6 = require("@dxos/context");
|
|
1281
1521
|
var import_credentials3 = require("@dxos/credentials");
|
|
1282
1522
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
1283
1523
|
var import_errors2 = require("@dxos/errors");
|
|
1524
|
+
var import_invariant8 = require("@dxos/invariant");
|
|
1284
1525
|
var import_log9 = require("@dxos/log");
|
|
1285
1526
|
var import_timeframe4 = require("@dxos/timeframe");
|
|
1527
|
+
var import_tracing2 = require("@dxos/tracing");
|
|
1286
1528
|
var import_util6 = require("@dxos/util");
|
|
1287
|
-
function
|
|
1529
|
+
function _ts_decorate6(decorators, target, key, desc) {
|
|
1288
1530
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1289
1531
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1290
1532
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1294,14 +1536,14 @@ function _ts_decorate5(decorators, target, key, desc) {
|
|
|
1294
1536
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1295
1537
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1296
1538
|
}
|
|
1297
|
-
var
|
|
1539
|
+
var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
|
|
1298
1540
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1299
1541
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1300
1542
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
1301
1543
|
var DataPipeline = class DataPipeline2 {
|
|
1302
1544
|
constructor(_params) {
|
|
1303
1545
|
this._params = _params;
|
|
1304
|
-
this._ctx = new
|
|
1546
|
+
this._ctx = new import_context6.Context();
|
|
1305
1547
|
this._pipeline = void 0;
|
|
1306
1548
|
this._targetTimeframe = void 0;
|
|
1307
1549
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
@@ -1309,6 +1551,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1309
1551
|
this._lastTimeframeSaveTime = 0;
|
|
1310
1552
|
this._lastSnapshotSaveTime = 0;
|
|
1311
1553
|
this._lastProcessedEpoch = -1;
|
|
1554
|
+
this._usage = new import_tracing2.TimeUsageCounter();
|
|
1555
|
+
this._mutations = new import_tracing2.TimeSeriesCounter();
|
|
1312
1556
|
this.currentEpoch = void 0;
|
|
1313
1557
|
this.appliedEpoch = void 0;
|
|
1314
1558
|
this.onNewEpoch = new import_async6.Event();
|
|
@@ -1320,13 +1564,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1320
1564
|
return this._pipeline;
|
|
1321
1565
|
}
|
|
1322
1566
|
get pipelineState() {
|
|
1323
|
-
|
|
1324
|
-
return (_a = this._pipeline) == null ? void 0 : _a.state;
|
|
1567
|
+
return this._pipeline?.state;
|
|
1325
1568
|
}
|
|
1326
1569
|
setTargetTimeframe(timeframe) {
|
|
1327
|
-
var _a;
|
|
1328
1570
|
this._targetTimeframe = timeframe;
|
|
1329
|
-
|
|
1571
|
+
this._pipeline?.state.setTargetTimeframe(timeframe);
|
|
1330
1572
|
}
|
|
1331
1573
|
async processCredential(credential) {
|
|
1332
1574
|
if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
|
|
@@ -1343,15 +1585,31 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1343
1585
|
}
|
|
1344
1586
|
this._pipeline = new Pipeline();
|
|
1345
1587
|
await this._params.onPipelineCreated(this._pipeline);
|
|
1346
|
-
await this._pipeline.start();
|
|
1347
1588
|
await this._pipeline.pause();
|
|
1589
|
+
await this._pipeline.start();
|
|
1348
1590
|
if (this._targetTimeframe) {
|
|
1349
1591
|
this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
|
|
1350
1592
|
}
|
|
1351
1593
|
const feedWriter = {
|
|
1352
1594
|
write: (data, options) => {
|
|
1353
|
-
(0,
|
|
1354
|
-
|
|
1595
|
+
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1596
|
+
F: __dxlog_file10,
|
|
1597
|
+
L: 154,
|
|
1598
|
+
S: this,
|
|
1599
|
+
A: [
|
|
1600
|
+
"this._pipeline",
|
|
1601
|
+
"'Pipeline is not initialized.'"
|
|
1602
|
+
]
|
|
1603
|
+
});
|
|
1604
|
+
(0, import_invariant8.invariant)(this.currentEpoch, "Epoch is not initialized.", {
|
|
1605
|
+
F: __dxlog_file10,
|
|
1606
|
+
L: 155,
|
|
1607
|
+
S: this,
|
|
1608
|
+
A: [
|
|
1609
|
+
"this.currentEpoch",
|
|
1610
|
+
"'Epoch is not initialized.'"
|
|
1611
|
+
]
|
|
1612
|
+
});
|
|
1355
1613
|
return this._pipeline.writer.write({
|
|
1356
1614
|
data
|
|
1357
1615
|
}, options);
|
|
@@ -1366,19 +1624,18 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1366
1624
|
this._isOpen = true;
|
|
1367
1625
|
}
|
|
1368
1626
|
async close() {
|
|
1369
|
-
var _a, _b, _c;
|
|
1370
1627
|
if (!this._isOpen) {
|
|
1371
1628
|
return;
|
|
1372
1629
|
}
|
|
1373
1630
|
(0, import_log9.log)("close", void 0, {
|
|
1374
|
-
F:
|
|
1375
|
-
L:
|
|
1631
|
+
F: __dxlog_file10,
|
|
1632
|
+
L: 179,
|
|
1376
1633
|
S: this,
|
|
1377
1634
|
C: (f, a) => f(...a)
|
|
1378
1635
|
});
|
|
1379
1636
|
this._isOpen = false;
|
|
1380
1637
|
await this._ctx.dispose();
|
|
1381
|
-
await
|
|
1638
|
+
await this._pipeline?.stop();
|
|
1382
1639
|
try {
|
|
1383
1640
|
await this._saveCache();
|
|
1384
1641
|
if (this._pipeline) {
|
|
@@ -1386,15 +1643,15 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1386
1643
|
}
|
|
1387
1644
|
} catch (err) {
|
|
1388
1645
|
import_log9.log.catch(err, void 0, {
|
|
1389
|
-
F:
|
|
1390
|
-
L:
|
|
1646
|
+
F: __dxlog_file10,
|
|
1647
|
+
L: 192,
|
|
1391
1648
|
S: this,
|
|
1392
1649
|
C: (f, a) => f(...a)
|
|
1393
1650
|
});
|
|
1394
1651
|
}
|
|
1395
|
-
await
|
|
1396
|
-
await
|
|
1397
|
-
this._ctx = new
|
|
1652
|
+
await this.databaseHost?.close();
|
|
1653
|
+
await this.itemManager?.destroy();
|
|
1654
|
+
this._ctx = new import_context6.Context();
|
|
1398
1655
|
this._pipeline = void 0;
|
|
1399
1656
|
this._targetTimeframe = void 0;
|
|
1400
1657
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
@@ -1409,15 +1666,25 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1409
1666
|
await this._processEpochInSeparateTask(this.currentEpoch);
|
|
1410
1667
|
await waitForOneEpoch;
|
|
1411
1668
|
}
|
|
1412
|
-
(0,
|
|
1669
|
+
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1670
|
+
F: __dxlog_file10,
|
|
1671
|
+
L: 215,
|
|
1672
|
+
S: this,
|
|
1673
|
+
A: [
|
|
1674
|
+
"this._pipeline",
|
|
1675
|
+
"'Pipeline is not initialized.'"
|
|
1676
|
+
]
|
|
1677
|
+
});
|
|
1413
1678
|
for await (const msg of this._pipeline.consume()) {
|
|
1679
|
+
const span = this._usage.beginRecording();
|
|
1680
|
+
this._mutations.inc();
|
|
1414
1681
|
const { feedKey, seq, data } = msg;
|
|
1415
1682
|
(0, import_log9.log)("processing message", {
|
|
1416
1683
|
feedKey,
|
|
1417
1684
|
seq
|
|
1418
1685
|
}, {
|
|
1419
|
-
F:
|
|
1420
|
-
L:
|
|
1686
|
+
F: __dxlog_file10,
|
|
1687
|
+
L: 221,
|
|
1421
1688
|
S: this,
|
|
1422
1689
|
C: (f, a) => f(...a)
|
|
1423
1690
|
});
|
|
@@ -1428,8 +1695,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1428
1695
|
import_log9.log.warn("Could not find feed", {
|
|
1429
1696
|
feedKey
|
|
1430
1697
|
}, {
|
|
1431
|
-
F:
|
|
1432
|
-
L:
|
|
1698
|
+
F: __dxlog_file10,
|
|
1699
|
+
L: 227,
|
|
1433
1700
|
S: this,
|
|
1434
1701
|
C: (f, a) => f(...a)
|
|
1435
1702
|
});
|
|
@@ -1451,8 +1718,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1451
1718
|
seq,
|
|
1452
1719
|
spaceKey: this._params.spaceKey.toHex()
|
|
1453
1720
|
}, {
|
|
1454
|
-
F:
|
|
1455
|
-
L:
|
|
1721
|
+
F: __dxlog_file10,
|
|
1722
|
+
L: 244,
|
|
1456
1723
|
S: this,
|
|
1457
1724
|
C: (f, a) => f(...a)
|
|
1458
1725
|
});
|
|
@@ -1460,16 +1727,25 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1460
1727
|
}
|
|
1461
1728
|
} catch (err) {
|
|
1462
1729
|
import_log9.log.catch(err, void 0, {
|
|
1463
|
-
F:
|
|
1464
|
-
L:
|
|
1730
|
+
F: __dxlog_file10,
|
|
1731
|
+
L: 254,
|
|
1465
1732
|
S: this,
|
|
1466
1733
|
C: (f, a) => f(...a)
|
|
1467
1734
|
});
|
|
1468
1735
|
}
|
|
1736
|
+
span.end();
|
|
1469
1737
|
}
|
|
1470
1738
|
}
|
|
1471
1739
|
_createSnapshot() {
|
|
1472
|
-
(0,
|
|
1740
|
+
(0, import_invariant8.invariant)(this.databaseHost, "Database backend is not initialized.", {
|
|
1741
|
+
F: __dxlog_file10,
|
|
1742
|
+
L: 262,
|
|
1743
|
+
S: this,
|
|
1744
|
+
A: [
|
|
1745
|
+
"this.databaseHost",
|
|
1746
|
+
"'Database backend is not initialized.'"
|
|
1747
|
+
]
|
|
1748
|
+
});
|
|
1473
1749
|
return {
|
|
1474
1750
|
spaceKey: this._params.spaceKey.asUint8Array(),
|
|
1475
1751
|
timeframe: this._pipeline.state.timeframe,
|
|
@@ -1477,26 +1753,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1477
1753
|
};
|
|
1478
1754
|
}
|
|
1479
1755
|
async _saveTargetTimeframe(timeframe) {
|
|
1480
|
-
|
|
1481
|
-
const newTimeframe = import_timeframe4.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe4.Timeframe(), timeframe);
|
|
1756
|
+
const newTimeframe = import_timeframe4.Timeframe.merge(this._targetTimeframe ?? new import_timeframe4.Timeframe(), timeframe);
|
|
1482
1757
|
await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
|
|
1483
1758
|
this._targetTimeframe = newTimeframe;
|
|
1484
1759
|
}
|
|
1485
1760
|
async _saveCache() {
|
|
1486
|
-
var _a;
|
|
1487
1761
|
const cache = {};
|
|
1488
1762
|
try {
|
|
1489
|
-
const propertiesItem = this.itemManager.items.find((item) =>
|
|
1490
|
-
|
|
1491
|
-
return ((_a2 = item.modelMeta) == null ? void 0 : _a2.type) === "dxos:model/document" && ((_b = (0, import_echo_db3.getStateMachineFromItem)(item)) == null ? void 0 : _b.snapshot()).type === "dxos.sdk.client.Properties";
|
|
1492
|
-
});
|
|
1763
|
+
const propertiesItem = this.itemManager.items.find((item) => item.modelMeta?.type === "dxos:model/document" && // TODO(burdon): Document?
|
|
1764
|
+
((0, import_echo_db3.getStateMachineFromItem)(item)?.snapshot()).type === import_echo_db3.TYPE_PROPERTIES);
|
|
1493
1765
|
if (propertiesItem) {
|
|
1494
|
-
cache.properties = (
|
|
1766
|
+
cache.properties = (0, import_echo_db3.getStateMachineFromItem)(propertiesItem)?.snapshot();
|
|
1495
1767
|
}
|
|
1496
1768
|
} catch (err) {
|
|
1497
1769
|
import_log9.log.warn("Failed to cache properties", err, {
|
|
1498
|
-
F:
|
|
1499
|
-
L:
|
|
1770
|
+
F: __dxlog_file10,
|
|
1771
|
+
L: 291,
|
|
1500
1772
|
S: this,
|
|
1501
1773
|
C: (f, a) => f(...a)
|
|
1502
1774
|
});
|
|
@@ -1513,24 +1785,23 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1513
1785
|
}
|
|
1514
1786
|
}
|
|
1515
1787
|
async _processEpochInSeparateTask(epoch) {
|
|
1516
|
-
var _a;
|
|
1517
1788
|
if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
|
|
1518
1789
|
return;
|
|
1519
1790
|
}
|
|
1520
|
-
await
|
|
1521
|
-
const ctx = new
|
|
1791
|
+
await this._epochCtx?.dispose();
|
|
1792
|
+
const ctx = new import_context6.Context({
|
|
1522
1793
|
onError: (err) => {
|
|
1523
1794
|
if (err instanceof import_errors2.CancelledError) {
|
|
1524
1795
|
(0, import_log9.log)("Epoch processing cancelled.", void 0, {
|
|
1525
|
-
F:
|
|
1526
|
-
L:
|
|
1796
|
+
F: __dxlog_file10,
|
|
1797
|
+
L: 323,
|
|
1527
1798
|
S: this,
|
|
1528
1799
|
C: (f, a) => f(...a)
|
|
1529
1800
|
});
|
|
1530
1801
|
} else {
|
|
1531
1802
|
import_log9.log.catch(err, void 0, {
|
|
1532
|
-
F:
|
|
1533
|
-
L:
|
|
1803
|
+
F: __dxlog_file10,
|
|
1804
|
+
L: 325,
|
|
1534
1805
|
S: this,
|
|
1535
1806
|
C: (f, a) => f(...a)
|
|
1536
1807
|
});
|
|
@@ -1542,28 +1813,36 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1542
1813
|
if (!this._isOpen) {
|
|
1543
1814
|
return;
|
|
1544
1815
|
}
|
|
1545
|
-
(0, import_log9.log)("process epoch", {
|
|
1546
|
-
epoch
|
|
1547
|
-
}, {
|
|
1548
|
-
F: __dxlog_file9,
|
|
1549
|
-
L: 322,
|
|
1550
|
-
S: this,
|
|
1551
|
-
C: (f, a) => f(...a)
|
|
1552
|
-
});
|
|
1553
1816
|
await this._processEpoch(ctx, epoch.subject.assertion);
|
|
1554
1817
|
this.appliedEpoch = epoch;
|
|
1555
1818
|
this.onNewEpoch.emit(epoch);
|
|
1556
1819
|
});
|
|
1557
1820
|
}
|
|
1558
1821
|
async _processEpoch(ctx, epoch) {
|
|
1559
|
-
(0,
|
|
1560
|
-
|
|
1822
|
+
(0, import_invariant8.invariant)(this._isOpen, "Space is closed.", {
|
|
1823
|
+
F: __dxlog_file10,
|
|
1824
|
+
L: 344,
|
|
1825
|
+
S: this,
|
|
1826
|
+
A: [
|
|
1827
|
+
"this._isOpen",
|
|
1828
|
+
"'Space is closed.'"
|
|
1829
|
+
]
|
|
1830
|
+
});
|
|
1831
|
+
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1832
|
+
F: __dxlog_file10,
|
|
1833
|
+
L: 345,
|
|
1834
|
+
S: this,
|
|
1835
|
+
A: [
|
|
1836
|
+
"this._pipeline",
|
|
1837
|
+
""
|
|
1838
|
+
]
|
|
1839
|
+
});
|
|
1561
1840
|
this._lastProcessedEpoch = epoch.number;
|
|
1562
|
-
(0, import_log9.log)("
|
|
1563
|
-
epoch
|
|
1841
|
+
(0, import_log9.log)("processing", {
|
|
1842
|
+
epoch: (0, import_log9.omit)(epoch, "proof")
|
|
1564
1843
|
}, {
|
|
1565
|
-
F:
|
|
1566
|
-
L:
|
|
1844
|
+
F: __dxlog_file10,
|
|
1845
|
+
L: 348,
|
|
1567
1846
|
S: this,
|
|
1568
1847
|
C: (f, a) => f(...a)
|
|
1569
1848
|
});
|
|
@@ -1571,9 +1850,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1571
1850
|
const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
|
|
1572
1851
|
this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
|
|
1573
1852
|
}
|
|
1574
|
-
(0, import_log9.log)("restarting pipeline
|
|
1575
|
-
F:
|
|
1576
|
-
L:
|
|
1853
|
+
(0, import_log9.log)("restarting pipeline from epoch", void 0, {
|
|
1854
|
+
F: __dxlog_file10,
|
|
1855
|
+
L: 354,
|
|
1577
1856
|
S: this,
|
|
1578
1857
|
C: (f, a) => f(...a)
|
|
1579
1858
|
});
|
|
@@ -1582,12 +1861,36 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1582
1861
|
await this._pipeline.unpause();
|
|
1583
1862
|
}
|
|
1584
1863
|
async waitUntilTimeframe(timeframe) {
|
|
1585
|
-
(0,
|
|
1864
|
+
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1865
|
+
F: __dxlog_file10,
|
|
1866
|
+
L: 361,
|
|
1867
|
+
S: this,
|
|
1868
|
+
A: [
|
|
1869
|
+
"this._pipeline",
|
|
1870
|
+
"'Pipeline is not initialized.'"
|
|
1871
|
+
]
|
|
1872
|
+
});
|
|
1586
1873
|
await this._pipeline.state.waitUntilTimeframe(timeframe);
|
|
1587
1874
|
}
|
|
1588
1875
|
async createEpoch() {
|
|
1589
|
-
(0,
|
|
1590
|
-
|
|
1876
|
+
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1877
|
+
F: __dxlog_file10,
|
|
1878
|
+
L: 367,
|
|
1879
|
+
S: this,
|
|
1880
|
+
A: [
|
|
1881
|
+
"this._pipeline",
|
|
1882
|
+
""
|
|
1883
|
+
]
|
|
1884
|
+
});
|
|
1885
|
+
(0, import_invariant8.invariant)(this.currentEpoch, void 0, {
|
|
1886
|
+
F: __dxlog_file10,
|
|
1887
|
+
L: 368,
|
|
1888
|
+
S: this,
|
|
1889
|
+
A: [
|
|
1890
|
+
"this.currentEpoch",
|
|
1891
|
+
""
|
|
1892
|
+
]
|
|
1893
|
+
});
|
|
1591
1894
|
await this._pipeline.pause();
|
|
1592
1895
|
const snapshot = await this._createSnapshot();
|
|
1593
1896
|
const snapshotCid = await this._params.snapshotManager.store(snapshot);
|
|
@@ -1604,24 +1907,31 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1604
1907
|
await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
|
|
1605
1908
|
}
|
|
1606
1909
|
};
|
|
1607
|
-
|
|
1910
|
+
_ts_decorate6([
|
|
1911
|
+
import_tracing2.trace.metricsCounter()
|
|
1912
|
+
], DataPipeline.prototype, "_usage", void 0);
|
|
1913
|
+
_ts_decorate6([
|
|
1914
|
+
import_tracing2.trace.metricsCounter()
|
|
1915
|
+
], DataPipeline.prototype, "_mutations", void 0);
|
|
1916
|
+
_ts_decorate6([
|
|
1608
1917
|
import_async6.synchronized
|
|
1609
1918
|
], DataPipeline.prototype, "open", null);
|
|
1610
|
-
|
|
1919
|
+
_ts_decorate6([
|
|
1611
1920
|
import_async6.synchronized
|
|
1612
1921
|
], DataPipeline.prototype, "close", null);
|
|
1613
|
-
|
|
1922
|
+
_ts_decorate6([
|
|
1614
1923
|
import_async6.synchronized
|
|
1615
1924
|
], DataPipeline.prototype, "_processEpoch", null);
|
|
1616
|
-
|
|
1925
|
+
_ts_decorate6([
|
|
1617
1926
|
import_async6.synchronized
|
|
1618
1927
|
], DataPipeline.prototype, "createEpoch", null);
|
|
1619
|
-
DataPipeline =
|
|
1620
|
-
(0, import_async6.trackLeaks)("open", "close")
|
|
1928
|
+
DataPipeline = _ts_decorate6([
|
|
1929
|
+
(0, import_async6.trackLeaks)("open", "close"),
|
|
1930
|
+
import_tracing2.trace.resource()
|
|
1621
1931
|
], DataPipeline);
|
|
1622
1932
|
|
|
1623
1933
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
1624
|
-
function
|
|
1934
|
+
function _ts_decorate7(decorators, target, key, desc) {
|
|
1625
1935
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1626
1936
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1627
1937
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1631,14 +1941,22 @@ function _ts_decorate6(decorators, target, key, desc) {
|
|
|
1631
1941
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1632
1942
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1633
1943
|
}
|
|
1634
|
-
var
|
|
1944
|
+
var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
|
|
1635
1945
|
var Space = class Space2 {
|
|
1636
1946
|
constructor(params) {
|
|
1637
1947
|
this._addFeedLock = new import_async7.Lock();
|
|
1638
1948
|
this.onCredentialProcessed = new import_util7.Callback();
|
|
1639
1949
|
this.stateUpdate = new import_async7.Event();
|
|
1640
1950
|
this._isOpen = false;
|
|
1641
|
-
(0,
|
|
1951
|
+
(0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
|
|
1952
|
+
F: __dxlog_file11,
|
|
1953
|
+
L: 73,
|
|
1954
|
+
S: this,
|
|
1955
|
+
A: [
|
|
1956
|
+
"params.spaceKey && params.feedProvider",
|
|
1957
|
+
""
|
|
1958
|
+
]
|
|
1959
|
+
});
|
|
1642
1960
|
this._key = params.spaceKey;
|
|
1643
1961
|
this._genesisFeedKey = params.genesisFeed.key;
|
|
1644
1962
|
this._feedProvider = params.feedProvider;
|
|
@@ -1673,8 +1991,8 @@ var Space = class Space2 {
|
|
|
1673
1991
|
(0, import_log10.log)("onCredentialProcessed", {
|
|
1674
1992
|
credential
|
|
1675
1993
|
}, {
|
|
1676
|
-
F:
|
|
1677
|
-
L:
|
|
1994
|
+
F: __dxlog_file11,
|
|
1995
|
+
L: 109,
|
|
1678
1996
|
S: this,
|
|
1679
1997
|
C: (f, a) => f(...a)
|
|
1680
1998
|
});
|
|
@@ -1716,12 +2034,10 @@ var Space = class Space2 {
|
|
|
1716
2034
|
return this._genesisFeedKey;
|
|
1717
2035
|
}
|
|
1718
2036
|
get controlFeedKey() {
|
|
1719
|
-
|
|
1720
|
-
return (_a = this._controlFeed) == null ? void 0 : _a.key;
|
|
2037
|
+
return this._controlFeed?.key;
|
|
1721
2038
|
}
|
|
1722
2039
|
get dataFeedKey() {
|
|
1723
|
-
|
|
1724
|
-
return (_a = this._dataFeed) == null ? void 0 : _a.key;
|
|
2040
|
+
return this._dataFeed?.key;
|
|
1725
2041
|
}
|
|
1726
2042
|
get spaceState() {
|
|
1727
2043
|
return this._controlPipeline.spaceState;
|
|
@@ -1739,16 +2055,31 @@ var Space = class Space2 {
|
|
|
1739
2055
|
return this._snapshotManager;
|
|
1740
2056
|
}
|
|
1741
2057
|
setControlFeed(feed) {
|
|
1742
|
-
(0,
|
|
2058
|
+
(0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
|
|
2059
|
+
F: __dxlog_file11,
|
|
2060
|
+
L: 184,
|
|
2061
|
+
S: this,
|
|
2062
|
+
A: [
|
|
2063
|
+
"!this._controlFeed",
|
|
2064
|
+
"'Control feed already set.'"
|
|
2065
|
+
]
|
|
2066
|
+
});
|
|
1743
2067
|
this._controlFeed = feed;
|
|
1744
2068
|
this._controlPipeline.setWriteFeed(feed);
|
|
1745
2069
|
return this;
|
|
1746
2070
|
}
|
|
1747
2071
|
setDataFeed(feed) {
|
|
1748
|
-
|
|
1749
|
-
|
|
2072
|
+
(0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
|
|
2073
|
+
F: __dxlog_file11,
|
|
2074
|
+
L: 191,
|
|
2075
|
+
S: this,
|
|
2076
|
+
A: [
|
|
2077
|
+
"!this._dataFeed",
|
|
2078
|
+
"'Data feed already set.'"
|
|
2079
|
+
]
|
|
2080
|
+
});
|
|
1750
2081
|
this._dataFeed = feed;
|
|
1751
|
-
|
|
2082
|
+
this._dataPipeline.pipeline?.setWriteFeed(feed);
|
|
1752
2083
|
return this;
|
|
1753
2084
|
}
|
|
1754
2085
|
/**
|
|
@@ -1763,10 +2094,10 @@ var Space = class Space2 {
|
|
|
1763
2094
|
// getDataFeeds(): FeedInfo[] {
|
|
1764
2095
|
// return this._dataPipeline?.getFeeds();
|
|
1765
2096
|
// }
|
|
1766
|
-
async open() {
|
|
2097
|
+
async open(ctx) {
|
|
1767
2098
|
(0, import_log10.log)("opening...", void 0, {
|
|
1768
|
-
F:
|
|
1769
|
-
L:
|
|
2099
|
+
F: __dxlog_file11,
|
|
2100
|
+
L: 213,
|
|
1770
2101
|
S: this,
|
|
1771
2102
|
C: (f, a) => f(...a)
|
|
1772
2103
|
});
|
|
@@ -1778,8 +2109,8 @@ var Space = class Space2 {
|
|
|
1778
2109
|
await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
|
|
1779
2110
|
this._isOpen = true;
|
|
1780
2111
|
(0, import_log10.log)("opened", void 0, {
|
|
1781
|
-
F:
|
|
1782
|
-
L:
|
|
2112
|
+
F: __dxlog_file11,
|
|
2113
|
+
L: 224,
|
|
1783
2114
|
S: this,
|
|
1784
2115
|
C: (f, a) => f(...a)
|
|
1785
2116
|
});
|
|
@@ -1788,8 +2119,8 @@ var Space = class Space2 {
|
|
|
1788
2119
|
(0, import_log10.log)("closing...", {
|
|
1789
2120
|
key: this._key
|
|
1790
2121
|
}, {
|
|
1791
|
-
F:
|
|
1792
|
-
L:
|
|
2122
|
+
F: __dxlog_file11,
|
|
2123
|
+
L: 229,
|
|
1793
2124
|
S: this,
|
|
1794
2125
|
C: (f, a) => f(...a)
|
|
1795
2126
|
});
|
|
@@ -1802,42 +2133,53 @@ var Space = class Space2 {
|
|
|
1802
2133
|
await this._controlPipeline.stop();
|
|
1803
2134
|
this._isOpen = false;
|
|
1804
2135
|
(0, import_log10.log)("closed", void 0, {
|
|
1805
|
-
F:
|
|
1806
|
-
L:
|
|
2136
|
+
F: __dxlog_file11,
|
|
2137
|
+
L: 242,
|
|
1807
2138
|
S: this,
|
|
1808
2139
|
C: (f, a) => f(...a)
|
|
1809
2140
|
});
|
|
1810
2141
|
}
|
|
1811
2142
|
async initializeDataPipeline() {
|
|
1812
2143
|
(0, import_log10.log)("initializeDataPipeline", void 0, {
|
|
1813
|
-
F:
|
|
1814
|
-
L:
|
|
2144
|
+
F: __dxlog_file11,
|
|
2145
|
+
L: 247,
|
|
1815
2146
|
S: this,
|
|
1816
2147
|
C: (f, a) => f(...a)
|
|
1817
2148
|
});
|
|
1818
|
-
(0,
|
|
2149
|
+
(0, import_invariant9.invariant)(this._isOpen, "Space must be open to initialize data pipeline.", {
|
|
2150
|
+
F: __dxlog_file11,
|
|
2151
|
+
L: 248,
|
|
2152
|
+
S: this,
|
|
2153
|
+
A: [
|
|
2154
|
+
"this._isOpen",
|
|
2155
|
+
"'Space must be open to initialize data pipeline.'"
|
|
2156
|
+
]
|
|
2157
|
+
});
|
|
1819
2158
|
await this._dataPipeline.open();
|
|
1820
2159
|
}
|
|
1821
2160
|
};
|
|
1822
|
-
|
|
1823
|
-
import_log10.logInfo
|
|
2161
|
+
_ts_decorate7([
|
|
2162
|
+
import_log10.logInfo,
|
|
2163
|
+
import_tracing3.trace.info()
|
|
1824
2164
|
], Space.prototype, "key", null);
|
|
1825
|
-
|
|
1826
|
-
import_async7.synchronized
|
|
2165
|
+
_ts_decorate7([
|
|
2166
|
+
import_async7.synchronized,
|
|
2167
|
+
import_tracing3.trace.span()
|
|
1827
2168
|
], Space.prototype, "open", null);
|
|
1828
|
-
|
|
2169
|
+
_ts_decorate7([
|
|
1829
2170
|
import_async7.synchronized
|
|
1830
2171
|
], Space.prototype, "close", null);
|
|
1831
|
-
|
|
2172
|
+
_ts_decorate7([
|
|
1832
2173
|
import_async7.synchronized
|
|
1833
2174
|
], Space.prototype, "initializeDataPipeline", null);
|
|
1834
|
-
Space =
|
|
1835
|
-
(0, import_async7.trackLeaks)("open", "close")
|
|
2175
|
+
Space = _ts_decorate7([
|
|
2176
|
+
(0, import_async7.trackLeaks)("open", "close"),
|
|
2177
|
+
import_tracing3.trace.resource()
|
|
1836
2178
|
], Space);
|
|
1837
2179
|
|
|
1838
2180
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
1839
2181
|
var import_async8 = require("@dxos/async");
|
|
1840
|
-
var
|
|
2182
|
+
var import_debug4 = require("@dxos/debug");
|
|
1841
2183
|
var import_keys6 = require("@dxos/keys");
|
|
1842
2184
|
var import_log12 = require("@dxos/log");
|
|
1843
2185
|
var import_protocols6 = require("@dxos/protocols");
|
|
@@ -1852,7 +2194,7 @@ var import_teleport2 = require("@dxos/teleport");
|
|
|
1852
2194
|
var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
|
|
1853
2195
|
var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
|
|
1854
2196
|
var import_util8 = require("@dxos/util");
|
|
1855
|
-
function
|
|
2197
|
+
function _ts_decorate8(decorators, target, key, desc) {
|
|
1856
2198
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1857
2199
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1858
2200
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1862,7 +2204,7 @@ function _ts_decorate7(decorators, target, key, desc) {
|
|
|
1862
2204
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1863
2205
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1864
2206
|
}
|
|
1865
|
-
var
|
|
2207
|
+
var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
|
|
1866
2208
|
var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
|
|
1867
2209
|
var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
|
|
1868
2210
|
var SpaceProtocol = class {
|
|
@@ -1892,8 +2234,8 @@ var SpaceProtocol = class {
|
|
|
1892
2234
|
(0, import_log11.log)("addFeed", {
|
|
1893
2235
|
key: feed.key
|
|
1894
2236
|
}, {
|
|
1895
|
-
F:
|
|
1896
|
-
L:
|
|
2237
|
+
F: __dxlog_file12,
|
|
2238
|
+
L: 96,
|
|
1897
2239
|
S: this,
|
|
1898
2240
|
C: (f, a) => f(...a)
|
|
1899
2241
|
});
|
|
@@ -1902,6 +2244,7 @@ var SpaceProtocol = class {
|
|
|
1902
2244
|
session.replicator.addFeed(feed);
|
|
1903
2245
|
}
|
|
1904
2246
|
}
|
|
2247
|
+
// TODO(burdon): Rename open? Common open/close interfaces for all services?
|
|
1905
2248
|
async start() {
|
|
1906
2249
|
if (this._connection) {
|
|
1907
2250
|
return;
|
|
@@ -1914,8 +2257,8 @@ var SpaceProtocol = class {
|
|
|
1914
2257
|
};
|
|
1915
2258
|
await this.blobSync.open();
|
|
1916
2259
|
(0, import_log11.log)("starting...", void 0, {
|
|
1917
|
-
F:
|
|
1918
|
-
L:
|
|
2260
|
+
F: __dxlog_file12,
|
|
2261
|
+
L: 122,
|
|
1919
2262
|
S: this,
|
|
1920
2263
|
C: (f, a) => f(...a)
|
|
1921
2264
|
});
|
|
@@ -1925,11 +2268,11 @@ var SpaceProtocol = class {
|
|
|
1925
2268
|
peerId: this._swarmIdentity.peerKey,
|
|
1926
2269
|
topic,
|
|
1927
2270
|
topology: new import_network_manager.MMSTTopology(topologyConfig),
|
|
1928
|
-
label: `
|
|
2271
|
+
label: `space swarm ${topic.truncate()}`
|
|
1929
2272
|
});
|
|
1930
2273
|
(0, import_log11.log)("started", void 0, {
|
|
1931
|
-
F:
|
|
1932
|
-
L:
|
|
2274
|
+
F: __dxlog_file12,
|
|
2275
|
+
L: 132,
|
|
1933
2276
|
S: this,
|
|
1934
2277
|
C: (f, a) => f(...a)
|
|
1935
2278
|
});
|
|
@@ -1938,15 +2281,15 @@ var SpaceProtocol = class {
|
|
|
1938
2281
|
await this.blobSync.close();
|
|
1939
2282
|
if (this._connection) {
|
|
1940
2283
|
(0, import_log11.log)("stopping...", void 0, {
|
|
1941
|
-
F:
|
|
1942
|
-
L:
|
|
2284
|
+
F: __dxlog_file12,
|
|
2285
|
+
L: 139,
|
|
1943
2286
|
S: this,
|
|
1944
2287
|
C: (f, a) => f(...a)
|
|
1945
2288
|
});
|
|
1946
2289
|
await this._connection.close();
|
|
1947
2290
|
(0, import_log11.log)("stopped", void 0, {
|
|
1948
|
-
F:
|
|
1949
|
-
L:
|
|
2291
|
+
F: __dxlog_file12,
|
|
2292
|
+
L: 141,
|
|
1950
2293
|
S: this,
|
|
1951
2294
|
C: (f, a) => f(...a)
|
|
1952
2295
|
});
|
|
@@ -1969,10 +2312,10 @@ var SpaceProtocol = class {
|
|
|
1969
2312
|
};
|
|
1970
2313
|
}
|
|
1971
2314
|
};
|
|
1972
|
-
|
|
2315
|
+
_ts_decorate8([
|
|
1973
2316
|
import_log11.logInfo
|
|
1974
2317
|
], SpaceProtocol.prototype, "_topic", void 0);
|
|
1975
|
-
|
|
2318
|
+
_ts_decorate8([
|
|
1976
2319
|
import_log11.logInfo
|
|
1977
2320
|
], SpaceProtocol.prototype, "_ownPeerKey", null);
|
|
1978
2321
|
var AuthStatus;
|
|
@@ -2011,20 +2354,18 @@ var SpaceProtocolSession = class {
|
|
|
2011
2354
|
provider: this._swarmIdentity.credentialProvider,
|
|
2012
2355
|
verifier: this._swarmIdentity.credentialAuthenticator,
|
|
2013
2356
|
onAuthSuccess: () => {
|
|
2014
|
-
var _a;
|
|
2015
2357
|
(0, import_log11.log)("Peer authenticated", void 0, {
|
|
2016
|
-
F:
|
|
2017
|
-
L:
|
|
2358
|
+
F: __dxlog_file12,
|
|
2359
|
+
L: 238,
|
|
2018
2360
|
S: this,
|
|
2019
2361
|
C: (f, a) => f(...a)
|
|
2020
2362
|
});
|
|
2021
2363
|
this._authStatus = AuthStatus.SUCCESS;
|
|
2022
|
-
|
|
2364
|
+
this._onSessionAuth?.(this._teleport);
|
|
2023
2365
|
},
|
|
2024
2366
|
onAuthFailure: () => {
|
|
2025
|
-
var _a;
|
|
2026
2367
|
this._authStatus = AuthStatus.FAILURE;
|
|
2027
|
-
|
|
2368
|
+
this._onAuthFailure?.(this._teleport);
|
|
2028
2369
|
}
|
|
2029
2370
|
}));
|
|
2030
2371
|
this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
|
|
@@ -2034,15 +2375,15 @@ var SpaceProtocolSession = class {
|
|
|
2034
2375
|
await this._teleport.close();
|
|
2035
2376
|
}
|
|
2036
2377
|
};
|
|
2037
|
-
|
|
2378
|
+
_ts_decorate8([
|
|
2038
2379
|
import_log11.logInfo
|
|
2039
2380
|
], SpaceProtocolSession.prototype, "_wireParams", void 0);
|
|
2040
|
-
|
|
2381
|
+
_ts_decorate8([
|
|
2041
2382
|
import_log11.logInfo
|
|
2042
2383
|
], SpaceProtocolSession.prototype, "authStatus", null);
|
|
2043
2384
|
|
|
2044
2385
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
2045
|
-
function
|
|
2386
|
+
function _ts_decorate9(decorators, target, key, desc) {
|
|
2046
2387
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2047
2388
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
2048
2389
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2052,7 +2393,7 @@ function _ts_decorate8(decorators, target, key, desc) {
|
|
|
2052
2393
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2053
2394
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2054
2395
|
}
|
|
2055
|
-
var
|
|
2396
|
+
var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
|
|
2056
2397
|
var SpaceManager = class SpaceManager2 {
|
|
2057
2398
|
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
|
|
2058
2399
|
this._spaces = new import_util9.ComplexMap(import_keys6.PublicKey.hash);
|
|
@@ -2076,11 +2417,10 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2076
2417
|
].map((space) => space.close()));
|
|
2077
2418
|
}
|
|
2078
2419
|
async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
|
|
2079
|
-
var _a;
|
|
2080
2420
|
import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.begin({
|
|
2081
2421
|
id: this._instanceId
|
|
2082
2422
|
}), {
|
|
2083
|
-
F:
|
|
2423
|
+
F: __dxlog_file13,
|
|
2084
2424
|
L: 97,
|
|
2085
2425
|
S: this,
|
|
2086
2426
|
C: (f, a) => f(...a)
|
|
@@ -2088,12 +2428,12 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2088
2428
|
(0, import_log12.log)("constructing space...", {
|
|
2089
2429
|
spaceKey: metadata.genesisFeedKey
|
|
2090
2430
|
}, {
|
|
2091
|
-
F:
|
|
2431
|
+
F: __dxlog_file13,
|
|
2092
2432
|
L: 98,
|
|
2093
2433
|
S: this,
|
|
2094
2434
|
C: (f, a) => f(...a)
|
|
2095
2435
|
});
|
|
2096
|
-
const genesisFeed = await this._feedStore.openFeed(
|
|
2436
|
+
const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? (0, import_debug4.failUndefined)());
|
|
2097
2437
|
const spaceKey = metadata.key;
|
|
2098
2438
|
const protocol = new SpaceProtocol({
|
|
2099
2439
|
topic: spaceKey,
|
|
@@ -2118,21 +2458,21 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2118
2458
|
import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.end({
|
|
2119
2459
|
id: this._instanceId
|
|
2120
2460
|
}), {
|
|
2121
|
-
F:
|
|
2122
|
-
L:
|
|
2461
|
+
F: __dxlog_file13,
|
|
2462
|
+
L: 126,
|
|
2123
2463
|
S: this,
|
|
2124
2464
|
C: (f, a) => f(...a)
|
|
2125
2465
|
});
|
|
2126
2466
|
return space;
|
|
2127
2467
|
}
|
|
2128
2468
|
};
|
|
2129
|
-
|
|
2469
|
+
_ts_decorate9([
|
|
2130
2470
|
import_async8.synchronized
|
|
2131
2471
|
], SpaceManager.prototype, "open", null);
|
|
2132
|
-
|
|
2472
|
+
_ts_decorate9([
|
|
2133
2473
|
import_async8.synchronized
|
|
2134
2474
|
], SpaceManager.prototype, "close", null);
|
|
2135
|
-
SpaceManager =
|
|
2475
|
+
SpaceManager = _ts_decorate9([
|
|
2136
2476
|
(0, import_async8.trackLeaks)("open", "close")
|
|
2137
2477
|
], SpaceManager);
|
|
2138
2478
|
|
|
@@ -2162,10 +2502,10 @@ var WebsocketNetworkManagerProvider = (signalUrl) => () => new import_network_ma
|
|
|
2162
2502
|
var TestAgentBuilder = class {
|
|
2163
2503
|
constructor({ storage, networkManagerProvider } = {}) {
|
|
2164
2504
|
this._agents = new import_util10.ComplexMap(import_keys7.PublicKey.hash);
|
|
2165
|
-
this._storage = storage
|
|
2505
|
+
this._storage = storage ?? (0, import_random_access_storage.createStorage)({
|
|
2166
2506
|
type: import_random_access_storage.StorageType.RAM
|
|
2167
2507
|
});
|
|
2168
|
-
this._networkManagerProvider = networkManagerProvider
|
|
2508
|
+
this._networkManagerProvider = networkManagerProvider ?? MemoryNetworkManagerProvider(new import_messaging.MemorySignalManagerContext());
|
|
2169
2509
|
}
|
|
2170
2510
|
async close() {
|
|
2171
2511
|
return Promise.all(this.agents.map((agent) => agent.close()));
|
|
@@ -2187,16 +2527,13 @@ var TestAgentBuilder = class {
|
|
|
2187
2527
|
};
|
|
2188
2528
|
var TestAgent = class {
|
|
2189
2529
|
get metadataStore() {
|
|
2190
|
-
|
|
2191
|
-
return (_a = this._metadataStore) != null ? _a : this._metadataStore = new MetadataStore(this.storage.createDirectory("metadata"));
|
|
2530
|
+
return this._metadataStore ??= new MetadataStore(this.storage.createDirectory("metadata"));
|
|
2192
2531
|
}
|
|
2193
2532
|
get snapshotStore() {
|
|
2194
|
-
|
|
2195
|
-
return (_a = this._snapshotStore) != null ? _a : this._snapshotStore = new SnapshotStore(this.storage.createDirectory("snapshots"));
|
|
2533
|
+
return this._snapshotStore ??= new SnapshotStore(this.storage.createDirectory("snapshots"));
|
|
2196
2534
|
}
|
|
2197
2535
|
get blobStore() {
|
|
2198
|
-
|
|
2199
|
-
return (_a = this._blobStore) != null ? _a : this._blobStore = new import_teleport_extension_object_sync2.BlobStore(this.storage.createDirectory("blobs"));
|
|
2536
|
+
return this._blobStore ??= new import_teleport_extension_object_sync2.BlobStore(this.storage.createDirectory("blobs"));
|
|
2200
2537
|
}
|
|
2201
2538
|
constructor(_networkManagerProvider, _feedBuilder, identityKey, deviceKey) {
|
|
2202
2539
|
this._networkManagerProvider = _networkManagerProvider;
|
|
@@ -2221,8 +2558,7 @@ var TestAgent = class {
|
|
|
2221
2558
|
return this._spaces.get(spaceKey);
|
|
2222
2559
|
}
|
|
2223
2560
|
get spaceManager() {
|
|
2224
|
-
|
|
2225
|
-
return (_a = this._spaceManager) != null ? _a : this._spaceManager = new SpaceManager({
|
|
2561
|
+
return this._spaceManager ??= new SpaceManager({
|
|
2226
2562
|
feedStore: this.feedStore,
|
|
2227
2563
|
networkManager: this._networkManagerProvider(),
|
|
2228
2564
|
modelFactory: this.modelFactory,
|
|
@@ -2242,7 +2578,7 @@ var TestAgent = class {
|
|
|
2242
2578
|
const controlFeed = await this.feedStore.openFeed(genesisKey, {
|
|
2243
2579
|
writable: true
|
|
2244
2580
|
});
|
|
2245
|
-
const dataFeed = await this.feedStore.openFeed(dataKey
|
|
2581
|
+
const dataFeed = await this.feedStore.openFeed(dataKey ?? await this.keyring.createKey(), {
|
|
2246
2582
|
writable: true,
|
|
2247
2583
|
sparse: true
|
|
2248
2584
|
});
|
|
@@ -2272,7 +2608,7 @@ var TestAgent = class {
|
|
|
2272
2608
|
});
|
|
2273
2609
|
space.setControlFeed(controlFeed);
|
|
2274
2610
|
space.setDataFeed(dataFeed);
|
|
2275
|
-
await space.open();
|
|
2611
|
+
await space.open(new import_context7.Context());
|
|
2276
2612
|
this._spaces.set(spaceKey, space);
|
|
2277
2613
|
return space;
|
|
2278
2614
|
}
|
|
@@ -2287,7 +2623,7 @@ var TestAgent = class {
|
|
|
2287
2623
|
networkManager: this._networkManagerProvider(),
|
|
2288
2624
|
blobStore: this.blobStore,
|
|
2289
2625
|
onSessionAuth: (session) => {
|
|
2290
|
-
session.addExtension("dxos.mesh.teleport.gossip", (gossip
|
|
2626
|
+
session.addExtension("dxos.mesh.teleport.gossip", (gossip ?? this.createGossip()).createExtension({
|
|
2291
2627
|
remotePeerId: session.remotePeerId
|
|
2292
2628
|
}));
|
|
2293
2629
|
}
|
|
@@ -2303,7 +2639,7 @@ var TestAgent = class {
|
|
|
2303
2639
|
announceInterval: 30,
|
|
2304
2640
|
offlineTimeout: 200,
|
|
2305
2641
|
identityKey: this.identityKey,
|
|
2306
|
-
gossip: gossip
|
|
2642
|
+
gossip: gossip ?? this.createGossip()
|
|
2307
2643
|
});
|
|
2308
2644
|
}
|
|
2309
2645
|
async spaceGenesis(space) {
|
|
@@ -2368,9 +2704,8 @@ var createRemoteDatabaseFromDataServiceHost = async (modelFactory, dataServiceHo
|
|
|
2368
2704
|
};
|
|
2369
2705
|
};
|
|
2370
2706
|
var testLocalDatabase = async (create, check = create) => {
|
|
2371
|
-
var _a;
|
|
2372
2707
|
const objectId = import_keys8.PublicKey.random().toHex();
|
|
2373
|
-
await
|
|
2708
|
+
await create.databaseHost.getWriteStream()?.write({
|
|
2374
2709
|
batch: {
|
|
2375
2710
|
objects: [
|
|
2376
2711
|
{
|
|
@@ -2381,7 +2716,7 @@ var testLocalDatabase = async (create, check = create) => {
|
|
|
2381
2716
|
}
|
|
2382
2717
|
]
|
|
2383
2718
|
}
|
|
2384
|
-
})
|
|
2719
|
+
});
|
|
2385
2720
|
await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 2e3);
|
|
2386
2721
|
};
|
|
2387
2722
|
// Annotate the CommonJS export names for ESM import in node:
|