@dxos/echo-pipeline 0.1.56-main.e47dfd1 → 0.1.56-main.e79d64a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{chunk-7U2NXI2P.mjs → chunk-GDF4DQU2.mjs} +692 -355
- package/dist/lib/browser/chunk-GDF4DQU2.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1 -1
- package/dist/lib/browser/index.mjs.map +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/testing/index.mjs +14 -18
- package/dist/lib/browser/testing/index.mjs.map +3 -3
- package/dist/lib/node/index.cjs +699 -362
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/testing/index.cjs +696 -363
- package/dist/lib/node/testing/index.cjs.map +3 -3
- package/dist/types/src/common/feeds.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service-host.d.ts.map +1 -1
- package/dist/types/src/db-host/data-service.d.ts.map +1 -1
- package/dist/types/src/db-host/snapshot-manager.d.ts.map +1 -1
- package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
- package/dist/types/src/pipeline/message-selector.d.ts.map +1 -1
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts +2 -0
- package/dist/types/src/pipeline/pipeline-stress.test.d.ts.map +1 -0
- package/dist/types/src/pipeline/pipeline.d.ts +4 -3
- package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
- package/dist/types/src/space/auth.d.ts.map +1 -1
- package/dist/types/src/space/control-pipeline.d.ts +4 -0
- package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/data-pipeline.d.ts +2 -0
- package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
- package/dist/types/src/space/space-manager.d.ts.map +1 -1
- package/dist/types/src/space/space-protocol.d.ts.map +1 -1
- package/dist/types/src/space/space.d.ts +2 -1
- package/dist/types/src/space/space.d.ts.map +1 -1
- package/dist/types/src/testing/database-test-rig.d.ts +4 -1
- package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
- package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
- package/package.json +32 -31
- package/src/common/feeds.ts +1 -2
- package/src/db-host/data-service-host.ts +10 -4
- package/src/db-host/data-service.ts +1 -2
- package/src/db-host/snapshot-manager.ts +0 -2
- package/src/metadata/metadata-store.ts +2 -2
- package/src/pipeline/message-selector.ts +1 -2
- package/src/pipeline/pipeline-stress.test.ts +226 -0
- package/src/pipeline/pipeline.test.ts +30 -206
- package/src/pipeline/pipeline.ts +50 -24
- package/src/space/auth.ts +1 -2
- package/src/space/control-pipeline.ts +47 -22
- package/src/space/data-pipeline.ts +21 -12
- package/src/space/space-manager.ts +1 -0
- package/src/space/space-protocol.ts +3 -1
- package/src/space/space.test.ts +18 -17
- package/src/space/space.ts +7 -3
- package/src/testing/database-test-rig.ts +27 -2
- package/src/testing/test-agent-builder.ts +2 -1
- package/src/tests/database-unit.test.ts +39 -0
- package/dist/lib/browser/chunk-7U2NXI2P.mjs.map +0 -7
|
@@ -42,6 +42,7 @@ __export(testing_exports, {
|
|
|
42
42
|
module.exports = __toCommonJS(testing_exports);
|
|
43
43
|
|
|
44
44
|
// packages/core/echo/echo-pipeline/src/testing/test-agent-builder.ts
|
|
45
|
+
var import_context7 = require("@dxos/context");
|
|
45
46
|
var import_credentials5 = require("@dxos/credentials");
|
|
46
47
|
var import_document_model = require("@dxos/document-model");
|
|
47
48
|
var import_keys7 = require("@dxos/keys");
|
|
@@ -55,11 +56,12 @@ var import_teleport_extension_object_sync2 = require("@dxos/teleport-extension-o
|
|
|
55
56
|
var import_util10 = require("@dxos/util");
|
|
56
57
|
|
|
57
58
|
// packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
|
|
58
|
-
var import_tiny_invariant = __toESM(require("tiny-invariant"));
|
|
59
59
|
var import_codec_protobuf = require("@dxos/codec-protobuf");
|
|
60
60
|
var import_context = require("@dxos/context");
|
|
61
61
|
var import_echo_db = require("@dxos/echo-db");
|
|
62
|
+
var import_invariant = require("@dxos/invariant");
|
|
62
63
|
var import_log = require("@dxos/log");
|
|
64
|
+
var import_service = require("@dxos/protocols/proto/dxos/echo/service");
|
|
63
65
|
var import_util = require("@dxos/util");
|
|
64
66
|
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
|
|
65
67
|
var DataServiceHost = class {
|
|
@@ -87,16 +89,31 @@ var DataServiceHost = class {
|
|
|
87
89
|
objects
|
|
88
90
|
}
|
|
89
91
|
});
|
|
92
|
+
this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
|
|
93
|
+
next({
|
|
94
|
+
action: import_service.EchoEvent.DatabaseAction.RESET,
|
|
95
|
+
batch: {
|
|
96
|
+
objects: snapshot.items
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
});
|
|
90
100
|
this._itemDemuxer.mutation.on(ctx, (message) => {
|
|
91
|
-
var _a;
|
|
92
101
|
const { batch, meta } = message;
|
|
93
|
-
(0,
|
|
102
|
+
(0, import_invariant.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
|
|
103
|
+
F: __dxlog_file,
|
|
104
|
+
L: 68,
|
|
105
|
+
S: this,
|
|
106
|
+
A: [
|
|
107
|
+
"!(meta as any).clientTag",
|
|
108
|
+
"'Unexpected client tag in mutation message'"
|
|
109
|
+
]
|
|
110
|
+
});
|
|
94
111
|
(0, import_log.log)("message", {
|
|
95
|
-
batch,
|
|
112
|
+
batch: batch.objects?.length,
|
|
96
113
|
meta
|
|
97
114
|
}, {
|
|
98
115
|
F: __dxlog_file,
|
|
99
|
-
L:
|
|
116
|
+
L: 69,
|
|
100
117
|
S: this,
|
|
101
118
|
C: (f, a) => f(...a)
|
|
102
119
|
});
|
|
@@ -104,7 +121,7 @@ var DataServiceHost = class {
|
|
|
104
121
|
message.meta.feedKey,
|
|
105
122
|
message.meta.seq
|
|
106
123
|
]);
|
|
107
|
-
|
|
124
|
+
batch.objects?.forEach((object) => {
|
|
108
125
|
(0, import_echo_db.setMetadataOnObject)(object, {
|
|
109
126
|
...meta
|
|
110
127
|
});
|
|
@@ -122,15 +139,30 @@ var DataServiceHost = class {
|
|
|
122
139
|
});
|
|
123
140
|
}
|
|
124
141
|
async write(request) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
142
|
+
(0, import_invariant.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
|
|
143
|
+
F: __dxlog_file,
|
|
144
|
+
L: 95,
|
|
145
|
+
S: this,
|
|
146
|
+
A: [
|
|
147
|
+
"!this._ctx.disposed",
|
|
148
|
+
"'Cannot write to closed DataServiceHost'"
|
|
149
|
+
]
|
|
150
|
+
});
|
|
151
|
+
(0, import_invariant.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
|
|
152
|
+
F: __dxlog_file,
|
|
153
|
+
L: 96,
|
|
154
|
+
S: this,
|
|
155
|
+
A: [
|
|
156
|
+
"this._writeStream",
|
|
157
|
+
"'Cannot write mutations in readonly mode'"
|
|
158
|
+
]
|
|
159
|
+
});
|
|
128
160
|
(0, import_log.log)("write", {
|
|
129
161
|
clientTag: request.clientTag,
|
|
130
|
-
objectCount:
|
|
162
|
+
objectCount: request.batch.objects?.length ?? 0
|
|
131
163
|
}, {
|
|
132
164
|
F: __dxlog_file,
|
|
133
|
-
L:
|
|
165
|
+
L: 98,
|
|
134
166
|
S: this,
|
|
135
167
|
C: (f, a) => f(...a)
|
|
136
168
|
});
|
|
@@ -144,7 +176,7 @@ var DataServiceHost = class {
|
|
|
144
176
|
seq: receipt2.seq
|
|
145
177
|
}, {
|
|
146
178
|
F: __dxlog_file,
|
|
147
|
-
L:
|
|
179
|
+
L: 107,
|
|
148
180
|
S: this,
|
|
149
181
|
C: (f, a) => f(...a)
|
|
150
182
|
});
|
|
@@ -158,24 +190,18 @@ var DataServiceHost = class {
|
|
|
158
190
|
return receipt;
|
|
159
191
|
}
|
|
160
192
|
};
|
|
161
|
-
var createDataMessage = (batch) => {
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
meta: void 0
|
|
174
|
-
};
|
|
175
|
-
})
|
|
176
|
-
}
|
|
177
|
-
};
|
|
178
|
-
};
|
|
193
|
+
var createDataMessage = (batch) => ({
|
|
194
|
+
batch: {
|
|
195
|
+
objects: batch.objects?.map((object) => ({
|
|
196
|
+
...object,
|
|
197
|
+
mutations: object.mutations?.map((mutation) => ({
|
|
198
|
+
...mutation,
|
|
199
|
+
meta: void 0
|
|
200
|
+
})),
|
|
201
|
+
meta: void 0
|
|
202
|
+
}))
|
|
203
|
+
}
|
|
204
|
+
});
|
|
179
205
|
|
|
180
206
|
// packages/core/echo/echo-pipeline/src/db-host/database-host.ts
|
|
181
207
|
var import_echo_db2 = require("@dxos/echo-db");
|
|
@@ -204,15 +230,13 @@ var DatabaseHost = class {
|
|
|
204
230
|
return this._itemDemuxer.createSnapshot();
|
|
205
231
|
}
|
|
206
232
|
createDataServiceHost() {
|
|
207
|
-
|
|
208
|
-
return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
|
|
233
|
+
return new DataServiceHost(this._itemManager, this._itemDemuxer, this._outboundStream ?? void 0);
|
|
209
234
|
}
|
|
210
235
|
};
|
|
211
236
|
|
|
212
237
|
// packages/core/echo/echo-pipeline/src/db-host/snapshot-manager.ts
|
|
213
238
|
var import_async = require("@dxos/async");
|
|
214
239
|
var import_context2 = require("@dxos/context");
|
|
215
|
-
var import_debug = require("@dxos/debug");
|
|
216
240
|
var import_keys = require("@dxos/keys");
|
|
217
241
|
var import_protocols = require("@dxos/protocols");
|
|
218
242
|
var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
|
|
@@ -257,9 +281,6 @@ var SnapshotManager = class SnapshotManager2 {
|
|
|
257
281
|
return import_keys.PublicKey.from(id).toHex();
|
|
258
282
|
}
|
|
259
283
|
};
|
|
260
|
-
_ts_decorate([
|
|
261
|
-
(0, import_debug.timed)(1e4)
|
|
262
|
-
], SnapshotManager.prototype, "load", null);
|
|
263
284
|
SnapshotManager = _ts_decorate([
|
|
264
285
|
(0, import_async.trackLeaks)("open", "close")
|
|
265
286
|
], SnapshotManager);
|
|
@@ -311,8 +332,8 @@ var SnapshotStore = class {
|
|
|
311
332
|
};
|
|
312
333
|
|
|
313
334
|
// packages/core/echo/echo-pipeline/src/db-host/data-service.ts
|
|
314
|
-
var
|
|
315
|
-
var
|
|
335
|
+
var import_debug = require("@dxos/debug");
|
|
336
|
+
var import_invariant2 = require("@dxos/invariant");
|
|
316
337
|
var import_keys2 = require("@dxos/keys");
|
|
317
338
|
var import_log2 = require("@dxos/log");
|
|
318
339
|
var import_util2 = require("@dxos/util");
|
|
@@ -329,11 +350,19 @@ var DataServiceSubscriptions = class {
|
|
|
329
350
|
spaceKey
|
|
330
351
|
}, {
|
|
331
352
|
F: __dxlog_file2,
|
|
332
|
-
L:
|
|
353
|
+
L: 30,
|
|
333
354
|
S: this,
|
|
334
355
|
C: (f, a) => f(...a)
|
|
335
356
|
});
|
|
336
|
-
(0,
|
|
357
|
+
(0, import_invariant2.invariant)(!this._spaces.has(spaceKey), void 0, {
|
|
358
|
+
F: __dxlog_file2,
|
|
359
|
+
L: 31,
|
|
360
|
+
S: this,
|
|
361
|
+
A: [
|
|
362
|
+
"!this._spaces.has(spaceKey)",
|
|
363
|
+
""
|
|
364
|
+
]
|
|
365
|
+
});
|
|
337
366
|
await host.open();
|
|
338
367
|
this._spaces.set(spaceKey, host);
|
|
339
368
|
}
|
|
@@ -342,12 +371,12 @@ var DataServiceSubscriptions = class {
|
|
|
342
371
|
spaceKey
|
|
343
372
|
}, {
|
|
344
373
|
F: __dxlog_file2,
|
|
345
|
-
L:
|
|
374
|
+
L: 37,
|
|
346
375
|
S: this,
|
|
347
376
|
C: (f, a) => f(...a)
|
|
348
377
|
});
|
|
349
378
|
const host = this._spaces.get(spaceKey);
|
|
350
|
-
await
|
|
379
|
+
await host?.close();
|
|
351
380
|
this._spaces.delete(spaceKey);
|
|
352
381
|
}
|
|
353
382
|
getDataService(spaceKey) {
|
|
@@ -359,25 +388,47 @@ var DataServiceImpl = class {
|
|
|
359
388
|
this._subscriptions = _subscriptions;
|
|
360
389
|
}
|
|
361
390
|
subscribe(request) {
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
391
|
+
(0, import_invariant2.invariant)(request.spaceKey, void 0, {
|
|
392
|
+
F: __dxlog_file2,
|
|
393
|
+
L: 56,
|
|
394
|
+
S: this,
|
|
395
|
+
A: [
|
|
396
|
+
"request.spaceKey",
|
|
397
|
+
""
|
|
398
|
+
]
|
|
399
|
+
});
|
|
400
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
365
401
|
return host.subscribe();
|
|
366
402
|
}
|
|
367
403
|
write(request) {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
404
|
+
(0, import_invariant2.invariant)(request.spaceKey, void 0, {
|
|
405
|
+
F: __dxlog_file2,
|
|
406
|
+
L: 63,
|
|
407
|
+
S: this,
|
|
408
|
+
A: [
|
|
409
|
+
"request.spaceKey",
|
|
410
|
+
""
|
|
411
|
+
]
|
|
412
|
+
});
|
|
413
|
+
(0, import_invariant2.invariant)(request.batch, void 0, {
|
|
414
|
+
F: __dxlog_file2,
|
|
415
|
+
L: 64,
|
|
416
|
+
S: this,
|
|
417
|
+
A: [
|
|
418
|
+
"request.batch",
|
|
419
|
+
""
|
|
420
|
+
]
|
|
421
|
+
});
|
|
422
|
+
const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
|
|
372
423
|
return host.write(request);
|
|
373
424
|
}
|
|
374
425
|
};
|
|
375
426
|
|
|
376
427
|
// packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
|
|
377
428
|
var import_crc_32 = __toESM(require("crc-32"));
|
|
378
|
-
var import_tiny_invariant3 = __toESM(require("tiny-invariant"));
|
|
379
429
|
var import_async2 = require("@dxos/async");
|
|
380
430
|
var import_errors = require("@dxos/errors");
|
|
431
|
+
var import_invariant3 = require("@dxos/invariant");
|
|
381
432
|
var import_log3 = require("@dxos/log");
|
|
382
433
|
var import_protocols3 = require("@dxos/protocols");
|
|
383
434
|
var import_services = require("@dxos/protocols/proto/dxos/client/services");
|
|
@@ -396,8 +447,8 @@ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipelin
|
|
|
396
447
|
var emptyEchoMetadata = () => ({
|
|
397
448
|
version: import_protocols3.STORAGE_VERSION,
|
|
398
449
|
spaces: [],
|
|
399
|
-
created: new Date(),
|
|
400
|
-
updated: new Date()
|
|
450
|
+
created: /* @__PURE__ */ new Date(),
|
|
451
|
+
updated: /* @__PURE__ */ new Date()
|
|
401
452
|
});
|
|
402
453
|
var EchoMetadata = import_protocols3.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
|
|
403
454
|
var MetadataStore = class {
|
|
@@ -411,22 +462,19 @@ var MetadataStore = class {
|
|
|
411
462
|
return this._metadata;
|
|
412
463
|
}
|
|
413
464
|
get version() {
|
|
414
|
-
|
|
415
|
-
return (_a = this._metadata.version) != null ? _a : 0;
|
|
465
|
+
return this._metadata.version ?? 0;
|
|
416
466
|
}
|
|
417
467
|
/**
|
|
418
468
|
* Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
|
|
419
469
|
* addSpaceFeed functions.
|
|
420
470
|
*/
|
|
421
471
|
get spaces() {
|
|
422
|
-
|
|
423
|
-
return (_a = this._metadata.spaces) != null ? _a : [];
|
|
472
|
+
return this._metadata.spaces ?? [];
|
|
424
473
|
}
|
|
425
474
|
/**
|
|
426
475
|
* Loads metadata from persistent storage.
|
|
427
476
|
*/
|
|
428
477
|
async load() {
|
|
429
|
-
var _a;
|
|
430
478
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
431
479
|
try {
|
|
432
480
|
const { size: fileLength } = await file.stat();
|
|
@@ -445,7 +493,10 @@ var MetadataStore = class {
|
|
|
445
493
|
C: (f, a) => f(...a)
|
|
446
494
|
});
|
|
447
495
|
if (fileLength < dataSize + 8) {
|
|
448
|
-
throw new import_errors.DataCorruptionError("Metadata size is smaller than expected."
|
|
496
|
+
throw new import_errors.DataCorruptionError("Metadata size is smaller than expected.", {
|
|
497
|
+
fileLength,
|
|
498
|
+
dataSize
|
|
499
|
+
});
|
|
449
500
|
}
|
|
450
501
|
const data = await file.read(8, dataSize);
|
|
451
502
|
const calculatedChecksum = import_crc_32.default.buf(data);
|
|
@@ -453,9 +504,8 @@ var MetadataStore = class {
|
|
|
453
504
|
throw new import_errors.DataCorruptionError("Metadata checksum is invalid.");
|
|
454
505
|
}
|
|
455
506
|
this._metadata = EchoMetadata.decode(data);
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
(_a2 = space.state) != null ? _a2 : space.state = import_services.SpaceState.ACTIVE;
|
|
507
|
+
this._metadata.spaces?.forEach((space) => {
|
|
508
|
+
space.state ??= import_services.SpaceState.ACTIVE;
|
|
459
509
|
});
|
|
460
510
|
} catch (err) {
|
|
461
511
|
import_log3.log.error("failed to load metadata", {
|
|
@@ -472,12 +522,11 @@ var MetadataStore = class {
|
|
|
472
522
|
}
|
|
473
523
|
}
|
|
474
524
|
async _save() {
|
|
475
|
-
var _a;
|
|
476
525
|
const data = {
|
|
477
526
|
...this._metadata,
|
|
478
527
|
version: import_protocols3.STORAGE_VERSION,
|
|
479
|
-
created:
|
|
480
|
-
updated: new Date()
|
|
528
|
+
created: this._metadata.created ?? /* @__PURE__ */ new Date(),
|
|
529
|
+
updated: /* @__PURE__ */ new Date()
|
|
481
530
|
};
|
|
482
531
|
this.update.emit(data);
|
|
483
532
|
const file = this._directory.getOrCreateFile("EchoMetadata");
|
|
@@ -503,12 +552,19 @@ var MetadataStore = class {
|
|
|
503
552
|
}
|
|
504
553
|
}
|
|
505
554
|
_getSpace(spaceKey) {
|
|
506
|
-
|
|
507
|
-
if ((_a = this._metadata.identity) == null ? void 0 : _a.haloSpace.key.equals(spaceKey)) {
|
|
555
|
+
if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
|
|
508
556
|
return this._metadata.identity.haloSpace;
|
|
509
557
|
}
|
|
510
558
|
const space = this.spaces.find((space2) => space2.key === spaceKey);
|
|
511
|
-
(0,
|
|
559
|
+
(0, import_invariant3.invariant)(space, "Space not found", {
|
|
560
|
+
F: __dxlog_file3,
|
|
561
|
+
L: 137,
|
|
562
|
+
S: this,
|
|
563
|
+
A: [
|
|
564
|
+
"space",
|
|
565
|
+
"'Space not found'"
|
|
566
|
+
]
|
|
567
|
+
});
|
|
512
568
|
return space;
|
|
513
569
|
}
|
|
514
570
|
/**
|
|
@@ -528,14 +584,29 @@ var MetadataStore = class {
|
|
|
528
584
|
return this._metadata.identity;
|
|
529
585
|
}
|
|
530
586
|
async setIdentityRecord(record) {
|
|
531
|
-
(0,
|
|
587
|
+
(0, import_invariant3.invariant)(!this._metadata.identity, "Cannot overwrite existing identity in metadata", {
|
|
588
|
+
F: __dxlog_file3,
|
|
589
|
+
L: 155,
|
|
590
|
+
S: this,
|
|
591
|
+
A: [
|
|
592
|
+
"!this._metadata.identity",
|
|
593
|
+
"'Cannot overwrite existing identity in metadata'"
|
|
594
|
+
]
|
|
595
|
+
});
|
|
532
596
|
this._metadata.identity = record;
|
|
533
597
|
await this._save();
|
|
534
598
|
}
|
|
535
599
|
async addSpace(record) {
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
600
|
+
(0, import_invariant3.invariant)(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
|
|
601
|
+
F: __dxlog_file3,
|
|
602
|
+
L: 162,
|
|
603
|
+
S: this,
|
|
604
|
+
A: [
|
|
605
|
+
"!(this._metadata.spaces ?? []).find((space) => space.key === record.key)",
|
|
606
|
+
"'Cannot overwrite existing space in metadata'"
|
|
607
|
+
]
|
|
608
|
+
});
|
|
609
|
+
(this._metadata.spaces ??= []).push(record);
|
|
539
610
|
await this._save();
|
|
540
611
|
}
|
|
541
612
|
async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
|
|
@@ -570,10 +641,10 @@ _ts_decorate2([
|
|
|
570
641
|
var fromBytesInt32 = (buf) => buf.readInt32LE(0);
|
|
571
642
|
|
|
572
643
|
// packages/core/echo/echo-pipeline/src/space/auth.ts
|
|
573
|
-
var import_tiny_invariant4 = __toESM(require("tiny-invariant"));
|
|
574
644
|
var import_async3 = require("@dxos/async");
|
|
575
645
|
var import_context3 = require("@dxos/context");
|
|
576
646
|
var import_crypto2 = require("@dxos/crypto");
|
|
647
|
+
var import_invariant4 = require("@dxos/invariant");
|
|
577
648
|
var import_log4 = require("@dxos/log");
|
|
578
649
|
var import_protocols4 = require("@dxos/protocols");
|
|
579
650
|
var import_teleport = require("@dxos/teleport");
|
|
@@ -594,7 +665,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
594
665
|
onError: (err) => {
|
|
595
666
|
import_log4.log.catch(err, void 0, {
|
|
596
667
|
F: __dxlog_file4,
|
|
597
|
-
L:
|
|
668
|
+
L: 28,
|
|
598
669
|
S: this,
|
|
599
670
|
C: (f, a) => f(...a)
|
|
600
671
|
});
|
|
@@ -616,7 +687,7 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
616
687
|
} catch (err) {
|
|
617
688
|
import_log4.log.error("failed to generate auth credentials", err, {
|
|
618
689
|
F: __dxlog_file4,
|
|
619
|
-
L:
|
|
690
|
+
L: 55,
|
|
620
691
|
S: this,
|
|
621
692
|
C: (f, a) => f(...a)
|
|
622
693
|
});
|
|
@@ -634,14 +705,30 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
634
705
|
const { credential } = await this.rpc.AuthService.authenticate({
|
|
635
706
|
challenge
|
|
636
707
|
});
|
|
637
|
-
(0,
|
|
708
|
+
(0, import_invariant4.invariant)(credential?.length > 0, "invalid credential", {
|
|
709
|
+
F: __dxlog_file4,
|
|
710
|
+
L: 69,
|
|
711
|
+
S: this,
|
|
712
|
+
A: [
|
|
713
|
+
"credential?.length > 0",
|
|
714
|
+
"'invalid credential'"
|
|
715
|
+
]
|
|
716
|
+
});
|
|
638
717
|
const success = await this._authParams.verifier(challenge, credential);
|
|
639
|
-
(0,
|
|
718
|
+
(0, import_invariant4.invariant)(success, "credential not verified", {
|
|
719
|
+
F: __dxlog_file4,
|
|
720
|
+
L: 71,
|
|
721
|
+
S: this,
|
|
722
|
+
A: [
|
|
723
|
+
"success",
|
|
724
|
+
"'credential not verified'"
|
|
725
|
+
]
|
|
726
|
+
});
|
|
640
727
|
(0, import_async3.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
|
|
641
728
|
} catch (err) {
|
|
642
729
|
(0, import_log4.log)("auth failed", err, {
|
|
643
730
|
F: __dxlog_file4,
|
|
644
|
-
L:
|
|
731
|
+
L: 74,
|
|
645
732
|
S: this,
|
|
646
733
|
C: (f, a) => f(...a)
|
|
647
734
|
});
|
|
@@ -657,26 +744,29 @@ var AuthExtension = class extends import_teleport.RpcExtension {
|
|
|
657
744
|
};
|
|
658
745
|
|
|
659
746
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
660
|
-
var import_tiny_invariant9 = __toESM(require("tiny-invariant"));
|
|
661
747
|
var import_async7 = require("@dxos/async");
|
|
748
|
+
var import_invariant9 = require("@dxos/invariant");
|
|
662
749
|
var import_log10 = require("@dxos/log");
|
|
663
750
|
var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
751
|
+
var import_tracing3 = require("@dxos/tracing");
|
|
664
752
|
var import_util7 = require("@dxos/util");
|
|
665
753
|
|
|
666
754
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
755
|
+
var import_context5 = require("@dxos/context");
|
|
667
756
|
var import_credentials = require("@dxos/credentials");
|
|
668
757
|
var import_keys4 = require("@dxos/keys");
|
|
669
758
|
var import_log8 = require("@dxos/log");
|
|
670
759
|
var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
|
|
671
760
|
var import_timeframe3 = require("@dxos/timeframe");
|
|
761
|
+
var import_tracing = require("@dxos/tracing");
|
|
672
762
|
var import_util5 = require("@dxos/util");
|
|
673
763
|
|
|
674
764
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
675
|
-
var import_tiny_invariant7 = __toESM(require("tiny-invariant"));
|
|
676
765
|
var import_async5 = require("@dxos/async");
|
|
677
766
|
var import_context4 = require("@dxos/context");
|
|
678
|
-
var
|
|
767
|
+
var import_debug3 = require("@dxos/debug");
|
|
679
768
|
var import_feed_store = require("@dxos/feed-store");
|
|
769
|
+
var import_invariant7 = require("@dxos/invariant");
|
|
680
770
|
var import_keys3 = require("@dxos/keys");
|
|
681
771
|
var import_log7 = require("@dxos/log");
|
|
682
772
|
var import_timeframe2 = require("@dxos/timeframe");
|
|
@@ -689,31 +779,56 @@ var codec = import_protocols5.schema.getCodecForType("dxos.echo.feed.FeedMessage
|
|
|
689
779
|
var valueEncoding = (0, import_hypercore.createCodecEncoding)(codec);
|
|
690
780
|
|
|
691
781
|
// packages/core/echo/echo-pipeline/src/common/feeds.ts
|
|
692
|
-
var
|
|
782
|
+
var import_invariant5 = require("@dxos/invariant");
|
|
783
|
+
var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/common/feeds.ts";
|
|
693
784
|
var createMappedFeedWriter = (mapper, writer) => {
|
|
694
|
-
(0,
|
|
695
|
-
|
|
785
|
+
(0, import_invariant5.invariant)(mapper, void 0, {
|
|
786
|
+
F: __dxlog_file5,
|
|
787
|
+
L: 16,
|
|
788
|
+
S: void 0,
|
|
789
|
+
A: [
|
|
790
|
+
"mapper",
|
|
791
|
+
""
|
|
792
|
+
]
|
|
793
|
+
});
|
|
794
|
+
(0, import_invariant5.invariant)(writer, void 0, {
|
|
795
|
+
F: __dxlog_file5,
|
|
796
|
+
L: 17,
|
|
797
|
+
S: void 0,
|
|
798
|
+
A: [
|
|
799
|
+
"writer",
|
|
800
|
+
""
|
|
801
|
+
]
|
|
802
|
+
});
|
|
696
803
|
return {
|
|
697
804
|
write: async (data, options) => await writer.write(await mapper(data), options)
|
|
698
805
|
};
|
|
699
806
|
};
|
|
700
807
|
|
|
701
808
|
// packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
|
|
702
|
-
var
|
|
809
|
+
var import_invariant6 = require("@dxos/invariant");
|
|
703
810
|
var import_log5 = require("@dxos/log");
|
|
704
|
-
var
|
|
811
|
+
var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
|
|
705
812
|
var createMessageSelector = (timeframeClock) => {
|
|
706
813
|
return (messages) => {
|
|
707
814
|
for (let i = 0; i < messages.length; i++) {
|
|
708
815
|
const { data: { timeframe } } = messages[i];
|
|
709
|
-
(0,
|
|
816
|
+
(0, import_invariant6.invariant)(timeframe, void 0, {
|
|
817
|
+
F: __dxlog_file6,
|
|
818
|
+
L: 25,
|
|
819
|
+
S: void 0,
|
|
820
|
+
A: [
|
|
821
|
+
"timeframe",
|
|
822
|
+
""
|
|
823
|
+
]
|
|
824
|
+
});
|
|
710
825
|
if (!timeframeClock.hasGaps(timeframe)) {
|
|
711
826
|
return i;
|
|
712
827
|
}
|
|
713
828
|
}
|
|
714
829
|
(0, import_log5.log)("Skipping...", void 0, {
|
|
715
|
-
F:
|
|
716
|
-
L:
|
|
830
|
+
F: __dxlog_file6,
|
|
831
|
+
L: 33,
|
|
717
832
|
S: void 0,
|
|
718
833
|
C: (f, a) => f(...a)
|
|
719
834
|
});
|
|
@@ -722,7 +837,7 @@ var createMessageSelector = (timeframeClock) => {
|
|
|
722
837
|
|
|
723
838
|
// packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
|
|
724
839
|
var import_async4 = require("@dxos/async");
|
|
725
|
-
var
|
|
840
|
+
var import_debug2 = require("@dxos/debug");
|
|
726
841
|
var import_log6 = require("@dxos/log");
|
|
727
842
|
var import_timeframe = require("@dxos/timeframe");
|
|
728
843
|
function _ts_decorate3(decorators, target, key, desc) {
|
|
@@ -735,7 +850,7 @@ function _ts_decorate3(decorators, target, key, desc) {
|
|
|
735
850
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
736
851
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
737
852
|
}
|
|
738
|
-
var
|
|
853
|
+
var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
|
|
739
854
|
var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
|
|
740
855
|
feedKey,
|
|
741
856
|
index
|
|
@@ -790,7 +905,7 @@ var TimeframeClock = class {
|
|
|
790
905
|
target,
|
|
791
906
|
current: this._timeframe
|
|
792
907
|
}, {
|
|
793
|
-
F:
|
|
908
|
+
F: __dxlog_file7,
|
|
794
909
|
L: 73,
|
|
795
910
|
S: this,
|
|
796
911
|
C: (f, a) => f(...a)
|
|
@@ -801,7 +916,7 @@ var TimeframeClock = class {
|
|
|
801
916
|
current: this._timeframe,
|
|
802
917
|
deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
|
|
803
918
|
}, {
|
|
804
|
-
F:
|
|
919
|
+
F: __dxlog_file7,
|
|
805
920
|
L: 75,
|
|
806
921
|
S: this,
|
|
807
922
|
C: (f, a) => f(...a)
|
|
@@ -811,7 +926,7 @@ var TimeframeClock = class {
|
|
|
811
926
|
}
|
|
812
927
|
};
|
|
813
928
|
_ts_decorate3([
|
|
814
|
-
(0,
|
|
929
|
+
(0, import_debug2.timed)(5e3)
|
|
815
930
|
], TimeframeClock.prototype, "waitUntilReached", null);
|
|
816
931
|
|
|
817
932
|
// packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
|
|
@@ -825,7 +940,7 @@ function _ts_decorate4(decorators, target, key, desc) {
|
|
|
825
940
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
826
941
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
827
942
|
}
|
|
828
|
-
var
|
|
943
|
+
var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
|
|
829
944
|
var PipelineState = class {
|
|
830
945
|
// prettier-ignore
|
|
831
946
|
constructor(_feeds, _timeframeClock) {
|
|
@@ -876,18 +991,17 @@ var PipelineState = class {
|
|
|
876
991
|
* @param timeout Timeout in milliseconds to specify the maximum wait time.
|
|
877
992
|
*/
|
|
878
993
|
async waitUntilReachedTargetTimeframe({ ctx = new import_context4.Context(), timeout, breakOnStall = true } = {}) {
|
|
879
|
-
var _a;
|
|
880
994
|
(0, import_log7.log)("waitUntilReachedTargetTimeframe", {
|
|
881
995
|
timeout,
|
|
882
996
|
current: this.timeframe,
|
|
883
997
|
target: this.targetTimeframe
|
|
884
998
|
}, {
|
|
885
|
-
F:
|
|
886
|
-
L:
|
|
999
|
+
F: __dxlog_file8,
|
|
1000
|
+
L: 125,
|
|
887
1001
|
S: this,
|
|
888
1002
|
C: (f, a) => f(...a)
|
|
889
1003
|
});
|
|
890
|
-
|
|
1004
|
+
this._reachedTargetPromise ??= Promise.race([
|
|
891
1005
|
this._timeframeClock.update.waitForCondition(() => {
|
|
892
1006
|
return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
|
|
893
1007
|
}),
|
|
@@ -913,8 +1027,8 @@ var PipelineState = class {
|
|
|
913
1027
|
target: this.targetTimeframe,
|
|
914
1028
|
dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
|
|
915
1029
|
}, {
|
|
916
|
-
F:
|
|
917
|
-
L:
|
|
1030
|
+
F: __dxlog_file8,
|
|
1031
|
+
L: 152,
|
|
918
1032
|
S: this,
|
|
919
1033
|
C: (f, a) => f(...a)
|
|
920
1034
|
});
|
|
@@ -934,16 +1048,26 @@ var Pipeline = class {
|
|
|
934
1048
|
// Waits for the message consumer to process the message and yield control back to the pipeline.
|
|
935
1049
|
this._processingTrigger = new import_async5.Trigger().wake();
|
|
936
1050
|
this._pauseTrigger = new import_async5.Trigger().wake();
|
|
1051
|
+
// Pending downloads.
|
|
1052
|
+
this._downloads = new import_util4.ComplexMap((value) => import_keys3.PublicKey.hash(value.key));
|
|
937
1053
|
this._isStopping = false;
|
|
938
1054
|
this._isStarted = false;
|
|
939
|
-
this.
|
|
1055
|
+
this._isBeingConsumed = false;
|
|
940
1056
|
this._isPaused = false;
|
|
941
1057
|
}
|
|
942
1058
|
get state() {
|
|
943
1059
|
return this._state;
|
|
944
1060
|
}
|
|
945
1061
|
get writer() {
|
|
946
|
-
(0,
|
|
1062
|
+
(0, import_invariant7.invariant)(this._writer, "Writer not set.", {
|
|
1063
|
+
F: __dxlog_file8,
|
|
1064
|
+
L: 233,
|
|
1065
|
+
S: this,
|
|
1066
|
+
A: [
|
|
1067
|
+
"this._writer",
|
|
1068
|
+
"'Writer not set.'"
|
|
1069
|
+
]
|
|
1070
|
+
});
|
|
947
1071
|
return this._writer;
|
|
948
1072
|
}
|
|
949
1073
|
hasFeed(feedKey) {
|
|
@@ -959,20 +1083,47 @@ var Pipeline = class {
|
|
|
959
1083
|
if (this._feedSetIterator) {
|
|
960
1084
|
await this._feedSetIterator.addFeed(feed);
|
|
961
1085
|
}
|
|
962
|
-
this.
|
|
1086
|
+
if (this._isStarted && !this._isPaused) {
|
|
1087
|
+
this._setFeedDownloadState(feed);
|
|
1088
|
+
}
|
|
963
1089
|
}
|
|
964
1090
|
setWriteFeed(feed) {
|
|
965
|
-
(0,
|
|
966
|
-
|
|
1091
|
+
(0, import_invariant7.invariant)(!this._writer, "Writer already set.", {
|
|
1092
|
+
F: __dxlog_file8,
|
|
1093
|
+
L: 260,
|
|
1094
|
+
S: this,
|
|
1095
|
+
A: [
|
|
1096
|
+
"!this._writer",
|
|
1097
|
+
"'Writer already set.'"
|
|
1098
|
+
]
|
|
1099
|
+
});
|
|
1100
|
+
(0, import_invariant7.invariant)(feed.properties.writable, "Feed must be writable.", {
|
|
1101
|
+
F: __dxlog_file8,
|
|
1102
|
+
L: 261,
|
|
1103
|
+
S: this,
|
|
1104
|
+
A: [
|
|
1105
|
+
"feed.properties.writable",
|
|
1106
|
+
"'Feed must be writable.'"
|
|
1107
|
+
]
|
|
1108
|
+
});
|
|
967
1109
|
this._writer = createMappedFeedWriter((payload) => ({
|
|
968
1110
|
timeframe: this._timeframeClock.timeframe,
|
|
969
1111
|
payload
|
|
970
1112
|
}), feed.createFeedWriter());
|
|
971
1113
|
}
|
|
972
1114
|
async start() {
|
|
1115
|
+
(0, import_invariant7.invariant)(!this._isStarted, "Pipeline is already started.", {
|
|
1116
|
+
F: __dxlog_file8,
|
|
1117
|
+
L: 274,
|
|
1118
|
+
S: this,
|
|
1119
|
+
A: [
|
|
1120
|
+
"!this._isStarted",
|
|
1121
|
+
"'Pipeline is already started.'"
|
|
1122
|
+
]
|
|
1123
|
+
});
|
|
973
1124
|
(0, import_log7.log)("starting...", void 0, {
|
|
974
|
-
F:
|
|
975
|
-
L:
|
|
1125
|
+
F: __dxlog_file8,
|
|
1126
|
+
L: 275,
|
|
976
1127
|
S: this,
|
|
977
1128
|
C: (f, a) => f(...a)
|
|
978
1129
|
});
|
|
@@ -980,30 +1131,38 @@ var Pipeline = class {
|
|
|
980
1131
|
await this._feedSetIterator.open();
|
|
981
1132
|
this._isStarted = true;
|
|
982
1133
|
(0, import_log7.log)("started", void 0, {
|
|
983
|
-
F:
|
|
984
|
-
L:
|
|
1134
|
+
F: __dxlog_file8,
|
|
1135
|
+
L: 279,
|
|
985
1136
|
S: this,
|
|
986
1137
|
C: (f, a) => f(...a)
|
|
987
1138
|
});
|
|
1139
|
+
if (!this._isPaused) {
|
|
1140
|
+
for (const feed of this._feeds.values()) {
|
|
1141
|
+
this._setFeedDownloadState(feed);
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
988
1144
|
}
|
|
989
1145
|
async stop() {
|
|
990
|
-
var _a;
|
|
991
1146
|
(0, import_log7.log)("stopping...", void 0, {
|
|
992
|
-
F:
|
|
993
|
-
L:
|
|
1147
|
+
F: __dxlog_file8,
|
|
1148
|
+
L: 290,
|
|
994
1149
|
S: this,
|
|
995
1150
|
C: (f, a) => f(...a)
|
|
996
1151
|
});
|
|
997
1152
|
this._isStopping = true;
|
|
998
|
-
|
|
1153
|
+
for (const [feed, handle] of this._downloads.entries()) {
|
|
1154
|
+
feed.undownload(handle);
|
|
1155
|
+
}
|
|
1156
|
+
this._downloads.clear();
|
|
1157
|
+
await this._feedSetIterator?.close();
|
|
999
1158
|
await this._processingTrigger.wait();
|
|
1000
1159
|
await this._state._ctx.dispose();
|
|
1001
1160
|
this._state._ctx = new import_context4.Context();
|
|
1002
1161
|
this._state._reachedTargetPromise = void 0;
|
|
1003
1162
|
this._isStarted = false;
|
|
1004
1163
|
(0, import_log7.log)("stopped", void 0, {
|
|
1005
|
-
F:
|
|
1006
|
-
L:
|
|
1164
|
+
F: __dxlog_file8,
|
|
1165
|
+
L: 302,
|
|
1007
1166
|
S: this,
|
|
1008
1167
|
C: (f, a) => f(...a)
|
|
1009
1168
|
});
|
|
@@ -1013,12 +1172,17 @@ var Pipeline = class {
|
|
|
1013
1172
|
* The pipeline will start processing messages AFTER this timeframe.
|
|
1014
1173
|
*/
|
|
1015
1174
|
async setCursor(timeframe) {
|
|
1016
|
-
(0,
|
|
1175
|
+
(0, import_invariant7.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
|
|
1176
|
+
F: __dxlog_file8,
|
|
1177
|
+
L: 311,
|
|
1178
|
+
S: this,
|
|
1179
|
+
A: [
|
|
1180
|
+
"!this._isStarted || this._isPaused",
|
|
1181
|
+
"'Invalid state.'"
|
|
1182
|
+
]
|
|
1183
|
+
});
|
|
1017
1184
|
this._state._startTimeframe = timeframe;
|
|
1018
1185
|
this._timeframeClock.setTimeframe(timeframe);
|
|
1019
|
-
for (const feed of this._feeds.values()) {
|
|
1020
|
-
this._setFeedDownloadState(feed);
|
|
1021
|
-
}
|
|
1022
1186
|
if (this._feedSetIterator) {
|
|
1023
1187
|
await this._feedSetIterator.close();
|
|
1024
1188
|
await this._initIterator();
|
|
@@ -1029,7 +1193,6 @@ var Pipeline = class {
|
|
|
1029
1193
|
* Calling pause while processing will cause a deadlock.
|
|
1030
1194
|
*/
|
|
1031
1195
|
async pause() {
|
|
1032
|
-
(0, import_tiny_invariant7.default)(this._isStarted, "Pipeline is not open.");
|
|
1033
1196
|
if (this._isPaused) {
|
|
1034
1197
|
return;
|
|
1035
1198
|
}
|
|
@@ -1038,31 +1201,65 @@ var Pipeline = class {
|
|
|
1038
1201
|
this._isPaused = true;
|
|
1039
1202
|
}
|
|
1040
1203
|
async unpause() {
|
|
1041
|
-
(0,
|
|
1042
|
-
|
|
1204
|
+
(0, import_invariant7.invariant)(this._isPaused, "Pipeline is not paused.", {
|
|
1205
|
+
F: __dxlog_file8,
|
|
1206
|
+
L: 340,
|
|
1207
|
+
S: this,
|
|
1208
|
+
A: [
|
|
1209
|
+
"this._isPaused",
|
|
1210
|
+
"'Pipeline is not paused.'"
|
|
1211
|
+
]
|
|
1212
|
+
});
|
|
1043
1213
|
this._pauseTrigger.wake();
|
|
1044
1214
|
this._isPaused = false;
|
|
1215
|
+
for (const feed of this._feeds.values()) {
|
|
1216
|
+
this._setFeedDownloadState(feed);
|
|
1217
|
+
}
|
|
1045
1218
|
}
|
|
1046
1219
|
/**
|
|
1047
1220
|
* Starts to iterate over the ordered messages from the added feeds.
|
|
1048
1221
|
* Updates the timeframe clock after the message has bee processed.
|
|
1049
1222
|
*/
|
|
1050
1223
|
async *consume() {
|
|
1051
|
-
(0,
|
|
1052
|
-
|
|
1053
|
-
|
|
1224
|
+
(0, import_invariant7.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
|
|
1225
|
+
F: __dxlog_file8,
|
|
1226
|
+
L: 355,
|
|
1227
|
+
S: this,
|
|
1228
|
+
A: [
|
|
1229
|
+
"!this._isBeingConsumed",
|
|
1230
|
+
"'Pipeline is already being consumed.'"
|
|
1231
|
+
]
|
|
1232
|
+
});
|
|
1233
|
+
this._isBeingConsumed = true;
|
|
1234
|
+
(0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1235
|
+
F: __dxlog_file8,
|
|
1236
|
+
L: 358,
|
|
1237
|
+
S: this,
|
|
1238
|
+
A: [
|
|
1239
|
+
"this._feedSetIterator",
|
|
1240
|
+
"'Iterator not initialized.'"
|
|
1241
|
+
]
|
|
1242
|
+
});
|
|
1054
1243
|
let lastFeedSetIterator = this._feedSetIterator;
|
|
1055
1244
|
let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
1056
1245
|
while (!this._isStopping) {
|
|
1057
1246
|
await this._pauseTrigger.wait();
|
|
1058
1247
|
if (lastFeedSetIterator !== this._feedSetIterator) {
|
|
1059
|
-
(0,
|
|
1248
|
+
(0, import_invariant7.invariant)(this._feedSetIterator, "Iterator not initialized.", {
|
|
1249
|
+
F: __dxlog_file8,
|
|
1250
|
+
L: 367,
|
|
1251
|
+
S: this,
|
|
1252
|
+
A: [
|
|
1253
|
+
"this._feedSetIterator",
|
|
1254
|
+
"'Iterator not initialized.'"
|
|
1255
|
+
]
|
|
1256
|
+
});
|
|
1060
1257
|
lastFeedSetIterator = this._feedSetIterator;
|
|
1061
1258
|
iterable = lastFeedSetIterator[Symbol.asyncIterator]();
|
|
1062
1259
|
}
|
|
1063
1260
|
const { done, value } = await iterable.next();
|
|
1064
1261
|
if (!done) {
|
|
1065
|
-
const block = value
|
|
1262
|
+
const block = value ?? (0, import_debug3.failUndefined)();
|
|
1066
1263
|
this._processingTrigger.reset();
|
|
1067
1264
|
this._timeframeClock.updatePendingTimeframe(import_keys3.PublicKey.from(block.feedKey), block.seq);
|
|
1068
1265
|
yield block;
|
|
@@ -1070,33 +1267,40 @@ var Pipeline = class {
|
|
|
1070
1267
|
this._timeframeClock.updateTimeframe();
|
|
1071
1268
|
}
|
|
1072
1269
|
}
|
|
1073
|
-
this.
|
|
1270
|
+
this._isBeingConsumed = false;
|
|
1074
1271
|
}
|
|
1075
1272
|
_setFeedDownloadState(feed) {
|
|
1076
|
-
|
|
1273
|
+
let handle = this._downloads.get(feed);
|
|
1274
|
+
if (handle) {
|
|
1275
|
+
feed.undownload(handle);
|
|
1276
|
+
}
|
|
1077
1277
|
const timeframe = this._state._startTimeframe;
|
|
1078
|
-
const seq =
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1278
|
+
const seq = timeframe.get(feed.key) ?? -1;
|
|
1279
|
+
import_log7.log.info("download", {
|
|
1280
|
+
feed: feed.key.truncate(),
|
|
1281
|
+
seq,
|
|
1282
|
+
length: feed.length
|
|
1283
|
+
}, {
|
|
1284
|
+
F: __dxlog_file8,
|
|
1285
|
+
L: 396,
|
|
1286
|
+
S: this,
|
|
1287
|
+
C: (f, a) => f(...a)
|
|
1086
1288
|
});
|
|
1087
|
-
feed.download({
|
|
1289
|
+
handle = feed.download({
|
|
1088
1290
|
start: seq + 1,
|
|
1089
1291
|
linear: true
|
|
1090
|
-
}
|
|
1091
|
-
(
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1292
|
+
}, (err, data) => {
|
|
1293
|
+
if (err) {
|
|
1294
|
+
} else {
|
|
1295
|
+
import_log7.log.info("data", data, {
|
|
1296
|
+
F: __dxlog_file8,
|
|
1297
|
+
L: 401,
|
|
1298
|
+
S: this,
|
|
1299
|
+
C: (f, a) => f(...a)
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1099
1302
|
});
|
|
1303
|
+
this._downloads.set(feed, handle);
|
|
1100
1304
|
}
|
|
1101
1305
|
async _initIterator() {
|
|
1102
1306
|
this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
|
|
@@ -1105,8 +1309,8 @@ var Pipeline = class {
|
|
|
1105
1309
|
});
|
|
1106
1310
|
this._feedSetIterator.stalled.on((iterator) => {
|
|
1107
1311
|
import_log7.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
|
|
1108
|
-
F:
|
|
1109
|
-
L:
|
|
1312
|
+
F: __dxlog_file8,
|
|
1313
|
+
L: 415,
|
|
1110
1314
|
S: this,
|
|
1111
1315
|
C: (f, a) => f(...a)
|
|
1112
1316
|
});
|
|
@@ -1134,12 +1338,24 @@ _ts_decorate4([
|
|
|
1134
1338
|
], Pipeline.prototype, "unpause", null);
|
|
1135
1339
|
|
|
1136
1340
|
// packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
|
|
1137
|
-
|
|
1341
|
+
function _ts_decorate5(decorators, target, key, desc) {
|
|
1342
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1343
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1344
|
+
r = Reflect.decorate(decorators, target, key, desc);
|
|
1345
|
+
else
|
|
1346
|
+
for (var i = decorators.length - 1; i >= 0; i--)
|
|
1347
|
+
if (d = decorators[i])
|
|
1348
|
+
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1349
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1350
|
+
}
|
|
1351
|
+
var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
|
|
1138
1352
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
|
|
1139
|
-
var ControlPipeline = class {
|
|
1353
|
+
var ControlPipeline = class ControlPipeline2 {
|
|
1140
1354
|
constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
|
|
1141
1355
|
this._lastTimeframeSaveTime = Date.now();
|
|
1142
1356
|
this.onFeedAdmitted = new import_util5.Callback();
|
|
1357
|
+
this._usage = new import_tracing.TimeUsageCounter();
|
|
1358
|
+
this._mutations = new import_tracing.TimeSeriesCounter();
|
|
1143
1359
|
this._spaceKey = spaceKey;
|
|
1144
1360
|
this._metadata = metadataStore;
|
|
1145
1361
|
this._pipeline = new Pipeline();
|
|
@@ -1149,8 +1365,8 @@ var ControlPipeline = class {
|
|
|
1149
1365
|
(0, import_log8.log)("feed admitted", {
|
|
1150
1366
|
key: info.key
|
|
1151
1367
|
}, {
|
|
1152
|
-
F:
|
|
1153
|
-
L:
|
|
1368
|
+
F: __dxlog_file9,
|
|
1369
|
+
L: 61,
|
|
1154
1370
|
S: this,
|
|
1155
1371
|
C: (f, a) => f(...a)
|
|
1156
1372
|
});
|
|
@@ -1160,8 +1376,8 @@ var ControlPipeline = class {
|
|
|
1160
1376
|
await this._pipeline.addFeed(feed);
|
|
1161
1377
|
} catch (err) {
|
|
1162
1378
|
import_log8.log.catch(err, void 0, {
|
|
1163
|
-
F:
|
|
1164
|
-
L:
|
|
1379
|
+
F: __dxlog_file9,
|
|
1380
|
+
L: 69,
|
|
1165
1381
|
S: this,
|
|
1166
1382
|
C: (f, a) => f(...a)
|
|
1167
1383
|
});
|
|
@@ -1183,57 +1399,66 @@ var ControlPipeline = class {
|
|
|
1183
1399
|
}
|
|
1184
1400
|
async start() {
|
|
1185
1401
|
(0, import_log8.log)("starting...", void 0, {
|
|
1186
|
-
F:
|
|
1187
|
-
L:
|
|
1402
|
+
F: __dxlog_file9,
|
|
1403
|
+
L: 93,
|
|
1188
1404
|
S: this,
|
|
1189
1405
|
C: (f, a) => f(...a)
|
|
1190
1406
|
});
|
|
1191
1407
|
setTimeout(async () => {
|
|
1192
|
-
|
|
1193
|
-
try {
|
|
1194
|
-
(0, import_log8.log)("processing", {
|
|
1195
|
-
key: msg.feedKey,
|
|
1196
|
-
seq: msg.seq
|
|
1197
|
-
}, {
|
|
1198
|
-
F: __dxlog_file8,
|
|
1199
|
-
L: 88,
|
|
1200
|
-
S: this,
|
|
1201
|
-
C: (f, a) => f(...a)
|
|
1202
|
-
});
|
|
1203
|
-
if (msg.data.payload.credential) {
|
|
1204
|
-
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1205
|
-
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys4.PublicKey.from(msg.feedKey));
|
|
1206
|
-
timer.end();
|
|
1207
|
-
if (!result) {
|
|
1208
|
-
import_log8.log.warn("processing failed", {
|
|
1209
|
-
msg
|
|
1210
|
-
}, {
|
|
1211
|
-
F: __dxlog_file8,
|
|
1212
|
-
L: 98,
|
|
1213
|
-
S: this,
|
|
1214
|
-
C: (f, a) => f(...a)
|
|
1215
|
-
});
|
|
1216
|
-
} else {
|
|
1217
|
-
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1218
|
-
}
|
|
1219
|
-
}
|
|
1220
|
-
} catch (err) {
|
|
1221
|
-
import_log8.log.catch(err, void 0, {
|
|
1222
|
-
F: __dxlog_file8,
|
|
1223
|
-
L: 104,
|
|
1224
|
-
S: this,
|
|
1225
|
-
C: (f, a) => f(...a)
|
|
1226
|
-
});
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1408
|
+
void this._consumePipeline(new import_context5.Context());
|
|
1229
1409
|
});
|
|
1230
1410
|
await this._pipeline.start();
|
|
1231
1411
|
(0, import_log8.log)("started", void 0, {
|
|
1232
|
-
F:
|
|
1233
|
-
L:
|
|
1412
|
+
F: __dxlog_file9,
|
|
1413
|
+
L: 99,
|
|
1414
|
+
S: this,
|
|
1415
|
+
C: (f, a) => f(...a)
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
async _consumePipeline(ctx) {
|
|
1419
|
+
for await (const msg of this._pipeline.consume()) {
|
|
1420
|
+
const span = this._usage.beginRecording();
|
|
1421
|
+
this._mutations.inc();
|
|
1422
|
+
try {
|
|
1423
|
+
await this._processMessage(ctx, msg);
|
|
1424
|
+
} catch (err) {
|
|
1425
|
+
import_log8.log.catch(err, void 0, {
|
|
1426
|
+
F: __dxlog_file9,
|
|
1427
|
+
L: 111,
|
|
1428
|
+
S: this,
|
|
1429
|
+
C: (f, a) => f(...a)
|
|
1430
|
+
});
|
|
1431
|
+
}
|
|
1432
|
+
span.end();
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
async _processMessage(ctx, msg) {
|
|
1436
|
+
(0, import_log8.log)("processing", {
|
|
1437
|
+
key: msg.feedKey,
|
|
1438
|
+
seq: msg.seq
|
|
1439
|
+
}, {
|
|
1440
|
+
F: __dxlog_file9,
|
|
1441
|
+
L: 121,
|
|
1234
1442
|
S: this,
|
|
1235
1443
|
C: (f, a) => f(...a)
|
|
1236
1444
|
});
|
|
1445
|
+
if (msg.data.payload.credential) {
|
|
1446
|
+
const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
|
|
1447
|
+
const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, import_keys4.PublicKey.from(msg.feedKey));
|
|
1448
|
+
timer.end();
|
|
1449
|
+
if (!result) {
|
|
1450
|
+
import_log8.log.warn("processing failed", {
|
|
1451
|
+
msg
|
|
1452
|
+
}, {
|
|
1453
|
+
F: __dxlog_file9,
|
|
1454
|
+
L: 131,
|
|
1455
|
+
S: this,
|
|
1456
|
+
C: (f, a) => f(...a)
|
|
1457
|
+
});
|
|
1458
|
+
} else {
|
|
1459
|
+
await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1237
1462
|
}
|
|
1238
1463
|
async _noteTargetStateIfNeeded(timeframe) {
|
|
1239
1464
|
if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
|
|
@@ -1243,48 +1468,63 @@ var ControlPipeline = class {
|
|
|
1243
1468
|
}
|
|
1244
1469
|
async stop() {
|
|
1245
1470
|
(0, import_log8.log)("stopping...", void 0, {
|
|
1246
|
-
F:
|
|
1247
|
-
L:
|
|
1471
|
+
F: __dxlog_file9,
|
|
1472
|
+
L: 149,
|
|
1248
1473
|
S: this,
|
|
1249
1474
|
C: (f, a) => f(...a)
|
|
1250
1475
|
});
|
|
1251
1476
|
await this._pipeline.stop();
|
|
1252
1477
|
await this._saveTargetTimeframe(this._pipeline.state.timeframe);
|
|
1253
1478
|
(0, import_log8.log)("stopped", void 0, {
|
|
1254
|
-
F:
|
|
1255
|
-
L:
|
|
1479
|
+
F: __dxlog_file9,
|
|
1480
|
+
L: 152,
|
|
1256
1481
|
S: this,
|
|
1257
1482
|
C: (f, a) => f(...a)
|
|
1258
1483
|
});
|
|
1259
1484
|
}
|
|
1260
1485
|
async _saveTargetTimeframe(timeframe) {
|
|
1261
|
-
var _a;
|
|
1262
1486
|
try {
|
|
1263
|
-
const newTimeframe = import_timeframe3.Timeframe.merge(
|
|
1487
|
+
const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
|
|
1264
1488
|
await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
|
|
1265
1489
|
this._targetTimeframe = newTimeframe;
|
|
1266
1490
|
} catch (err) {
|
|
1267
1491
|
(0, import_log8.log)(err, void 0, {
|
|
1268
|
-
F:
|
|
1269
|
-
L:
|
|
1492
|
+
F: __dxlog_file9,
|
|
1493
|
+
L: 161,
|
|
1270
1494
|
S: this,
|
|
1271
1495
|
C: (f, a) => f(...a)
|
|
1272
1496
|
});
|
|
1273
1497
|
}
|
|
1274
1498
|
}
|
|
1275
1499
|
};
|
|
1500
|
+
_ts_decorate5([
|
|
1501
|
+
import_tracing.trace.metricsCounter()
|
|
1502
|
+
], ControlPipeline.prototype, "_usage", void 0);
|
|
1503
|
+
_ts_decorate5([
|
|
1504
|
+
import_tracing.trace.metricsCounter()
|
|
1505
|
+
], ControlPipeline.prototype, "_mutations", void 0);
|
|
1506
|
+
_ts_decorate5([
|
|
1507
|
+
import_tracing.trace.span()
|
|
1508
|
+
], ControlPipeline.prototype, "_consumePipeline", null);
|
|
1509
|
+
_ts_decorate5([
|
|
1510
|
+
import_tracing.trace.span()
|
|
1511
|
+
], ControlPipeline.prototype, "_processMessage", null);
|
|
1512
|
+
ControlPipeline = _ts_decorate5([
|
|
1513
|
+
import_tracing.trace.resource()
|
|
1514
|
+
], ControlPipeline);
|
|
1276
1515
|
|
|
1277
1516
|
// packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
|
|
1278
|
-
var import_tiny_invariant8 = __toESM(require("tiny-invariant"));
|
|
1279
1517
|
var import_async6 = require("@dxos/async");
|
|
1280
|
-
var
|
|
1518
|
+
var import_context6 = require("@dxos/context");
|
|
1281
1519
|
var import_credentials3 = require("@dxos/credentials");
|
|
1282
1520
|
var import_echo_db3 = require("@dxos/echo-db");
|
|
1283
1521
|
var import_errors2 = require("@dxos/errors");
|
|
1522
|
+
var import_invariant8 = require("@dxos/invariant");
|
|
1284
1523
|
var import_log9 = require("@dxos/log");
|
|
1285
1524
|
var import_timeframe4 = require("@dxos/timeframe");
|
|
1525
|
+
var import_tracing2 = require("@dxos/tracing");
|
|
1286
1526
|
var import_util6 = require("@dxos/util");
|
|
1287
|
-
function
|
|
1527
|
+
function _ts_decorate6(decorators, target, key, desc) {
|
|
1288
1528
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1289
1529
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1290
1530
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1294,14 +1534,14 @@ function _ts_decorate5(decorators, target, key, desc) {
|
|
|
1294
1534
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1295
1535
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1296
1536
|
}
|
|
1297
|
-
var
|
|
1537
|
+
var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
|
|
1298
1538
|
var MESSAGES_PER_SNAPSHOT = 10;
|
|
1299
1539
|
var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
|
|
1300
1540
|
var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
|
|
1301
1541
|
var DataPipeline = class DataPipeline2 {
|
|
1302
1542
|
constructor(_params) {
|
|
1303
1543
|
this._params = _params;
|
|
1304
|
-
this._ctx = new
|
|
1544
|
+
this._ctx = new import_context6.Context();
|
|
1305
1545
|
this._pipeline = void 0;
|
|
1306
1546
|
this._targetTimeframe = void 0;
|
|
1307
1547
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
@@ -1309,6 +1549,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1309
1549
|
this._lastTimeframeSaveTime = 0;
|
|
1310
1550
|
this._lastSnapshotSaveTime = 0;
|
|
1311
1551
|
this._lastProcessedEpoch = -1;
|
|
1552
|
+
this._usage = new import_tracing2.TimeUsageCounter();
|
|
1553
|
+
this._mutations = new import_tracing2.TimeSeriesCounter();
|
|
1312
1554
|
this.currentEpoch = void 0;
|
|
1313
1555
|
this.appliedEpoch = void 0;
|
|
1314
1556
|
this.onNewEpoch = new import_async6.Event();
|
|
@@ -1320,13 +1562,11 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1320
1562
|
return this._pipeline;
|
|
1321
1563
|
}
|
|
1322
1564
|
get pipelineState() {
|
|
1323
|
-
|
|
1324
|
-
return (_a = this._pipeline) == null ? void 0 : _a.state;
|
|
1565
|
+
return this._pipeline?.state;
|
|
1325
1566
|
}
|
|
1326
1567
|
setTargetTimeframe(timeframe) {
|
|
1327
|
-
var _a;
|
|
1328
1568
|
this._targetTimeframe = timeframe;
|
|
1329
|
-
|
|
1569
|
+
this._pipeline?.state.setTargetTimeframe(timeframe);
|
|
1330
1570
|
}
|
|
1331
1571
|
async processCredential(credential) {
|
|
1332
1572
|
if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
|
|
@@ -1343,15 +1583,31 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1343
1583
|
}
|
|
1344
1584
|
this._pipeline = new Pipeline();
|
|
1345
1585
|
await this._params.onPipelineCreated(this._pipeline);
|
|
1346
|
-
await this._pipeline.start();
|
|
1347
1586
|
await this._pipeline.pause();
|
|
1587
|
+
await this._pipeline.start();
|
|
1348
1588
|
if (this._targetTimeframe) {
|
|
1349
1589
|
this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
|
|
1350
1590
|
}
|
|
1351
1591
|
const feedWriter = {
|
|
1352
1592
|
write: (data, options) => {
|
|
1353
|
-
(0,
|
|
1354
|
-
|
|
1593
|
+
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1594
|
+
F: __dxlog_file10,
|
|
1595
|
+
L: 154,
|
|
1596
|
+
S: this,
|
|
1597
|
+
A: [
|
|
1598
|
+
"this._pipeline",
|
|
1599
|
+
"'Pipeline is not initialized.'"
|
|
1600
|
+
]
|
|
1601
|
+
});
|
|
1602
|
+
(0, import_invariant8.invariant)(this.currentEpoch, "Epoch is not initialized.", {
|
|
1603
|
+
F: __dxlog_file10,
|
|
1604
|
+
L: 155,
|
|
1605
|
+
S: this,
|
|
1606
|
+
A: [
|
|
1607
|
+
"this.currentEpoch",
|
|
1608
|
+
"'Epoch is not initialized.'"
|
|
1609
|
+
]
|
|
1610
|
+
});
|
|
1355
1611
|
return this._pipeline.writer.write({
|
|
1356
1612
|
data
|
|
1357
1613
|
}, options);
|
|
@@ -1366,19 +1622,18 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1366
1622
|
this._isOpen = true;
|
|
1367
1623
|
}
|
|
1368
1624
|
async close() {
|
|
1369
|
-
var _a, _b, _c;
|
|
1370
1625
|
if (!this._isOpen) {
|
|
1371
1626
|
return;
|
|
1372
1627
|
}
|
|
1373
1628
|
(0, import_log9.log)("close", void 0, {
|
|
1374
|
-
F:
|
|
1375
|
-
L:
|
|
1629
|
+
F: __dxlog_file10,
|
|
1630
|
+
L: 179,
|
|
1376
1631
|
S: this,
|
|
1377
1632
|
C: (f, a) => f(...a)
|
|
1378
1633
|
});
|
|
1379
1634
|
this._isOpen = false;
|
|
1380
1635
|
await this._ctx.dispose();
|
|
1381
|
-
await
|
|
1636
|
+
await this._pipeline?.stop();
|
|
1382
1637
|
try {
|
|
1383
1638
|
await this._saveCache();
|
|
1384
1639
|
if (this._pipeline) {
|
|
@@ -1386,15 +1641,15 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1386
1641
|
}
|
|
1387
1642
|
} catch (err) {
|
|
1388
1643
|
import_log9.log.catch(err, void 0, {
|
|
1389
|
-
F:
|
|
1390
|
-
L:
|
|
1644
|
+
F: __dxlog_file10,
|
|
1645
|
+
L: 192,
|
|
1391
1646
|
S: this,
|
|
1392
1647
|
C: (f, a) => f(...a)
|
|
1393
1648
|
});
|
|
1394
1649
|
}
|
|
1395
|
-
await
|
|
1396
|
-
await
|
|
1397
|
-
this._ctx = new
|
|
1650
|
+
await this.databaseHost?.close();
|
|
1651
|
+
await this.itemManager?.destroy();
|
|
1652
|
+
this._ctx = new import_context6.Context();
|
|
1398
1653
|
this._pipeline = void 0;
|
|
1399
1654
|
this._targetTimeframe = void 0;
|
|
1400
1655
|
this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
|
|
@@ -1409,15 +1664,25 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1409
1664
|
await this._processEpochInSeparateTask(this.currentEpoch);
|
|
1410
1665
|
await waitForOneEpoch;
|
|
1411
1666
|
}
|
|
1412
|
-
(0,
|
|
1667
|
+
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1668
|
+
F: __dxlog_file10,
|
|
1669
|
+
L: 215,
|
|
1670
|
+
S: this,
|
|
1671
|
+
A: [
|
|
1672
|
+
"this._pipeline",
|
|
1673
|
+
"'Pipeline is not initialized.'"
|
|
1674
|
+
]
|
|
1675
|
+
});
|
|
1413
1676
|
for await (const msg of this._pipeline.consume()) {
|
|
1677
|
+
const span = this._usage.beginRecording();
|
|
1678
|
+
this._mutations.inc();
|
|
1414
1679
|
const { feedKey, seq, data } = msg;
|
|
1415
1680
|
(0, import_log9.log)("processing message", {
|
|
1416
1681
|
feedKey,
|
|
1417
1682
|
seq
|
|
1418
1683
|
}, {
|
|
1419
|
-
F:
|
|
1420
|
-
L:
|
|
1684
|
+
F: __dxlog_file10,
|
|
1685
|
+
L: 221,
|
|
1421
1686
|
S: this,
|
|
1422
1687
|
C: (f, a) => f(...a)
|
|
1423
1688
|
});
|
|
@@ -1428,8 +1693,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1428
1693
|
import_log9.log.warn("Could not find feed", {
|
|
1429
1694
|
feedKey
|
|
1430
1695
|
}, {
|
|
1431
|
-
F:
|
|
1432
|
-
L:
|
|
1696
|
+
F: __dxlog_file10,
|
|
1697
|
+
L: 227,
|
|
1433
1698
|
S: this,
|
|
1434
1699
|
C: (f, a) => f(...a)
|
|
1435
1700
|
});
|
|
@@ -1451,8 +1716,8 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1451
1716
|
seq,
|
|
1452
1717
|
spaceKey: this._params.spaceKey.toHex()
|
|
1453
1718
|
}, {
|
|
1454
|
-
F:
|
|
1455
|
-
L:
|
|
1719
|
+
F: __dxlog_file10,
|
|
1720
|
+
L: 244,
|
|
1456
1721
|
S: this,
|
|
1457
1722
|
C: (f, a) => f(...a)
|
|
1458
1723
|
});
|
|
@@ -1460,16 +1725,25 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1460
1725
|
}
|
|
1461
1726
|
} catch (err) {
|
|
1462
1727
|
import_log9.log.catch(err, void 0, {
|
|
1463
|
-
F:
|
|
1464
|
-
L:
|
|
1728
|
+
F: __dxlog_file10,
|
|
1729
|
+
L: 254,
|
|
1465
1730
|
S: this,
|
|
1466
1731
|
C: (f, a) => f(...a)
|
|
1467
1732
|
});
|
|
1468
1733
|
}
|
|
1734
|
+
span.end();
|
|
1469
1735
|
}
|
|
1470
1736
|
}
|
|
1471
1737
|
_createSnapshot() {
|
|
1472
|
-
(0,
|
|
1738
|
+
(0, import_invariant8.invariant)(this.databaseHost, "Database backend is not initialized.", {
|
|
1739
|
+
F: __dxlog_file10,
|
|
1740
|
+
L: 262,
|
|
1741
|
+
S: this,
|
|
1742
|
+
A: [
|
|
1743
|
+
"this.databaseHost",
|
|
1744
|
+
"'Database backend is not initialized.'"
|
|
1745
|
+
]
|
|
1746
|
+
});
|
|
1473
1747
|
return {
|
|
1474
1748
|
spaceKey: this._params.spaceKey.asUint8Array(),
|
|
1475
1749
|
timeframe: this._pipeline.state.timeframe,
|
|
@@ -1477,26 +1751,22 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1477
1751
|
};
|
|
1478
1752
|
}
|
|
1479
1753
|
async _saveTargetTimeframe(timeframe) {
|
|
1480
|
-
|
|
1481
|
-
const newTimeframe = import_timeframe4.Timeframe.merge((_a = this._targetTimeframe) != null ? _a : new import_timeframe4.Timeframe(), timeframe);
|
|
1754
|
+
const newTimeframe = import_timeframe4.Timeframe.merge(this._targetTimeframe ?? new import_timeframe4.Timeframe(), timeframe);
|
|
1482
1755
|
await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
|
|
1483
1756
|
this._targetTimeframe = newTimeframe;
|
|
1484
1757
|
}
|
|
1485
1758
|
async _saveCache() {
|
|
1486
|
-
var _a;
|
|
1487
1759
|
const cache = {};
|
|
1488
1760
|
try {
|
|
1489
|
-
const propertiesItem = this.itemManager.items.find((item) =>
|
|
1490
|
-
|
|
1491
|
-
return ((_a2 = item.modelMeta) == null ? void 0 : _a2.type) === "dxos:model/document" && ((_b = (0, import_echo_db3.getStateMachineFromItem)(item)) == null ? void 0 : _b.snapshot()).type === "dxos.sdk.client.Properties";
|
|
1492
|
-
});
|
|
1761
|
+
const propertiesItem = this.itemManager.items.find((item) => item.modelMeta?.type === "dxos:model/document" && // TODO(burdon): Document?
|
|
1762
|
+
((0, import_echo_db3.getStateMachineFromItem)(item)?.snapshot()).type === import_echo_db3.TYPE_PROPERTIES);
|
|
1493
1763
|
if (propertiesItem) {
|
|
1494
|
-
cache.properties = (
|
|
1764
|
+
cache.properties = (0, import_echo_db3.getStateMachineFromItem)(propertiesItem)?.snapshot();
|
|
1495
1765
|
}
|
|
1496
1766
|
} catch (err) {
|
|
1497
1767
|
import_log9.log.warn("Failed to cache properties", err, {
|
|
1498
|
-
F:
|
|
1499
|
-
L:
|
|
1768
|
+
F: __dxlog_file10,
|
|
1769
|
+
L: 291,
|
|
1500
1770
|
S: this,
|
|
1501
1771
|
C: (f, a) => f(...a)
|
|
1502
1772
|
});
|
|
@@ -1513,24 +1783,23 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1513
1783
|
}
|
|
1514
1784
|
}
|
|
1515
1785
|
async _processEpochInSeparateTask(epoch) {
|
|
1516
|
-
var _a;
|
|
1517
1786
|
if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
|
|
1518
1787
|
return;
|
|
1519
1788
|
}
|
|
1520
|
-
await
|
|
1521
|
-
const ctx = new
|
|
1789
|
+
await this._epochCtx?.dispose();
|
|
1790
|
+
const ctx = new import_context6.Context({
|
|
1522
1791
|
onError: (err) => {
|
|
1523
1792
|
if (err instanceof import_errors2.CancelledError) {
|
|
1524
1793
|
(0, import_log9.log)("Epoch processing cancelled.", void 0, {
|
|
1525
|
-
F:
|
|
1526
|
-
L:
|
|
1794
|
+
F: __dxlog_file10,
|
|
1795
|
+
L: 323,
|
|
1527
1796
|
S: this,
|
|
1528
1797
|
C: (f, a) => f(...a)
|
|
1529
1798
|
});
|
|
1530
1799
|
} else {
|
|
1531
1800
|
import_log9.log.catch(err, void 0, {
|
|
1532
|
-
F:
|
|
1533
|
-
L:
|
|
1801
|
+
F: __dxlog_file10,
|
|
1802
|
+
L: 325,
|
|
1534
1803
|
S: this,
|
|
1535
1804
|
C: (f, a) => f(...a)
|
|
1536
1805
|
});
|
|
@@ -1542,28 +1811,36 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1542
1811
|
if (!this._isOpen) {
|
|
1543
1812
|
return;
|
|
1544
1813
|
}
|
|
1545
|
-
(0, import_log9.log)("process epoch", {
|
|
1546
|
-
epoch
|
|
1547
|
-
}, {
|
|
1548
|
-
F: __dxlog_file9,
|
|
1549
|
-
L: 322,
|
|
1550
|
-
S: this,
|
|
1551
|
-
C: (f, a) => f(...a)
|
|
1552
|
-
});
|
|
1553
1814
|
await this._processEpoch(ctx, epoch.subject.assertion);
|
|
1554
1815
|
this.appliedEpoch = epoch;
|
|
1555
1816
|
this.onNewEpoch.emit(epoch);
|
|
1556
1817
|
});
|
|
1557
1818
|
}
|
|
1558
1819
|
async _processEpoch(ctx, epoch) {
|
|
1559
|
-
(0,
|
|
1560
|
-
|
|
1820
|
+
(0, import_invariant8.invariant)(this._isOpen, "Space is closed.", {
|
|
1821
|
+
F: __dxlog_file10,
|
|
1822
|
+
L: 344,
|
|
1823
|
+
S: this,
|
|
1824
|
+
A: [
|
|
1825
|
+
"this._isOpen",
|
|
1826
|
+
"'Space is closed.'"
|
|
1827
|
+
]
|
|
1828
|
+
});
|
|
1829
|
+
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1830
|
+
F: __dxlog_file10,
|
|
1831
|
+
L: 345,
|
|
1832
|
+
S: this,
|
|
1833
|
+
A: [
|
|
1834
|
+
"this._pipeline",
|
|
1835
|
+
""
|
|
1836
|
+
]
|
|
1837
|
+
});
|
|
1561
1838
|
this._lastProcessedEpoch = epoch.number;
|
|
1562
|
-
(0, import_log9.log)("
|
|
1563
|
-
epoch
|
|
1839
|
+
(0, import_log9.log)("processing", {
|
|
1840
|
+
epoch: (0, import_log9.omit)(epoch, "proof")
|
|
1564
1841
|
}, {
|
|
1565
|
-
F:
|
|
1566
|
-
L:
|
|
1842
|
+
F: __dxlog_file10,
|
|
1843
|
+
L: 348,
|
|
1567
1844
|
S: this,
|
|
1568
1845
|
C: (f, a) => f(...a)
|
|
1569
1846
|
});
|
|
@@ -1571,9 +1848,9 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1571
1848
|
const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
|
|
1572
1849
|
this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
|
|
1573
1850
|
}
|
|
1574
|
-
(0, import_log9.log)("restarting pipeline
|
|
1575
|
-
F:
|
|
1576
|
-
L:
|
|
1851
|
+
(0, import_log9.log)("restarting pipeline from epoch", void 0, {
|
|
1852
|
+
F: __dxlog_file10,
|
|
1853
|
+
L: 354,
|
|
1577
1854
|
S: this,
|
|
1578
1855
|
C: (f, a) => f(...a)
|
|
1579
1856
|
});
|
|
@@ -1582,12 +1859,36 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1582
1859
|
await this._pipeline.unpause();
|
|
1583
1860
|
}
|
|
1584
1861
|
async waitUntilTimeframe(timeframe) {
|
|
1585
|
-
(0,
|
|
1862
|
+
(0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
|
|
1863
|
+
F: __dxlog_file10,
|
|
1864
|
+
L: 361,
|
|
1865
|
+
S: this,
|
|
1866
|
+
A: [
|
|
1867
|
+
"this._pipeline",
|
|
1868
|
+
"'Pipeline is not initialized.'"
|
|
1869
|
+
]
|
|
1870
|
+
});
|
|
1586
1871
|
await this._pipeline.state.waitUntilTimeframe(timeframe);
|
|
1587
1872
|
}
|
|
1588
1873
|
async createEpoch() {
|
|
1589
|
-
(0,
|
|
1590
|
-
|
|
1874
|
+
(0, import_invariant8.invariant)(this._pipeline, void 0, {
|
|
1875
|
+
F: __dxlog_file10,
|
|
1876
|
+
L: 367,
|
|
1877
|
+
S: this,
|
|
1878
|
+
A: [
|
|
1879
|
+
"this._pipeline",
|
|
1880
|
+
""
|
|
1881
|
+
]
|
|
1882
|
+
});
|
|
1883
|
+
(0, import_invariant8.invariant)(this.currentEpoch, void 0, {
|
|
1884
|
+
F: __dxlog_file10,
|
|
1885
|
+
L: 368,
|
|
1886
|
+
S: this,
|
|
1887
|
+
A: [
|
|
1888
|
+
"this.currentEpoch",
|
|
1889
|
+
""
|
|
1890
|
+
]
|
|
1891
|
+
});
|
|
1591
1892
|
await this._pipeline.pause();
|
|
1592
1893
|
const snapshot = await this._createSnapshot();
|
|
1593
1894
|
const snapshotCid = await this._params.snapshotManager.store(snapshot);
|
|
@@ -1604,24 +1905,31 @@ var DataPipeline = class DataPipeline2 {
|
|
|
1604
1905
|
await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
|
|
1605
1906
|
}
|
|
1606
1907
|
};
|
|
1607
|
-
|
|
1908
|
+
_ts_decorate6([
|
|
1909
|
+
import_tracing2.trace.metricsCounter()
|
|
1910
|
+
], DataPipeline.prototype, "_usage", void 0);
|
|
1911
|
+
_ts_decorate6([
|
|
1912
|
+
import_tracing2.trace.metricsCounter()
|
|
1913
|
+
], DataPipeline.prototype, "_mutations", void 0);
|
|
1914
|
+
_ts_decorate6([
|
|
1608
1915
|
import_async6.synchronized
|
|
1609
1916
|
], DataPipeline.prototype, "open", null);
|
|
1610
|
-
|
|
1917
|
+
_ts_decorate6([
|
|
1611
1918
|
import_async6.synchronized
|
|
1612
1919
|
], DataPipeline.prototype, "close", null);
|
|
1613
|
-
|
|
1920
|
+
_ts_decorate6([
|
|
1614
1921
|
import_async6.synchronized
|
|
1615
1922
|
], DataPipeline.prototype, "_processEpoch", null);
|
|
1616
|
-
|
|
1923
|
+
_ts_decorate6([
|
|
1617
1924
|
import_async6.synchronized
|
|
1618
1925
|
], DataPipeline.prototype, "createEpoch", null);
|
|
1619
|
-
DataPipeline =
|
|
1620
|
-
(0, import_async6.trackLeaks)("open", "close")
|
|
1926
|
+
DataPipeline = _ts_decorate6([
|
|
1927
|
+
(0, import_async6.trackLeaks)("open", "close"),
|
|
1928
|
+
import_tracing2.trace.resource()
|
|
1621
1929
|
], DataPipeline);
|
|
1622
1930
|
|
|
1623
1931
|
// packages/core/echo/echo-pipeline/src/space/space.ts
|
|
1624
|
-
function
|
|
1932
|
+
function _ts_decorate7(decorators, target, key, desc) {
|
|
1625
1933
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1626
1934
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1627
1935
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1631,14 +1939,22 @@ function _ts_decorate6(decorators, target, key, desc) {
|
|
|
1631
1939
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1632
1940
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1633
1941
|
}
|
|
1634
|
-
var
|
|
1942
|
+
var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
|
|
1635
1943
|
var Space = class Space2 {
|
|
1636
1944
|
constructor(params) {
|
|
1637
1945
|
this._addFeedLock = new import_async7.Lock();
|
|
1638
1946
|
this.onCredentialProcessed = new import_util7.Callback();
|
|
1639
1947
|
this.stateUpdate = new import_async7.Event();
|
|
1640
1948
|
this._isOpen = false;
|
|
1641
|
-
(0,
|
|
1949
|
+
(0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
|
|
1950
|
+
F: __dxlog_file11,
|
|
1951
|
+
L: 73,
|
|
1952
|
+
S: this,
|
|
1953
|
+
A: [
|
|
1954
|
+
"params.spaceKey && params.feedProvider",
|
|
1955
|
+
""
|
|
1956
|
+
]
|
|
1957
|
+
});
|
|
1642
1958
|
this._key = params.spaceKey;
|
|
1643
1959
|
this._genesisFeedKey = params.genesisFeed.key;
|
|
1644
1960
|
this._feedProvider = params.feedProvider;
|
|
@@ -1673,8 +1989,8 @@ var Space = class Space2 {
|
|
|
1673
1989
|
(0, import_log10.log)("onCredentialProcessed", {
|
|
1674
1990
|
credential
|
|
1675
1991
|
}, {
|
|
1676
|
-
F:
|
|
1677
|
-
L:
|
|
1992
|
+
F: __dxlog_file11,
|
|
1993
|
+
L: 109,
|
|
1678
1994
|
S: this,
|
|
1679
1995
|
C: (f, a) => f(...a)
|
|
1680
1996
|
});
|
|
@@ -1716,12 +2032,10 @@ var Space = class Space2 {
|
|
|
1716
2032
|
return this._genesisFeedKey;
|
|
1717
2033
|
}
|
|
1718
2034
|
get controlFeedKey() {
|
|
1719
|
-
|
|
1720
|
-
return (_a = this._controlFeed) == null ? void 0 : _a.key;
|
|
2035
|
+
return this._controlFeed?.key;
|
|
1721
2036
|
}
|
|
1722
2037
|
get dataFeedKey() {
|
|
1723
|
-
|
|
1724
|
-
return (_a = this._dataFeed) == null ? void 0 : _a.key;
|
|
2038
|
+
return this._dataFeed?.key;
|
|
1725
2039
|
}
|
|
1726
2040
|
get spaceState() {
|
|
1727
2041
|
return this._controlPipeline.spaceState;
|
|
@@ -1739,16 +2053,31 @@ var Space = class Space2 {
|
|
|
1739
2053
|
return this._snapshotManager;
|
|
1740
2054
|
}
|
|
1741
2055
|
setControlFeed(feed) {
|
|
1742
|
-
(0,
|
|
2056
|
+
(0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
|
|
2057
|
+
F: __dxlog_file11,
|
|
2058
|
+
L: 184,
|
|
2059
|
+
S: this,
|
|
2060
|
+
A: [
|
|
2061
|
+
"!this._controlFeed",
|
|
2062
|
+
"'Control feed already set.'"
|
|
2063
|
+
]
|
|
2064
|
+
});
|
|
1743
2065
|
this._controlFeed = feed;
|
|
1744
2066
|
this._controlPipeline.setWriteFeed(feed);
|
|
1745
2067
|
return this;
|
|
1746
2068
|
}
|
|
1747
2069
|
setDataFeed(feed) {
|
|
1748
|
-
|
|
1749
|
-
|
|
2070
|
+
(0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
|
|
2071
|
+
F: __dxlog_file11,
|
|
2072
|
+
L: 191,
|
|
2073
|
+
S: this,
|
|
2074
|
+
A: [
|
|
2075
|
+
"!this._dataFeed",
|
|
2076
|
+
"'Data feed already set.'"
|
|
2077
|
+
]
|
|
2078
|
+
});
|
|
1750
2079
|
this._dataFeed = feed;
|
|
1751
|
-
|
|
2080
|
+
this._dataPipeline.pipeline?.setWriteFeed(feed);
|
|
1752
2081
|
return this;
|
|
1753
2082
|
}
|
|
1754
2083
|
/**
|
|
@@ -1763,10 +2092,10 @@ var Space = class Space2 {
|
|
|
1763
2092
|
// getDataFeeds(): FeedInfo[] {
|
|
1764
2093
|
// return this._dataPipeline?.getFeeds();
|
|
1765
2094
|
// }
|
|
1766
|
-
async open() {
|
|
2095
|
+
async open(ctx) {
|
|
1767
2096
|
(0, import_log10.log)("opening...", void 0, {
|
|
1768
|
-
F:
|
|
1769
|
-
L:
|
|
2097
|
+
F: __dxlog_file11,
|
|
2098
|
+
L: 213,
|
|
1770
2099
|
S: this,
|
|
1771
2100
|
C: (f, a) => f(...a)
|
|
1772
2101
|
});
|
|
@@ -1778,8 +2107,8 @@ var Space = class Space2 {
|
|
|
1778
2107
|
await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
|
|
1779
2108
|
this._isOpen = true;
|
|
1780
2109
|
(0, import_log10.log)("opened", void 0, {
|
|
1781
|
-
F:
|
|
1782
|
-
L:
|
|
2110
|
+
F: __dxlog_file11,
|
|
2111
|
+
L: 224,
|
|
1783
2112
|
S: this,
|
|
1784
2113
|
C: (f, a) => f(...a)
|
|
1785
2114
|
});
|
|
@@ -1788,8 +2117,8 @@ var Space = class Space2 {
|
|
|
1788
2117
|
(0, import_log10.log)("closing...", {
|
|
1789
2118
|
key: this._key
|
|
1790
2119
|
}, {
|
|
1791
|
-
F:
|
|
1792
|
-
L:
|
|
2120
|
+
F: __dxlog_file11,
|
|
2121
|
+
L: 229,
|
|
1793
2122
|
S: this,
|
|
1794
2123
|
C: (f, a) => f(...a)
|
|
1795
2124
|
});
|
|
@@ -1802,42 +2131,53 @@ var Space = class Space2 {
|
|
|
1802
2131
|
await this._controlPipeline.stop();
|
|
1803
2132
|
this._isOpen = false;
|
|
1804
2133
|
(0, import_log10.log)("closed", void 0, {
|
|
1805
|
-
F:
|
|
1806
|
-
L:
|
|
2134
|
+
F: __dxlog_file11,
|
|
2135
|
+
L: 242,
|
|
1807
2136
|
S: this,
|
|
1808
2137
|
C: (f, a) => f(...a)
|
|
1809
2138
|
});
|
|
1810
2139
|
}
|
|
1811
2140
|
async initializeDataPipeline() {
|
|
1812
2141
|
(0, import_log10.log)("initializeDataPipeline", void 0, {
|
|
1813
|
-
F:
|
|
1814
|
-
L:
|
|
2142
|
+
F: __dxlog_file11,
|
|
2143
|
+
L: 247,
|
|
1815
2144
|
S: this,
|
|
1816
2145
|
C: (f, a) => f(...a)
|
|
1817
2146
|
});
|
|
1818
|
-
(0,
|
|
2147
|
+
(0, import_invariant9.invariant)(this._isOpen, "Space must be open to initialize data pipeline.", {
|
|
2148
|
+
F: __dxlog_file11,
|
|
2149
|
+
L: 248,
|
|
2150
|
+
S: this,
|
|
2151
|
+
A: [
|
|
2152
|
+
"this._isOpen",
|
|
2153
|
+
"'Space must be open to initialize data pipeline.'"
|
|
2154
|
+
]
|
|
2155
|
+
});
|
|
1819
2156
|
await this._dataPipeline.open();
|
|
1820
2157
|
}
|
|
1821
2158
|
};
|
|
1822
|
-
|
|
1823
|
-
import_log10.logInfo
|
|
2159
|
+
_ts_decorate7([
|
|
2160
|
+
import_log10.logInfo,
|
|
2161
|
+
import_tracing3.trace.info()
|
|
1824
2162
|
], Space.prototype, "key", null);
|
|
1825
|
-
|
|
1826
|
-
import_async7.synchronized
|
|
2163
|
+
_ts_decorate7([
|
|
2164
|
+
import_async7.synchronized,
|
|
2165
|
+
import_tracing3.trace.span()
|
|
1827
2166
|
], Space.prototype, "open", null);
|
|
1828
|
-
|
|
2167
|
+
_ts_decorate7([
|
|
1829
2168
|
import_async7.synchronized
|
|
1830
2169
|
], Space.prototype, "close", null);
|
|
1831
|
-
|
|
2170
|
+
_ts_decorate7([
|
|
1832
2171
|
import_async7.synchronized
|
|
1833
2172
|
], Space.prototype, "initializeDataPipeline", null);
|
|
1834
|
-
Space =
|
|
1835
|
-
(0, import_async7.trackLeaks)("open", "close")
|
|
2173
|
+
Space = _ts_decorate7([
|
|
2174
|
+
(0, import_async7.trackLeaks)("open", "close"),
|
|
2175
|
+
import_tracing3.trace.resource()
|
|
1836
2176
|
], Space);
|
|
1837
2177
|
|
|
1838
2178
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
1839
2179
|
var import_async8 = require("@dxos/async");
|
|
1840
|
-
var
|
|
2180
|
+
var import_debug4 = require("@dxos/debug");
|
|
1841
2181
|
var import_keys6 = require("@dxos/keys");
|
|
1842
2182
|
var import_log12 = require("@dxos/log");
|
|
1843
2183
|
var import_protocols6 = require("@dxos/protocols");
|
|
@@ -1852,7 +2192,7 @@ var import_teleport2 = require("@dxos/teleport");
|
|
|
1852
2192
|
var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
|
|
1853
2193
|
var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
|
|
1854
2194
|
var import_util8 = require("@dxos/util");
|
|
1855
|
-
function
|
|
2195
|
+
function _ts_decorate8(decorators, target, key, desc) {
|
|
1856
2196
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
1857
2197
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
1858
2198
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -1862,7 +2202,7 @@ function _ts_decorate7(decorators, target, key, desc) {
|
|
|
1862
2202
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
1863
2203
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
1864
2204
|
}
|
|
1865
|
-
var
|
|
2205
|
+
var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
|
|
1866
2206
|
var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
|
|
1867
2207
|
var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
|
|
1868
2208
|
var SpaceProtocol = class {
|
|
@@ -1892,8 +2232,8 @@ var SpaceProtocol = class {
|
|
|
1892
2232
|
(0, import_log11.log)("addFeed", {
|
|
1893
2233
|
key: feed.key
|
|
1894
2234
|
}, {
|
|
1895
|
-
F:
|
|
1896
|
-
L:
|
|
2235
|
+
F: __dxlog_file12,
|
|
2236
|
+
L: 96,
|
|
1897
2237
|
S: this,
|
|
1898
2238
|
C: (f, a) => f(...a)
|
|
1899
2239
|
});
|
|
@@ -1902,6 +2242,7 @@ var SpaceProtocol = class {
|
|
|
1902
2242
|
session.replicator.addFeed(feed);
|
|
1903
2243
|
}
|
|
1904
2244
|
}
|
|
2245
|
+
// TODO(burdon): Rename open? Common open/close interfaces for all services?
|
|
1905
2246
|
async start() {
|
|
1906
2247
|
if (this._connection) {
|
|
1907
2248
|
return;
|
|
@@ -1914,8 +2255,8 @@ var SpaceProtocol = class {
|
|
|
1914
2255
|
};
|
|
1915
2256
|
await this.blobSync.open();
|
|
1916
2257
|
(0, import_log11.log)("starting...", void 0, {
|
|
1917
|
-
F:
|
|
1918
|
-
L:
|
|
2258
|
+
F: __dxlog_file12,
|
|
2259
|
+
L: 122,
|
|
1919
2260
|
S: this,
|
|
1920
2261
|
C: (f, a) => f(...a)
|
|
1921
2262
|
});
|
|
@@ -1925,11 +2266,11 @@ var SpaceProtocol = class {
|
|
|
1925
2266
|
peerId: this._swarmIdentity.peerKey,
|
|
1926
2267
|
topic,
|
|
1927
2268
|
topology: new import_network_manager.MMSTTopology(topologyConfig),
|
|
1928
|
-
label: `
|
|
2269
|
+
label: `space swarm ${topic.truncate()}`
|
|
1929
2270
|
});
|
|
1930
2271
|
(0, import_log11.log)("started", void 0, {
|
|
1931
|
-
F:
|
|
1932
|
-
L:
|
|
2272
|
+
F: __dxlog_file12,
|
|
2273
|
+
L: 132,
|
|
1933
2274
|
S: this,
|
|
1934
2275
|
C: (f, a) => f(...a)
|
|
1935
2276
|
});
|
|
@@ -1938,15 +2279,15 @@ var SpaceProtocol = class {
|
|
|
1938
2279
|
await this.blobSync.close();
|
|
1939
2280
|
if (this._connection) {
|
|
1940
2281
|
(0, import_log11.log)("stopping...", void 0, {
|
|
1941
|
-
F:
|
|
1942
|
-
L:
|
|
2282
|
+
F: __dxlog_file12,
|
|
2283
|
+
L: 139,
|
|
1943
2284
|
S: this,
|
|
1944
2285
|
C: (f, a) => f(...a)
|
|
1945
2286
|
});
|
|
1946
2287
|
await this._connection.close();
|
|
1947
2288
|
(0, import_log11.log)("stopped", void 0, {
|
|
1948
|
-
F:
|
|
1949
|
-
L:
|
|
2289
|
+
F: __dxlog_file12,
|
|
2290
|
+
L: 141,
|
|
1950
2291
|
S: this,
|
|
1951
2292
|
C: (f, a) => f(...a)
|
|
1952
2293
|
});
|
|
@@ -1969,10 +2310,10 @@ var SpaceProtocol = class {
|
|
|
1969
2310
|
};
|
|
1970
2311
|
}
|
|
1971
2312
|
};
|
|
1972
|
-
|
|
2313
|
+
_ts_decorate8([
|
|
1973
2314
|
import_log11.logInfo
|
|
1974
2315
|
], SpaceProtocol.prototype, "_topic", void 0);
|
|
1975
|
-
|
|
2316
|
+
_ts_decorate8([
|
|
1976
2317
|
import_log11.logInfo
|
|
1977
2318
|
], SpaceProtocol.prototype, "_ownPeerKey", null);
|
|
1978
2319
|
var AuthStatus;
|
|
@@ -2011,20 +2352,18 @@ var SpaceProtocolSession = class {
|
|
|
2011
2352
|
provider: this._swarmIdentity.credentialProvider,
|
|
2012
2353
|
verifier: this._swarmIdentity.credentialAuthenticator,
|
|
2013
2354
|
onAuthSuccess: () => {
|
|
2014
|
-
var _a;
|
|
2015
2355
|
(0, import_log11.log)("Peer authenticated", void 0, {
|
|
2016
|
-
F:
|
|
2017
|
-
L:
|
|
2356
|
+
F: __dxlog_file12,
|
|
2357
|
+
L: 238,
|
|
2018
2358
|
S: this,
|
|
2019
2359
|
C: (f, a) => f(...a)
|
|
2020
2360
|
});
|
|
2021
2361
|
this._authStatus = AuthStatus.SUCCESS;
|
|
2022
|
-
|
|
2362
|
+
this._onSessionAuth?.(this._teleport);
|
|
2023
2363
|
},
|
|
2024
2364
|
onAuthFailure: () => {
|
|
2025
|
-
var _a;
|
|
2026
2365
|
this._authStatus = AuthStatus.FAILURE;
|
|
2027
|
-
|
|
2366
|
+
this._onAuthFailure?.(this._teleport);
|
|
2028
2367
|
}
|
|
2029
2368
|
}));
|
|
2030
2369
|
this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
|
|
@@ -2034,15 +2373,15 @@ var SpaceProtocolSession = class {
|
|
|
2034
2373
|
await this._teleport.close();
|
|
2035
2374
|
}
|
|
2036
2375
|
};
|
|
2037
|
-
|
|
2376
|
+
_ts_decorate8([
|
|
2038
2377
|
import_log11.logInfo
|
|
2039
2378
|
], SpaceProtocolSession.prototype, "_wireParams", void 0);
|
|
2040
|
-
|
|
2379
|
+
_ts_decorate8([
|
|
2041
2380
|
import_log11.logInfo
|
|
2042
2381
|
], SpaceProtocolSession.prototype, "authStatus", null);
|
|
2043
2382
|
|
|
2044
2383
|
// packages/core/echo/echo-pipeline/src/space/space-manager.ts
|
|
2045
|
-
function
|
|
2384
|
+
function _ts_decorate9(decorators, target, key, desc) {
|
|
2046
2385
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
2047
2386
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
|
|
2048
2387
|
r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -2052,7 +2391,7 @@ function _ts_decorate8(decorators, target, key, desc) {
|
|
|
2052
2391
|
r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
2053
2392
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
2054
2393
|
}
|
|
2055
|
-
var
|
|
2394
|
+
var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
|
|
2056
2395
|
var SpaceManager = class SpaceManager2 {
|
|
2057
2396
|
constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
|
|
2058
2397
|
this._spaces = new import_util9.ComplexMap(import_keys6.PublicKey.hash);
|
|
@@ -2076,11 +2415,10 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2076
2415
|
].map((space) => space.close()));
|
|
2077
2416
|
}
|
|
2078
2417
|
async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
|
|
2079
|
-
var _a;
|
|
2080
2418
|
import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.begin({
|
|
2081
2419
|
id: this._instanceId
|
|
2082
2420
|
}), {
|
|
2083
|
-
F:
|
|
2421
|
+
F: __dxlog_file13,
|
|
2084
2422
|
L: 97,
|
|
2085
2423
|
S: this,
|
|
2086
2424
|
C: (f, a) => f(...a)
|
|
@@ -2088,12 +2426,12 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2088
2426
|
(0, import_log12.log)("constructing space...", {
|
|
2089
2427
|
spaceKey: metadata.genesisFeedKey
|
|
2090
2428
|
}, {
|
|
2091
|
-
F:
|
|
2429
|
+
F: __dxlog_file13,
|
|
2092
2430
|
L: 98,
|
|
2093
2431
|
S: this,
|
|
2094
2432
|
C: (f, a) => f(...a)
|
|
2095
2433
|
});
|
|
2096
|
-
const genesisFeed = await this._feedStore.openFeed(
|
|
2434
|
+
const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? (0, import_debug4.failUndefined)());
|
|
2097
2435
|
const spaceKey = metadata.key;
|
|
2098
2436
|
const protocol = new SpaceProtocol({
|
|
2099
2437
|
topic: spaceKey,
|
|
@@ -2118,21 +2456,21 @@ var SpaceManager = class SpaceManager2 {
|
|
|
2118
2456
|
import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols6.trace.end({
|
|
2119
2457
|
id: this._instanceId
|
|
2120
2458
|
}), {
|
|
2121
|
-
F:
|
|
2122
|
-
L:
|
|
2459
|
+
F: __dxlog_file13,
|
|
2460
|
+
L: 126,
|
|
2123
2461
|
S: this,
|
|
2124
2462
|
C: (f, a) => f(...a)
|
|
2125
2463
|
});
|
|
2126
2464
|
return space;
|
|
2127
2465
|
}
|
|
2128
2466
|
};
|
|
2129
|
-
|
|
2467
|
+
_ts_decorate9([
|
|
2130
2468
|
import_async8.synchronized
|
|
2131
2469
|
], SpaceManager.prototype, "open", null);
|
|
2132
|
-
|
|
2470
|
+
_ts_decorate9([
|
|
2133
2471
|
import_async8.synchronized
|
|
2134
2472
|
], SpaceManager.prototype, "close", null);
|
|
2135
|
-
SpaceManager =
|
|
2473
|
+
SpaceManager = _ts_decorate9([
|
|
2136
2474
|
(0, import_async8.trackLeaks)("open", "close")
|
|
2137
2475
|
], SpaceManager);
|
|
2138
2476
|
|
|
@@ -2162,10 +2500,10 @@ var WebsocketNetworkManagerProvider = (signalUrl) => () => new import_network_ma
|
|
|
2162
2500
|
var TestAgentBuilder = class {
|
|
2163
2501
|
constructor({ storage, networkManagerProvider } = {}) {
|
|
2164
2502
|
this._agents = new import_util10.ComplexMap(import_keys7.PublicKey.hash);
|
|
2165
|
-
this._storage = storage
|
|
2503
|
+
this._storage = storage ?? (0, import_random_access_storage.createStorage)({
|
|
2166
2504
|
type: import_random_access_storage.StorageType.RAM
|
|
2167
2505
|
});
|
|
2168
|
-
this._networkManagerProvider = networkManagerProvider
|
|
2506
|
+
this._networkManagerProvider = networkManagerProvider ?? MemoryNetworkManagerProvider(new import_messaging.MemorySignalManagerContext());
|
|
2169
2507
|
}
|
|
2170
2508
|
async close() {
|
|
2171
2509
|
return Promise.all(this.agents.map((agent) => agent.close()));
|
|
@@ -2187,16 +2525,13 @@ var TestAgentBuilder = class {
|
|
|
2187
2525
|
};
|
|
2188
2526
|
var TestAgent = class {
|
|
2189
2527
|
get metadataStore() {
|
|
2190
|
-
|
|
2191
|
-
return (_a = this._metadataStore) != null ? _a : this._metadataStore = new MetadataStore(this.storage.createDirectory("metadata"));
|
|
2528
|
+
return this._metadataStore ??= new MetadataStore(this.storage.createDirectory("metadata"));
|
|
2192
2529
|
}
|
|
2193
2530
|
get snapshotStore() {
|
|
2194
|
-
|
|
2195
|
-
return (_a = this._snapshotStore) != null ? _a : this._snapshotStore = new SnapshotStore(this.storage.createDirectory("snapshots"));
|
|
2531
|
+
return this._snapshotStore ??= new SnapshotStore(this.storage.createDirectory("snapshots"));
|
|
2196
2532
|
}
|
|
2197
2533
|
get blobStore() {
|
|
2198
|
-
|
|
2199
|
-
return (_a = this._blobStore) != null ? _a : this._blobStore = new import_teleport_extension_object_sync2.BlobStore(this.storage.createDirectory("blobs"));
|
|
2534
|
+
return this._blobStore ??= new import_teleport_extension_object_sync2.BlobStore(this.storage.createDirectory("blobs"));
|
|
2200
2535
|
}
|
|
2201
2536
|
constructor(_networkManagerProvider, _feedBuilder, identityKey, deviceKey) {
|
|
2202
2537
|
this._networkManagerProvider = _networkManagerProvider;
|
|
@@ -2221,8 +2556,7 @@ var TestAgent = class {
|
|
|
2221
2556
|
return this._spaces.get(spaceKey);
|
|
2222
2557
|
}
|
|
2223
2558
|
get spaceManager() {
|
|
2224
|
-
|
|
2225
|
-
return (_a = this._spaceManager) != null ? _a : this._spaceManager = new SpaceManager({
|
|
2559
|
+
return this._spaceManager ??= new SpaceManager({
|
|
2226
2560
|
feedStore: this.feedStore,
|
|
2227
2561
|
networkManager: this._networkManagerProvider(),
|
|
2228
2562
|
modelFactory: this.modelFactory,
|
|
@@ -2242,7 +2576,7 @@ var TestAgent = class {
|
|
|
2242
2576
|
const controlFeed = await this.feedStore.openFeed(genesisKey, {
|
|
2243
2577
|
writable: true
|
|
2244
2578
|
});
|
|
2245
|
-
const dataFeed = await this.feedStore.openFeed(dataKey
|
|
2579
|
+
const dataFeed = await this.feedStore.openFeed(dataKey ?? await this.keyring.createKey(), {
|
|
2246
2580
|
writable: true,
|
|
2247
2581
|
sparse: true
|
|
2248
2582
|
});
|
|
@@ -2272,7 +2606,7 @@ var TestAgent = class {
|
|
|
2272
2606
|
});
|
|
2273
2607
|
space.setControlFeed(controlFeed);
|
|
2274
2608
|
space.setDataFeed(dataFeed);
|
|
2275
|
-
await space.open();
|
|
2609
|
+
await space.open(new import_context7.Context());
|
|
2276
2610
|
this._spaces.set(spaceKey, space);
|
|
2277
2611
|
return space;
|
|
2278
2612
|
}
|
|
@@ -2287,7 +2621,7 @@ var TestAgent = class {
|
|
|
2287
2621
|
networkManager: this._networkManagerProvider(),
|
|
2288
2622
|
blobStore: this.blobStore,
|
|
2289
2623
|
onSessionAuth: (session) => {
|
|
2290
|
-
session.addExtension("dxos.mesh.teleport.gossip", (gossip
|
|
2624
|
+
session.addExtension("dxos.mesh.teleport.gossip", (gossip ?? this.createGossip()).createExtension({
|
|
2291
2625
|
remotePeerId: session.remotePeerId
|
|
2292
2626
|
}));
|
|
2293
2627
|
}
|
|
@@ -2303,7 +2637,7 @@ var TestAgent = class {
|
|
|
2303
2637
|
announceInterval: 30,
|
|
2304
2638
|
offlineTimeout: 200,
|
|
2305
2639
|
identityKey: this.identityKey,
|
|
2306
|
-
gossip: gossip
|
|
2640
|
+
gossip: gossip ?? this.createGossip()
|
|
2307
2641
|
});
|
|
2308
2642
|
}
|
|
2309
2643
|
async spaceGenesis(space) {
|
|
@@ -2368,9 +2702,8 @@ var createRemoteDatabaseFromDataServiceHost = async (modelFactory, dataServiceHo
|
|
|
2368
2702
|
};
|
|
2369
2703
|
};
|
|
2370
2704
|
var testLocalDatabase = async (create, check = create) => {
|
|
2371
|
-
var _a;
|
|
2372
2705
|
const objectId = import_keys8.PublicKey.random().toHex();
|
|
2373
|
-
await
|
|
2706
|
+
await create.databaseHost.getWriteStream()?.write({
|
|
2374
2707
|
batch: {
|
|
2375
2708
|
objects: [
|
|
2376
2709
|
{
|
|
@@ -2381,7 +2714,7 @@ var testLocalDatabase = async (create, check = create) => {
|
|
|
2381
2714
|
}
|
|
2382
2715
|
]
|
|
2383
2716
|
}
|
|
2384
|
-
})
|
|
2717
|
+
});
|
|
2385
2718
|
await (0, import_async9.asyncTimeout)(check.databaseHost._itemDemuxer.mutation.waitForCondition(() => check.itemManager.entities.has(objectId)), 2e3);
|
|
2386
2719
|
};
|
|
2387
2720
|
// Annotate the CommonJS export names for ESM import in node:
|