@dxos/echo-pipeline 0.3.8-next.f4e0086 → 0.3.9-main.325dd0a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,7 @@
1
1
  "use strict";
2
- var __create = Object.create;
3
2
  var __defProp = Object.defineProperty;
4
3
  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
4
  var __getOwnPropNames = Object.getOwnPropertyNames;
6
- var __getProtoOf = Object.getPrototypeOf;
7
5
  var __hasOwnProp = Object.prototype.hasOwnProperty;
8
6
  var __export = (target, all) => {
9
7
  for (var name in all)
@@ -17,2733 +15,36 @@ var __copyProps = (to, from, except, desc) => {
17
15
  }
18
16
  return to;
19
17
  };
20
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
- // If the importer is in node compatibility mode or this is not an ESM
22
- // file that has been converted to a CommonJS file using a Babel-
23
- // compatible transform (i.e. "__esModule" has not been set), then set
24
- // "default" to the CommonJS "module.exports" for node compatibility.
25
- isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
- mod
27
- ));
28
18
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
-
30
- // packages/core/echo/echo-pipeline/src/index.ts
31
- var src_exports = {};
32
- __export(src_exports, {
33
- AuthExtension: () => AuthExtension,
34
- AuthStatus: () => AuthStatus,
35
- DataPipeline: () => DataPipeline,
36
- DataServiceHost: () => DataServiceHost,
37
- DataServiceImpl: () => DataServiceImpl,
38
- DataServiceSubscriptions: () => DataServiceSubscriptions,
39
- DatabaseHost: () => DatabaseHost,
40
- MOCK_AUTH_PROVIDER: () => MOCK_AUTH_PROVIDER,
41
- MOCK_AUTH_VERIFIER: () => MOCK_AUTH_VERIFIER,
42
- MetadataStore: () => MetadataStore,
43
- Pipeline: () => Pipeline,
44
- SnapshotManager: () => SnapshotManager,
45
- SnapshotStore: () => SnapshotStore,
46
- Space: () => Space,
47
- SpaceManager: () => SpaceManager,
48
- SpaceProtocol: () => SpaceProtocol,
49
- SpaceProtocolSession: () => SpaceProtocolSession,
50
- TimeframeClock: () => TimeframeClock,
51
- codec: () => codec,
52
- createMappedFeedWriter: () => createMappedFeedWriter,
53
- mapFeedIndexesToTimeframe: () => mapFeedIndexesToTimeframe,
54
- mapTimeframeToFeedIndexes: () => mapTimeframeToFeedIndexes,
55
- startAfter: () => startAfter,
56
- valueEncoding: () => valueEncoding
19
+ var node_exports = {};
20
+ __export(node_exports, {
21
+ AuthExtension: () => import_chunk_YJAADRTG.AuthExtension,
22
+ AuthStatus: () => import_chunk_YJAADRTG.AuthStatus,
23
+ DataPipeline: () => import_chunk_YJAADRTG.DataPipeline,
24
+ DataServiceHost: () => import_chunk_YJAADRTG.DataServiceHost,
25
+ DataServiceImpl: () => import_chunk_YJAADRTG.DataServiceImpl,
26
+ DataServiceSubscriptions: () => import_chunk_YJAADRTG.DataServiceSubscriptions,
27
+ DatabaseHost: () => import_chunk_YJAADRTG.DatabaseHost,
28
+ MOCK_AUTH_PROVIDER: () => import_chunk_YJAADRTG.MOCK_AUTH_PROVIDER,
29
+ MOCK_AUTH_VERIFIER: () => import_chunk_YJAADRTG.MOCK_AUTH_VERIFIER,
30
+ MetadataStore: () => import_chunk_YJAADRTG.MetadataStore,
31
+ Pipeline: () => import_chunk_YJAADRTG.Pipeline,
32
+ SnapshotManager: () => import_chunk_YJAADRTG.SnapshotManager,
33
+ SnapshotStore: () => import_chunk_YJAADRTG.SnapshotStore,
34
+ Space: () => import_chunk_YJAADRTG.Space,
35
+ SpaceManager: () => import_chunk_YJAADRTG.SpaceManager,
36
+ SpaceProtocol: () => import_chunk_YJAADRTG.SpaceProtocol,
37
+ SpaceProtocolSession: () => import_chunk_YJAADRTG.SpaceProtocolSession,
38
+ TimeframeClock: () => import_chunk_YJAADRTG.TimeframeClock,
39
+ codec: () => import_chunk_YJAADRTG.codec,
40
+ createMappedFeedWriter: () => import_chunk_YJAADRTG.createMappedFeedWriter,
41
+ mapFeedIndexesToTimeframe: () => import_chunk_YJAADRTG.mapFeedIndexesToTimeframe,
42
+ mapTimeframeToFeedIndexes: () => import_chunk_YJAADRTG.mapTimeframeToFeedIndexes,
43
+ startAfter: () => import_chunk_YJAADRTG.startAfter,
44
+ valueEncoding: () => import_chunk_YJAADRTG.valueEncoding
57
45
  });
58
- module.exports = __toCommonJS(src_exports);
59
-
60
- // packages/core/echo/echo-pipeline/src/common/codec.ts
61
- var import_hypercore = require("@dxos/hypercore");
62
- var import_protocols = require("@dxos/protocols");
63
- var codec = import_protocols.schema.getCodecForType("dxos.echo.feed.FeedMessage");
64
- var valueEncoding = (0, import_hypercore.createCodecEncoding)(codec);
65
-
66
- // packages/core/echo/echo-pipeline/src/common/feeds.ts
67
- var import_invariant = require("@dxos/invariant");
68
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/common/feeds.ts";
69
- var createMappedFeedWriter = (mapper, writer) => {
70
- (0, import_invariant.invariant)(mapper, void 0, {
71
- F: __dxlog_file,
72
- L: 16,
73
- S: void 0,
74
- A: [
75
- "mapper",
76
- ""
77
- ]
78
- });
79
- (0, import_invariant.invariant)(writer, void 0, {
80
- F: __dxlog_file,
81
- L: 17,
82
- S: void 0,
83
- A: [
84
- "writer",
85
- ""
86
- ]
87
- });
88
- return {
89
- write: async (data, options) => await writer.write(await mapper(data), options)
90
- };
91
- };
92
-
93
- // packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
94
- var import_async = require("@dxos/async");
95
- var import_codec_protobuf = require("@dxos/codec-protobuf");
96
- var import_context = require("@dxos/context");
97
- var import_echo_db = require("@dxos/echo-db");
98
- var import_invariant2 = require("@dxos/invariant");
99
- var import_log = require("@dxos/log");
100
- var import_service = require("@dxos/protocols/proto/dxos/echo/service");
101
- var import_util = require("@dxos/util");
102
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
103
- var MUTATION_LIMIT_PER_OBJECT = 10;
104
- var DataServiceHost = class {
105
- constructor(_itemManager, _itemDemuxer, _flush, _writeStream, _opts = {}) {
106
- this._itemManager = _itemManager;
107
- this._itemDemuxer = _itemDemuxer;
108
- this._flush = _flush;
109
- this._writeStream = _writeStream;
110
- this._opts = _opts;
111
- this._ctx = new import_context.Context();
112
- this._clientTagMap = new import_util.ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
113
- }
114
- async open() {
115
- }
116
- async close() {
117
- await this._ctx.dispose();
118
- }
119
- get _deferEvents() {
120
- return this._opts.deferEvents ?? true;
121
- }
122
- /**
123
- * Real-time subscription to data objects in a space.
124
- */
125
- subscribe() {
126
- return new import_codec_protobuf.Stream(({ next, close, ctx }) => {
127
- ctx.onDispose(this._ctx.onDispose(close));
128
- const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
129
- next({
130
- batch: {
131
- objects
132
- }
133
- });
134
- const updateScheduler = new import_async.UpdateScheduler(ctx, async () => {
135
- flushPendingUpdate();
136
- }, {
137
- maxFrequency: 10
138
- });
139
- const pendingUpdates = [];
140
- const mutationsPerObject = /* @__PURE__ */ new Map();
141
- const clearPendingUpdates = () => {
142
- pendingUpdates.length = 0;
143
- mutationsPerObject.clear();
144
- };
145
- const flushPendingUpdate = () => {
146
- const stagedEvents = [];
147
- const objectsWithSnapshots = /* @__PURE__ */ new Set();
148
- for (const [id, count] of mutationsPerObject) {
149
- if (count >= MUTATION_LIMIT_PER_OBJECT) {
150
- objectsWithSnapshots.add(id);
151
- const entity = this._itemManager.entities.get(id);
152
- if (entity) {
153
- stagedEvents.push(entity.createSnapshot());
154
- }
155
- }
156
- }
157
- for (const obj of pendingUpdates) {
158
- if (!objectsWithSnapshots.has(obj.objectId)) {
159
- stagedEvents.push(obj);
160
- }
161
- }
162
- next({
163
- batch: {
164
- objects: stagedEvents
165
- }
166
- });
167
- clearPendingUpdates();
168
- };
169
- this._itemDemuxer.snapshot.on(ctx, (snapshot) => {
170
- clearPendingUpdates();
171
- next({
172
- action: import_service.EchoEvent.DatabaseAction.RESET,
173
- batch: {
174
- objects: snapshot.items
175
- }
176
- });
177
- });
178
- this._itemDemuxer.mutation.on(ctx, (message) => {
179
- const { batch, meta } = message;
180
- (0, import_invariant2.invariant)(!meta.clientTag, "Unexpected client tag in mutation message", {
181
- F: __dxlog_file2,
182
- L: 131,
183
- S: this,
184
- A: [
185
- "!(meta as any).clientTag",
186
- "'Unexpected client tag in mutation message'"
187
- ]
188
- });
189
- (0, import_log.log)("message", {
190
- batch: batch.objects?.length,
191
- meta
192
- }, {
193
- F: __dxlog_file2,
194
- L: 132,
195
- S: this,
196
- C: (f, a) => f(...a)
197
- });
198
- const clientTag = this._clientTagMap.get([
199
- message.meta.feedKey,
200
- message.meta.seq
201
- ]);
202
- batch.objects?.forEach((object) => {
203
- (0, import_echo_db.setMetadataOnObject)(object, {
204
- ...meta
205
- });
206
- });
207
- if (clientTag) {
208
- flushPendingUpdate();
209
- (0, import_echo_db.tagMutationsInBatch)(batch, clientTag, 0);
210
- next({
211
- clientTag,
212
- feedKey: message.meta.feedKey,
213
- seq: message.meta.seq,
214
- batch
215
- });
216
- } else {
217
- for (const obj of batch.objects ?? []) {
218
- const newCount = (mutationsPerObject.get(obj.objectId) ?? 0) + 1;
219
- mutationsPerObject.set(obj.objectId, newCount);
220
- }
221
- for (const obj of batch.objects ?? []) {
222
- if ((mutationsPerObject.get(obj.objectId) ?? 0) < MUTATION_LIMIT_PER_OBJECT) {
223
- pendingUpdates.push(obj);
224
- }
225
- }
226
- if (this._deferEvents) {
227
- updateScheduler.trigger();
228
- } else {
229
- flushPendingUpdate();
230
- }
231
- }
232
- });
233
- });
234
- }
235
- async write(request) {
236
- (0, import_invariant2.invariant)(!this._ctx.disposed, "Cannot write to closed DataServiceHost", {
237
- F: __dxlog_file2,
238
- L: 177,
239
- S: this,
240
- A: [
241
- "!this._ctx.disposed",
242
- "'Cannot write to closed DataServiceHost'"
243
- ]
244
- });
245
- (0, import_invariant2.invariant)(this._writeStream, "Cannot write mutations in readonly mode", {
246
- F: __dxlog_file2,
247
- L: 178,
248
- S: this,
249
- A: [
250
- "this._writeStream",
251
- "'Cannot write mutations in readonly mode'"
252
- ]
253
- });
254
- (0, import_log.log)("write", {
255
- clientTag: request.clientTag,
256
- objectCount: request.batch.objects?.length ?? 0
257
- }, {
258
- F: __dxlog_file2,
259
- L: 180,
260
- S: this,
261
- C: (f, a) => f(...a)
262
- });
263
- const message = createDataMessage(request.batch);
264
- const receipt = await this._writeStream.write(message, {
265
- afterWrite: async (receipt2) => {
266
- if (request.clientTag) {
267
- (0, import_log.log)("tag", {
268
- clientTag: request.clientTag,
269
- feedKey: receipt2.feedKey,
270
- seq: receipt2.seq
271
- }, {
272
- F: __dxlog_file2,
273
- L: 189,
274
- S: this,
275
- C: (f, a) => f(...a)
276
- });
277
- this._clientTagMap.set([
278
- receipt2.feedKey,
279
- receipt2.seq
280
- ], request.clientTag);
281
- }
282
- }
283
- });
284
- return receipt;
285
- }
286
- async flush() {
287
- await this._flush();
288
- }
289
- };
290
- var createDataMessage = (batch) => ({
291
- batch: {
292
- objects: batch.objects?.map((object) => ({
293
- ...object,
294
- mutations: object.mutations?.map((mutation) => ({
295
- ...mutation,
296
- meta: void 0
297
- })),
298
- meta: void 0
299
- }))
300
- }
301
- });
302
-
303
- // packages/core/echo/echo-pipeline/src/db-host/database-host.ts
304
- var import_echo_db2 = require("@dxos/echo-db");
305
- var DatabaseHost = class {
306
- constructor(_outboundStream, _flush) {
307
- this._outboundStream = _outboundStream;
308
- this._flush = _flush;
309
- }
310
- get isReadOnly() {
311
- return !!this._outboundStream;
312
- }
313
- get echoProcessor() {
314
- return this._echoProcessor;
315
- }
316
- async open(itemManager, modelFactory) {
317
- this._itemManager = itemManager;
318
- this._itemManager._debugLabel = "host";
319
- this._itemDemuxer = new import_echo_db2.ItemDemuxer(itemManager, modelFactory);
320
- this._echoProcessor = this._itemDemuxer.open();
321
- }
322
- async close() {
323
- }
324
- getWriteStream() {
325
- return this._outboundStream;
326
- }
327
- createSnapshot() {
328
- return this._itemDemuxer.createSnapshot();
329
- }
330
- createDataServiceHost(opts = {}) {
331
- return new DataServiceHost(this._itemManager, this._itemDemuxer, this._flush, this._outboundStream ?? void 0, opts);
332
- }
333
- };
334
-
335
- // packages/core/echo/echo-pipeline/src/db-host/snapshot-manager.ts
336
- var import_context2 = require("@dxos/context");
337
- var import_keys = require("@dxos/keys");
338
- var import_protocols2 = require("@dxos/protocols");
339
- var import_blob = require("@dxos/protocols/proto/dxos/echo/blob");
340
- var SpaceSnapshot = import_protocols2.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
341
- var SnapshotManager = class {
342
- constructor(_snapshotStore, _blobStore, _blobSync) {
343
- this._snapshotStore = _snapshotStore;
344
- this._blobStore = _blobStore;
345
- this._blobSync = _blobSync;
346
- }
347
- async _getBlob(blobId) {
348
- const blob = await this._blobStore.get(blobId);
349
- return SpaceSnapshot.decode(blob);
350
- }
351
- async load(ctx, id) {
352
- const blobId = import_keys.PublicKey.fromHex(id).asUint8Array();
353
- const blobMeta = await this._blobStore.getMeta(blobId);
354
- if (blobMeta && blobMeta.state === import_blob.BlobMeta.State.FULLY_PRESENT) {
355
- return this._getBlob(blobId);
356
- }
357
- const fallbackStore = await (0, import_context2.cancelWithContext)(ctx, this._snapshotStore.loadSnapshot(id));
358
- if (fallbackStore) {
359
- return fallbackStore;
360
- }
361
- await this._blobSync.download(ctx, blobId);
362
- return this._getBlob(blobId);
363
- }
364
- async store(snapshot) {
365
- const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
366
- await this._blobSync.notifyBlobAdded(id);
367
- return import_keys.PublicKey.from(id).toHex();
368
- }
369
- };
370
-
371
- // packages/core/echo/echo-pipeline/src/db-host/snapshot-store.ts
372
- var import_crypto = require("@dxos/crypto");
373
- var import_protocols3 = require("@dxos/protocols");
374
- var SpaceSnapshot2 = import_protocols3.schema.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
375
- var SnapshotStore = class {
376
- constructor(_directory) {
377
- this._directory = _directory;
378
- }
379
- async saveSnapshot(snapshot) {
380
- const encoded = SpaceSnapshot2.encode(snapshot);
381
- const key = await import_crypto.subtleCrypto.digest("SHA-256", encoded);
382
- const keyString = Buffer.from(key).toString("hex");
383
- const file = await this._directory.getOrCreateFile(keyString);
384
- try {
385
- await file.write(0, Buffer.from(encoded));
386
- } finally {
387
- await file.close();
388
- }
389
- return keyString;
390
- }
391
- async loadSnapshot(key) {
392
- const file = await this._directory.getOrCreateFile(key);
393
- try {
394
- const { size } = await file.stat();
395
- if (size === 0) {
396
- return void 0;
397
- }
398
- const buffer = await file.read(0, size);
399
- return SpaceSnapshot2.decode(buffer);
400
- } finally {
401
- await file.close();
402
- }
403
- }
404
- async listSnapshots() {
405
- const entries = await this._directory.list();
406
- return await Promise.all(entries.map(async (key) => {
407
- const { size } = await this._directory.getOrCreateFile(key).stat();
408
- return {
409
- key,
410
- size
411
- };
412
- }));
413
- }
414
- };
415
-
416
- // packages/core/echo/echo-pipeline/src/db-host/data-service.ts
417
- var import_debug = require("@dxos/debug");
418
- var import_invariant3 = require("@dxos/invariant");
419
- var import_keys2 = require("@dxos/keys");
420
- var import_log2 = require("@dxos/log");
421
- var import_util2 = require("@dxos/util");
422
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
423
- var DataServiceSubscriptions = class {
424
- constructor() {
425
- this._spaces = new import_util2.ComplexMap(import_keys2.PublicKey.hash);
426
- }
427
- clear() {
428
- this._spaces.clear();
429
- }
430
- async registerSpace(spaceKey, host) {
431
- (0, import_log2.log)("Registering space", {
432
- spaceKey
433
- }, {
434
- F: __dxlog_file3,
435
- L: 31,
436
- S: this,
437
- C: (f, a) => f(...a)
438
- });
439
- (0, import_invariant3.invariant)(!this._spaces.has(spaceKey), void 0, {
440
- F: __dxlog_file3,
441
- L: 32,
442
- S: this,
443
- A: [
444
- "!this._spaces.has(spaceKey)",
445
- ""
446
- ]
447
- });
448
- await host.open();
449
- this._spaces.set(spaceKey, host);
450
- }
451
- async unregisterSpace(spaceKey) {
452
- (0, import_log2.log)("Unregistering space", {
453
- spaceKey
454
- }, {
455
- F: __dxlog_file3,
456
- L: 38,
457
- S: this,
458
- C: (f, a) => f(...a)
459
- });
460
- const host = this._spaces.get(spaceKey);
461
- await host?.close();
462
- this._spaces.delete(spaceKey);
463
- }
464
- getDataService(spaceKey) {
465
- return this._spaces.get(spaceKey);
466
- }
467
- };
468
- var DataServiceImpl = class {
469
- constructor(_subscriptions) {
470
- this._subscriptions = _subscriptions;
471
- }
472
- subscribe(request) {
473
- (0, import_invariant3.invariant)(request.spaceKey, void 0, {
474
- F: __dxlog_file3,
475
- L: 57,
476
- S: this,
477
- A: [
478
- "request.spaceKey",
479
- ""
480
- ]
481
- });
482
- const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
483
- return host.subscribe();
484
- }
485
- write(request) {
486
- (0, import_invariant3.invariant)(request.spaceKey, void 0, {
487
- F: __dxlog_file3,
488
- L: 64,
489
- S: this,
490
- A: [
491
- "request.spaceKey",
492
- ""
493
- ]
494
- });
495
- (0, import_invariant3.invariant)(request.batch, void 0, {
496
- F: __dxlog_file3,
497
- L: 65,
498
- S: this,
499
- A: [
500
- "request.batch",
501
- ""
502
- ]
503
- });
504
- const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
505
- return host.write(request);
506
- }
507
- flush(request) {
508
- (0, import_invariant3.invariant)(request.spaceKey, void 0, {
509
- F: __dxlog_file3,
510
- L: 72,
511
- S: this,
512
- A: [
513
- "request.spaceKey",
514
- ""
515
- ]
516
- });
517
- const host = this._subscriptions.getDataService(request.spaceKey) ?? (0, import_debug.raise)(new Error(`space not found: ${request.spaceKey}`));
518
- return host.flush();
519
- }
520
- };
521
-
522
- // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
523
- var import_crc_32 = __toESM(require("crc-32"));
524
- var import_async2 = require("@dxos/async");
525
- var import_invariant4 = require("@dxos/invariant");
526
- var import_keys3 = require("@dxos/keys");
527
- var import_log3 = require("@dxos/log");
528
- var import_protocols4 = require("@dxos/protocols");
529
- var import_services = require("@dxos/protocols/proto/dxos/client/services");
530
- var import_util3 = require("@dxos/util");
531
- function _ts_decorate(decorators, target, key, desc) {
532
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
533
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
534
- r = Reflect.decorate(decorators, target, key, desc);
535
- else
536
- for (var i = decorators.length - 1; i >= 0; i--)
537
- if (d = decorators[i])
538
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
539
- return c > 3 && r && Object.defineProperty(target, key, r), r;
540
- }
541
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
542
- var emptyEchoMetadata = () => ({
543
- version: import_protocols4.STORAGE_VERSION,
544
- spaces: [],
545
- created: /* @__PURE__ */ new Date(),
546
- updated: /* @__PURE__ */ new Date()
547
- });
548
- var emptyLargeSpaceMetadata = () => ({});
549
- var EchoMetadata = import_protocols4.schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
550
- var LargeSpaceMetadata = import_protocols4.schema.getCodecForType("dxos.echo.metadata.LargeSpaceMetadata");
551
- var MetadataStore = class {
552
- constructor(_directory) {
553
- this._directory = _directory;
554
- this._metadata = emptyEchoMetadata();
555
- this._spaceLargeMetadata = new import_util3.ComplexMap(import_keys3.PublicKey.hash);
556
- this._metadataFile = void 0;
557
- this.update = new import_async2.Event();
558
- }
559
- get metadata() {
560
- return this._metadata;
561
- }
562
- get version() {
563
- return this._metadata.version ?? 0;
564
- }
565
- /**
566
- * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
567
- * addSpaceFeed functions.
568
- */
569
- get spaces() {
570
- return this._metadata.spaces ?? [];
571
- }
572
- async _readFile(file, codec2) {
573
- try {
574
- const { size: fileLength } = await file.stat();
575
- if (fileLength < 8) {
576
- return;
577
- }
578
- const dataSize = fromBytesInt32(await file.read(0, 4));
579
- const checksum = fromBytesInt32(await file.read(4, 4));
580
- (0, import_log3.log)("loaded", {
581
- size: dataSize,
582
- checksum,
583
- name: file.filename
584
- }, {
585
- F: __dxlog_file4,
586
- L: 78,
587
- S: this,
588
- C: (f, a) => f(...a)
589
- });
590
- if (fileLength < dataSize + 8) {
591
- throw new import_protocols4.DataCorruptionError("Metadata size is smaller than expected.", {
592
- fileLength,
593
- dataSize
594
- });
595
- }
596
- const data = await file.read(8, dataSize);
597
- const calculatedChecksum = import_crc_32.default.buf(data);
598
- if (calculatedChecksum !== checksum) {
599
- throw new import_protocols4.DataCorruptionError("Metadata checksum is invalid.");
600
- }
601
- return codec2.decode(data);
602
- } finally {
603
- await file.close();
604
- }
605
- }
606
- async _writeFile(file, codec2, data) {
607
- const encoded = (0, import_util3.arrayToBuffer)(codec2.encode(data));
608
- const checksum = import_crc_32.default.buf(encoded);
609
- const result = Buffer.alloc(8 + encoded.length);
610
- result.writeInt32LE(encoded.length, 0);
611
- result.writeInt32LE(checksum, 4);
612
- encoded.copy(result, 8);
613
- await file.write(0, result);
614
- (0, import_log3.log)("saved", {
615
- size: encoded.length,
616
- checksum
617
- }, {
618
- F: __dxlog_file4,
619
- L: 110,
620
- S: this,
621
- C: (f, a) => f(...a)
622
- });
623
- }
624
- async close() {
625
- await this.flush();
626
- await this._metadataFile?.close();
627
- this._metadataFile = void 0;
628
- this._metadata = emptyEchoMetadata();
629
- this._spaceLargeMetadata.clear();
630
- }
631
- /**
632
- * Loads metadata from persistent storage.
633
- */
634
- async load() {
635
- if (!this._metadataFile || this._metadataFile.closed) {
636
- this._metadataFile = this._directory.getOrCreateFile("EchoMetadata");
637
- }
638
- try {
639
- const metadata = await this._readFile(this._metadataFile, EchoMetadata);
640
- if (metadata) {
641
- this._metadata = metadata;
642
- }
643
- this._metadata.spaces?.forEach((space) => {
644
- space.state ??= import_services.SpaceState.ACTIVE;
645
- });
646
- } catch (err) {
647
- import_log3.log.error("failed to load metadata", {
648
- err
649
- }, {
650
- F: __dxlog_file4,
651
- L: 141,
652
- S: this,
653
- C: (f, a) => f(...a)
654
- });
655
- this._metadata = emptyEchoMetadata();
656
- }
657
- await (0, import_util3.forEachAsync)([
658
- this._metadata.identity?.haloSpace.key,
659
- ...this._metadata.spaces?.map((space) => space.key) ?? []
660
- ].filter(import_util3.isNotNullOrUndefined), async (key) => {
661
- try {
662
- await this._loadSpaceLargeMetadata(key);
663
- } catch (err) {
664
- import_log3.log.error("failed to load space large metadata", {
665
- err
666
- }, {
667
- F: __dxlog_file4,
668
- L: 153,
669
- S: this,
670
- C: (f, a) => f(...a)
671
- });
672
- }
673
- });
674
- }
675
- async _save() {
676
- const data = {
677
- ...this._metadata,
678
- version: import_protocols4.STORAGE_VERSION,
679
- created: this._metadata.created ?? /* @__PURE__ */ new Date(),
680
- updated: /* @__PURE__ */ new Date()
681
- };
682
- this.update.emit(data);
683
- const file = this._directory.getOrCreateFile("EchoMetadata");
684
- await this._writeFile(file, EchoMetadata, data);
685
- }
686
- async _loadSpaceLargeMetadata(key) {
687
- const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
688
- try {
689
- const metadata = await this._readFile(file, LargeSpaceMetadata);
690
- if (metadata) {
691
- this._spaceLargeMetadata.set(key, metadata);
692
- }
693
- } catch (err) {
694
- import_log3.log.error("failed to load space large metadata", {
695
- err
696
- }, {
697
- F: __dxlog_file4,
698
- L: 182,
699
- S: this,
700
- C: (f, a) => f(...a)
701
- });
702
- }
703
- }
704
- async _saveSpaceLargeMetadata(key) {
705
- const data = this._getLargeSpaceMetadata(key);
706
- const file = this._directory.getOrCreateFile(`space_${key.toHex()}_large`);
707
- await this._writeFile(file, LargeSpaceMetadata, data);
708
- }
709
- async flush() {
710
- await this._directory.flush();
711
- }
712
- _getSpace(spaceKey) {
713
- if (this._metadata.identity?.haloSpace.key.equals(spaceKey)) {
714
- return this._metadata.identity.haloSpace;
715
- }
716
- const space = this.spaces.find((space2) => space2.key === spaceKey);
717
- (0, import_invariant4.invariant)(space, "Space not found", {
718
- F: __dxlog_file4,
719
- L: 204,
720
- S: this,
721
- A: [
722
- "space",
723
- "'Space not found'"
724
- ]
725
- });
726
- return space;
727
- }
728
- _getLargeSpaceMetadata(key) {
729
- let entry = this._spaceLargeMetadata.get(key);
730
- if (entry) {
731
- return entry;
732
- }
733
- entry = emptyLargeSpaceMetadata();
734
- this._spaceLargeMetadata.set(key, entry);
735
- return entry;
736
- }
737
- /**
738
- * Clears storage - doesn't work for now.
739
- */
740
- async clear() {
741
- (0, import_log3.log)("clearing all metadata", void 0, {
742
- F: __dxlog_file4,
743
- L: 223,
744
- S: this,
745
- C: (f, a) => f(...a)
746
- });
747
- await this._directory.delete();
748
- this._metadata = emptyEchoMetadata();
749
- }
750
- getIdentityRecord() {
751
- return this._metadata.identity;
752
- }
753
- async setIdentityRecord(record) {
754
- (0, import_invariant4.invariant)(!this._metadata.identity, "Cannot overwrite existing identity in metadata", {
755
- F: __dxlog_file4,
756
- L: 233,
757
- S: this,
758
- A: [
759
- "!this._metadata.identity",
760
- "'Cannot overwrite existing identity in metadata'"
761
- ]
762
- });
763
- this._metadata.identity = record;
764
- await this._save();
765
- await this.flush();
766
- }
767
- async addSpace(record) {
768
- (0, import_invariant4.invariant)(!(this._metadata.spaces ?? []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata", {
769
- F: __dxlog_file4,
770
- L: 241,
771
- S: this,
772
- A: [
773
- "!(this._metadata.spaces ?? []).find((space) => space.key === record.key)",
774
- "'Cannot overwrite existing space in metadata'"
775
- ]
776
- });
777
- (this._metadata.spaces ??= []).push(record);
778
- await this._save();
779
- await this.flush();
780
- }
781
- async setSpaceDataLatestTimeframe(spaceKey, timeframe) {
782
- this._getSpace(spaceKey).dataTimeframe = timeframe;
783
- await this._save();
784
- }
785
- async setSpaceControlLatestTimeframe(spaceKey, timeframe) {
786
- this._getSpace(spaceKey).controlTimeframe = timeframe;
787
- await this._save();
788
- await this.flush();
789
- }
790
- async setCache(spaceKey, cache) {
791
- this._getSpace(spaceKey).cache = cache;
792
- await this._save();
793
- }
794
- async setWritableFeedKeys(spaceKey, controlFeedKey, dataFeedKey) {
795
- const space = this._getSpace(spaceKey);
796
- space.controlFeedKey = controlFeedKey;
797
- space.dataFeedKey = dataFeedKey;
798
- await this._save();
799
- await this.flush();
800
- }
801
- async setSpaceState(spaceKey, state) {
802
- this._getSpace(spaceKey).state = state;
803
- await this._save();
804
- await this.flush();
805
- }
806
- getSpaceControlPipelineSnapshot(spaceKey) {
807
- return this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot;
808
- }
809
- async setSpaceControlPipelineSnapshot(spaceKey, snapshot) {
810
- this._getLargeSpaceMetadata(spaceKey).controlPipelineSnapshot = snapshot;
811
- await this._saveSpaceLargeMetadata(spaceKey);
812
- await this.flush();
813
- }
814
- };
815
- _ts_decorate([
816
- import_async2.synchronized
817
- ], MetadataStore.prototype, "load", null);
818
- _ts_decorate([
819
- import_async2.synchronized
820
- ], MetadataStore.prototype, "_save", null);
821
- _ts_decorate([
822
- import_async2.synchronized
823
- ], MetadataStore.prototype, "_saveSpaceLargeMetadata", null);
824
- var fromBytesInt32 = (buf) => buf.readInt32LE(0);
825
-
826
- // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
827
- var import_async4 = require("@dxos/async");
828
- var import_context3 = require("@dxos/context");
829
- var import_debug3 = require("@dxos/debug");
830
- var import_feed_store = require("@dxos/feed-store");
831
- var import_invariant6 = require("@dxos/invariant");
832
- var import_keys4 = require("@dxos/keys");
833
- var import_log6 = require("@dxos/log");
834
- var import_timeframe2 = require("@dxos/timeframe");
835
- var import_util4 = require("@dxos/util");
836
-
837
- // packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
838
- var import_invariant5 = require("@dxos/invariant");
839
- var import_log4 = require("@dxos/log");
840
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
841
- var createMessageSelector = (timeframeClock) => {
842
- return (messages) => {
843
- for (let i = 0; i < messages.length; i++) {
844
- const { data: { timeframe } } = messages[i];
845
- (0, import_invariant5.invariant)(timeframe, void 0, {
846
- F: __dxlog_file5,
847
- L: 25,
848
- S: void 0,
849
- A: [
850
- "timeframe",
851
- ""
852
- ]
853
- });
854
- if (!timeframeClock.hasGaps(timeframe)) {
855
- return i;
856
- }
857
- }
858
- (0, import_log4.log)("Skipping...", void 0, {
859
- F: __dxlog_file5,
860
- L: 33,
861
- S: void 0,
862
- C: (f, a) => f(...a)
863
- });
864
- };
865
- };
866
-
867
- // packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
868
- var import_async3 = require("@dxos/async");
869
- var import_debug2 = require("@dxos/debug");
870
- var import_log5 = require("@dxos/log");
871
- var import_timeframe = require("@dxos/timeframe");
872
- function _ts_decorate2(decorators, target, key, desc) {
873
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
874
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
875
- r = Reflect.decorate(decorators, target, key, desc);
876
- else
877
- for (var i = decorators.length - 1; i >= 0; i--)
878
- if (d = decorators[i])
879
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
880
- return c > 3 && r && Object.defineProperty(target, key, r), r;
881
- }
882
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
883
- var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
884
- feedKey,
885
- index
886
- }));
887
- var mapFeedIndexesToTimeframe = (indexes) => new import_timeframe.Timeframe(indexes.map(({ feedKey, index }) => [
888
- feedKey,
889
- index
890
- ]));
891
- var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
892
- feedKey,
893
- index: index + 1
894
- }));
895
- var TimeframeClock = class {
896
- constructor(_timeframe = new import_timeframe.Timeframe()) {
897
- this._timeframe = _timeframe;
898
- this.update = new import_async3.Event();
899
- this._pendingTimeframe = _timeframe;
900
- }
901
- /**
902
- * Timeframe that was processed by ECHO.
903
- */
904
- get timeframe() {
905
- return this._timeframe;
906
- }
907
- /**
908
- * Timeframe that is currently being processed by ECHO.
909
- * Will be equal to `timeframe` after the processing is complete.
910
- */
911
- get pendingTimeframe() {
912
- return this._pendingTimeframe;
913
- }
914
- setTimeframe(timeframe) {
915
- this._timeframe = timeframe;
916
- this._pendingTimeframe = timeframe;
917
- this.update.emit(this._timeframe);
918
- }
919
- updatePendingTimeframe(key, seq) {
920
- this._pendingTimeframe = import_timeframe.Timeframe.merge(this._pendingTimeframe, new import_timeframe.Timeframe([
921
- [
922
- key,
923
- seq
924
- ]
925
- ]));
926
- }
927
- updateTimeframe() {
928
- this._timeframe = this._pendingTimeframe;
929
- this.update.emit(this._timeframe);
930
- }
931
- hasGaps(timeframe) {
932
- const gaps = import_timeframe.Timeframe.dependencies(timeframe, this._timeframe);
933
- return !gaps.isEmpty();
934
- }
935
- async waitUntilReached(target) {
936
- (0, import_log5.log)("waitUntilReached", {
937
- target,
938
- current: this._timeframe
939
- }, {
940
- F: __dxlog_file6,
941
- L: 70,
942
- S: this,
943
- C: (f, a) => f(...a)
944
- });
945
- await this.update.waitForCondition(() => {
946
- (0, import_log5.log)("check if reached", {
947
- target,
948
- current: this._timeframe,
949
- deps: import_timeframe.Timeframe.dependencies(target, this._timeframe)
950
- }, {
951
- F: __dxlog_file6,
952
- L: 72,
953
- S: this,
954
- C: (f, a) => f(...a)
955
- });
956
- return import_timeframe.Timeframe.dependencies(target, this._timeframe).isEmpty();
957
- });
958
- }
959
- };
960
- _ts_decorate2([
961
- (0, import_debug2.timed)(5e3)
962
- ], TimeframeClock.prototype, "waitUntilReached", null);
963
-
964
- // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
965
- function _ts_decorate3(decorators, target, key, desc) {
966
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
967
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
968
- r = Reflect.decorate(decorators, target, key, desc);
969
- else
970
- for (var i = decorators.length - 1; i >= 0; i--)
971
- if (d = decorators[i])
972
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
973
- return c > 3 && r && Object.defineProperty(target, key, r), r;
974
- }
975
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
976
- var PipelineState = class {
977
- constructor(_feeds, _timeframeClock) {
978
- this._feeds = _feeds;
979
- this._timeframeClock = _timeframeClock;
980
- this._ctx = new import_context3.Context();
981
- this.timeframeUpdate = this._timeframeClock.update;
982
- this.stalled = new import_async4.Event();
983
- this._startTimeframe = new import_timeframe2.Timeframe();
984
- this._reachedTarget = false;
985
- }
986
- /**
987
- * Latest theoretical timeframe based on the last mutation in each feed.
988
- * NOTE: This might never be reached if the mutation dependencies
989
- */
990
- // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
991
- get endTimeframe() {
992
- return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
993
- feedKey: feed.key,
994
- index: feed.length - 1
995
- })));
996
- }
997
- get startTimeframe() {
998
- return this._startTimeframe;
999
- }
1000
- get timeframe() {
1001
- return this._timeframeClock.timeframe;
1002
- }
1003
- get pendingTimeframe() {
1004
- return this._timeframeClock.pendingTimeframe;
1005
- }
1006
- get targetTimeframe() {
1007
- return this._targetTimeframe ? this._targetTimeframe : new import_timeframe2.Timeframe();
1008
- }
1009
- get reachedTarget() {
1010
- return this._reachedTarget;
1011
- }
1012
- get feeds() {
1013
- return Array.from(this._feeds.values());
1014
- }
1015
- async waitUntilTimeframe(target) {
1016
- await this._timeframeClock.waitUntilReached(target);
1017
- }
1018
- setTargetTimeframe(target) {
1019
- this._targetTimeframe = target;
1020
- }
1021
- /**
1022
- * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
1023
- *
1024
- * This function will resolve immediately if the pipeline is stalled.
1025
- *
1026
- * @param timeout Timeout in milliseconds to specify the maximum wait time.
1027
- */
1028
- async waitUntilReachedTargetTimeframe({ ctx = new import_context3.Context(), timeout, breakOnStall = true } = {}) {
1029
- (0, import_log6.log)("waitUntilReachedTargetTimeframe", {
1030
- timeout,
1031
- current: this.timeframe,
1032
- target: this.targetTimeframe
1033
- }, {
1034
- F: __dxlog_file7,
1035
- L: 133,
1036
- S: this,
1037
- C: (f, a) => f(...a)
1038
- });
1039
- this._reachedTargetPromise ??= Promise.race([
1040
- this._timeframeClock.update.waitForCondition(() => {
1041
- return import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
1042
- }),
1043
- ...breakOnStall ? [
1044
- this.stalled.discardParameter().waitForCount(1)
1045
- ] : []
1046
- ]);
1047
- let done = false;
1048
- if (timeout) {
1049
- return Promise.race([
1050
- (0, import_context3.rejectOnDispose)(ctx),
1051
- (0, import_context3.rejectOnDispose)(this._ctx),
1052
- this._reachedTargetPromise.then(() => {
1053
- done = true;
1054
- this._reachedTarget = true;
1055
- }),
1056
- (0, import_async4.sleepWithContext)(this._ctx, timeout).then(() => {
1057
- if (done) {
1058
- return;
1059
- }
1060
- import_log6.log.warn("waitUntilReachedTargetTimeframe timed out", {
1061
- timeout,
1062
- current: this.timeframe,
1063
- target: this.targetTimeframe,
1064
- dependencies: import_timeframe2.Timeframe.dependencies(this.targetTimeframe, this.timeframe)
1065
- }, {
1066
- F: __dxlog_file7,
1067
- L: 161,
1068
- S: this,
1069
- C: (f, a) => f(...a)
1070
- });
1071
- })
1072
- ]);
1073
- } else {
1074
- return this._reachedTargetPromise;
1075
- }
1076
- }
1077
- };
1078
- var Pipeline = class {
1079
- constructor() {
1080
- this._timeframeClock = new TimeframeClock(new import_timeframe2.Timeframe());
1081
- this._feeds = new import_util4.ComplexMap(import_keys4.PublicKey.hash);
1082
- // External state accessor.
1083
- this._state = new PipelineState(this._feeds, this._timeframeClock);
1084
- // Waits for the message consumer to process the message and yield control back to the pipeline.
1085
- this._processingTrigger = new import_async4.Trigger().wake();
1086
- this._pauseTrigger = new import_async4.Trigger().wake();
1087
- // Pending downloads.
1088
- this._downloads = new import_util4.ComplexMap((value) => import_keys4.PublicKey.hash(value.key));
1089
- this._isStopping = false;
1090
- this._isStarted = false;
1091
- this._isBeingConsumed = false;
1092
- this._isPaused = false;
1093
- }
1094
- get state() {
1095
- return this._state;
1096
- }
1097
- get writer() {
1098
- (0, import_invariant6.invariant)(this._writer, "Writer not set.", {
1099
- F: __dxlog_file7,
1100
- L: 242,
1101
- S: this,
1102
- A: [
1103
- "this._writer",
1104
- "'Writer not set.'"
1105
- ]
1106
- });
1107
- return this._writer;
1108
- }
1109
- hasFeed(feedKey) {
1110
- return this._feeds.has(feedKey);
1111
- }
1112
- getFeeds() {
1113
- return this._feedSetIterator.feeds;
1114
- }
1115
- // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
1116
- // which might be opening feeds during the mutation processing, which w
1117
- async addFeed(feed) {
1118
- this._feeds.set(feed.key, feed);
1119
- if (this._feedSetIterator) {
1120
- await this._feedSetIterator.addFeed(feed);
1121
- }
1122
- if (this._isStarted && !this._isPaused) {
1123
- this._setFeedDownloadState(feed);
1124
- }
1125
- }
1126
- setWriteFeed(feed) {
1127
- (0, import_invariant6.invariant)(!this._writer, "Writer already set.", {
1128
- F: __dxlog_file7,
1129
- L: 269,
1130
- S: this,
1131
- A: [
1132
- "!this._writer",
1133
- "'Writer already set.'"
1134
- ]
1135
- });
1136
- (0, import_invariant6.invariant)(feed.properties.writable, "Feed must be writable.", {
1137
- F: __dxlog_file7,
1138
- L: 270,
1139
- S: this,
1140
- A: [
1141
- "feed.properties.writable",
1142
- "'Feed must be writable.'"
1143
- ]
1144
- });
1145
- this._writer = createMappedFeedWriter((payload) => ({
1146
- timeframe: this._timeframeClock.timeframe,
1147
- payload
1148
- }), feed.createFeedWriter());
1149
- }
1150
- async start() {
1151
- (0, import_invariant6.invariant)(!this._isStarted, "Pipeline is already started.", {
1152
- F: __dxlog_file7,
1153
- L: 283,
1154
- S: this,
1155
- A: [
1156
- "!this._isStarted",
1157
- "'Pipeline is already started.'"
1158
- ]
1159
- });
1160
- (0, import_log6.log)("starting...", void 0, {
1161
- F: __dxlog_file7,
1162
- L: 284,
1163
- S: this,
1164
- C: (f, a) => f(...a)
1165
- });
1166
- await this._initIterator();
1167
- await this._feedSetIterator.open();
1168
- this._isStarted = true;
1169
- (0, import_log6.log)("started", void 0, {
1170
- F: __dxlog_file7,
1171
- L: 288,
1172
- S: this,
1173
- C: (f, a) => f(...a)
1174
- });
1175
- if (!this._isPaused) {
1176
- for (const feed of this._feeds.values()) {
1177
- this._setFeedDownloadState(feed);
1178
- }
1179
- }
1180
- }
1181
- async stop() {
1182
- (0, import_log6.log)("stopping...", void 0, {
1183
- F: __dxlog_file7,
1184
- L: 299,
1185
- S: this,
1186
- C: (f, a) => f(...a)
1187
- });
1188
- this._isStopping = true;
1189
- for (const [feed, handle] of this._downloads.entries()) {
1190
- feed.undownload(handle);
1191
- }
1192
- this._downloads.clear();
1193
- await this._feedSetIterator?.close();
1194
- await this._processingTrigger.wait();
1195
- await this._state._ctx.dispose();
1196
- this._state._ctx = new import_context3.Context();
1197
- this._state._reachedTargetPromise = void 0;
1198
- this._state._reachedTarget = false;
1199
- this._isStarted = false;
1200
- (0, import_log6.log)("stopped", void 0, {
1201
- F: __dxlog_file7,
1202
- L: 312,
1203
- S: this,
1204
- C: (f, a) => f(...a)
1205
- });
1206
- }
1207
- /**
1208
- * @param timeframe Timeframe of already processed messages.
1209
- * The pipeline will start processing messages AFTER this timeframe.
1210
- */
1211
- async setCursor(timeframe) {
1212
- (0, import_invariant6.invariant)(!this._isStarted || this._isPaused, "Invalid state.", {
1213
- F: __dxlog_file7,
1214
- L: 321,
1215
- S: this,
1216
- A: [
1217
- "!this._isStarted || this._isPaused",
1218
- "'Invalid state.'"
1219
- ]
1220
- });
1221
- this._state._startTimeframe = timeframe;
1222
- this._timeframeClock.setTimeframe(timeframe);
1223
- if (this._feedSetIterator) {
1224
- await this._feedSetIterator.close();
1225
- await this._initIterator();
1226
- await this._feedSetIterator.open();
1227
- }
1228
- }
1229
- /**
1230
- * Calling pause while processing will cause a deadlock.
1231
- */
1232
- async pause() {
1233
- if (this._isPaused) {
1234
- return;
1235
- }
1236
- this._pauseTrigger.reset();
1237
- await this._processingTrigger.wait();
1238
- this._isPaused = true;
1239
- }
1240
- async unpause() {
1241
- (0, import_invariant6.invariant)(this._isPaused, "Pipeline is not paused.", {
1242
- F: __dxlog_file7,
1243
- L: 350,
1244
- S: this,
1245
- A: [
1246
- "this._isPaused",
1247
- "'Pipeline is not paused.'"
1248
- ]
1249
- });
1250
- this._pauseTrigger.wake();
1251
- this._isPaused = false;
1252
- for (const feed of this._feeds.values()) {
1253
- this._setFeedDownloadState(feed);
1254
- }
1255
- }
1256
- /**
1257
- * Starts to iterate over the ordered messages from the added feeds.
1258
- * Updates the timeframe clock after the message has bee processed.
1259
- */
1260
- async *consume() {
1261
- (0, import_invariant6.invariant)(!this._isBeingConsumed, "Pipeline is already being consumed.", {
1262
- F: __dxlog_file7,
1263
- L: 365,
1264
- S: this,
1265
- A: [
1266
- "!this._isBeingConsumed",
1267
- "'Pipeline is already being consumed.'"
1268
- ]
1269
- });
1270
- this._isBeingConsumed = true;
1271
- (0, import_invariant6.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1272
- F: __dxlog_file7,
1273
- L: 368,
1274
- S: this,
1275
- A: [
1276
- "this._feedSetIterator",
1277
- "'Iterator not initialized.'"
1278
- ]
1279
- });
1280
- let lastFeedSetIterator = this._feedSetIterator;
1281
- let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1282
- while (!this._isStopping) {
1283
- await this._pauseTrigger.wait();
1284
- if (lastFeedSetIterator !== this._feedSetIterator) {
1285
- (0, import_invariant6.invariant)(this._feedSetIterator, "Iterator not initialized.", {
1286
- F: __dxlog_file7,
1287
- L: 377,
1288
- S: this,
1289
- A: [
1290
- "this._feedSetIterator",
1291
- "'Iterator not initialized.'"
1292
- ]
1293
- });
1294
- lastFeedSetIterator = this._feedSetIterator;
1295
- iterable = lastFeedSetIterator[Symbol.asyncIterator]();
1296
- }
1297
- const { done, value } = await iterable.next();
1298
- if (!done) {
1299
- const block = value ?? (0, import_debug3.failUndefined)();
1300
- this._processingTrigger.reset();
1301
- this._timeframeClock.updatePendingTimeframe(import_keys4.PublicKey.from(block.feedKey), block.seq);
1302
- yield block;
1303
- this._processingTrigger.wake();
1304
- this._timeframeClock.updateTimeframe();
1305
- }
1306
- }
1307
- this._isBeingConsumed = false;
1308
- }
1309
- _setFeedDownloadState(feed) {
1310
- let handle = this._downloads.get(feed);
1311
- if (handle) {
1312
- feed.undownload(handle);
1313
- }
1314
- const timeframe = this._state._startTimeframe;
1315
- const seq = timeframe.get(feed.key) ?? -1;
1316
- (0, import_log6.log)("download", {
1317
- feed: feed.key.truncate(),
1318
- seq,
1319
- length: feed.length
1320
- }, {
1321
- F: __dxlog_file7,
1322
- L: 406,
1323
- S: this,
1324
- C: (f, a) => f(...a)
1325
- });
1326
- handle = feed.download({
1327
- start: seq + 1,
1328
- linear: true
1329
- }, (err, data) => {
1330
- if (err) {
1331
- } else {
1332
- import_log6.log.info("downloaded", {
1333
- data
1334
- }, {
1335
- F: __dxlog_file7,
1336
- L: 411,
1337
- S: this,
1338
- C: (f, a) => f(...a)
1339
- });
1340
- }
1341
- });
1342
- this._downloads.set(feed, handle);
1343
- }
1344
- async _initIterator() {
1345
- this._feedSetIterator = new import_feed_store.FeedSetIterator(createMessageSelector(this._timeframeClock), {
1346
- start: startAfter(this._timeframeClock.timeframe),
1347
- stallTimeout: 1e3
1348
- });
1349
- this._feedSetIterator.stalled.on((iterator) => {
1350
- import_log6.log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
1351
- F: __dxlog_file7,
1352
- L: 425,
1353
- S: this,
1354
- C: (f, a) => f(...a)
1355
- });
1356
- this._state.stalled.emit();
1357
- });
1358
- for (const feed of this._feeds.values()) {
1359
- await this._feedSetIterator.addFeed(feed);
1360
- }
1361
- }
1362
- };
1363
- _ts_decorate3([
1364
- import_async4.synchronized
1365
- ], Pipeline.prototype, "start", null);
1366
- _ts_decorate3([
1367
- import_async4.synchronized
1368
- ], Pipeline.prototype, "stop", null);
1369
- _ts_decorate3([
1370
- import_async4.synchronized
1371
- ], Pipeline.prototype, "setCursor", null);
1372
- _ts_decorate3([
1373
- import_async4.synchronized
1374
- ], Pipeline.prototype, "pause", null);
1375
- _ts_decorate3([
1376
- import_async4.synchronized
1377
- ], Pipeline.prototype, "unpause", null);
1378
-
1379
- // packages/core/echo/echo-pipeline/src/space/auth.ts
1380
- var import_async5 = require("@dxos/async");
1381
- var import_context4 = require("@dxos/context");
1382
- var import_crypto2 = require("@dxos/crypto");
1383
- var import_invariant7 = require("@dxos/invariant");
1384
- var import_log7 = require("@dxos/log");
1385
- var import_protocols5 = require("@dxos/protocols");
1386
- var import_teleport = require("@dxos/teleport");
1387
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
1388
- var AuthExtension = class extends import_teleport.RpcExtension {
1389
- constructor(_authParams) {
1390
- super({
1391
- requested: {
1392
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1393
- },
1394
- exposed: {
1395
- AuthService: import_protocols5.schema.getService("dxos.mesh.teleport.auth.AuthService")
1396
- },
1397
- timeout: 60 * 1e3
1398
- });
1399
- this._authParams = _authParams;
1400
- this._ctx = new import_context4.Context({
1401
- onError: (err) => {
1402
- import_log7.log.catch(err, void 0, {
1403
- F: __dxlog_file8,
1404
- L: 28,
1405
- S: this,
1406
- C: (f, a) => f(...a)
1407
- });
1408
- }
1409
- });
1410
- }
1411
- async getHandlers() {
1412
- return {
1413
- AuthService: {
1414
- authenticate: async ({ challenge }) => {
1415
- try {
1416
- const credential = await this._authParams.provider(challenge);
1417
- if (!credential) {
1418
- throw new Error("auth rejected");
1419
- }
1420
- return {
1421
- credential
1422
- };
1423
- } catch (err) {
1424
- import_log7.log.error("failed to generate auth credentials", err, {
1425
- F: __dxlog_file8,
1426
- L: 55,
1427
- S: this,
1428
- C: (f, a) => f(...a)
1429
- });
1430
- throw new Error("auth rejected");
1431
- }
1432
- }
1433
- }
1434
- };
1435
- }
1436
- async onOpen(context) {
1437
- await super.onOpen(context);
1438
- (0, import_async5.scheduleTask)(this._ctx, async () => {
1439
- try {
1440
- const challenge = (0, import_crypto2.randomBytes)(32);
1441
- const { credential } = await this.rpc.AuthService.authenticate({
1442
- challenge
1443
- });
1444
- (0, import_invariant7.invariant)(credential?.length > 0, "invalid credential", {
1445
- F: __dxlog_file8,
1446
- L: 69,
1447
- S: this,
1448
- A: [
1449
- "credential?.length > 0",
1450
- "'invalid credential'"
1451
- ]
1452
- });
1453
- const success = await this._authParams.verifier(challenge, credential);
1454
- (0, import_invariant7.invariant)(success, "credential not verified", {
1455
- F: __dxlog_file8,
1456
- L: 71,
1457
- S: this,
1458
- A: [
1459
- "success",
1460
- "'credential not verified'"
1461
- ]
1462
- });
1463
- (0, import_async5.runInContext)(this._ctx, () => this._authParams.onAuthSuccess());
1464
- } catch (err) {
1465
- (0, import_log7.log)("auth failed", err, {
1466
- F: __dxlog_file8,
1467
- L: 74,
1468
- S: this,
1469
- C: (f, a) => f(...a)
1470
- });
1471
- this.close();
1472
- this._authParams.onAuthFailure();
1473
- }
1474
- });
1475
- }
1476
- async onClose() {
1477
- await this._ctx.dispose();
1478
- await super.onClose();
1479
- }
1480
- async onAbort() {
1481
- await this._ctx.dispose();
1482
- await super.onAbort();
1483
- }
1484
- };
1485
-
1486
- // packages/core/echo/echo-pipeline/src/space/space.ts
1487
- var import_async8 = require("@dxos/async");
1488
- var import_invariant9 = require("@dxos/invariant");
1489
- var import_log10 = require("@dxos/log");
1490
- var import_credentials4 = require("@dxos/protocols/proto/dxos/halo/credentials");
1491
- var import_tracing3 = require("@dxos/tracing");
1492
- var import_util7 = require("@dxos/util");
1493
-
1494
- // packages/core/echo/echo-pipeline/src/space/control-pipeline.ts
1495
- var import_async6 = require("@dxos/async");
1496
- var import_context5 = require("@dxos/context");
1497
- var import_credentials = require("@dxos/credentials");
1498
- var import_keys5 = require("@dxos/keys");
1499
- var import_log8 = require("@dxos/log");
1500
- var import_credentials2 = require("@dxos/protocols/proto/dxos/halo/credentials");
1501
- var import_timeframe3 = require("@dxos/timeframe");
1502
- var import_tracing = require("@dxos/tracing");
1503
- var import_util5 = require("@dxos/util");
1504
- function _ts_decorate4(decorators, target, key, desc) {
1505
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1506
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1507
- r = Reflect.decorate(decorators, target, key, desc);
1508
- else
1509
- for (var i = decorators.length - 1; i >= 0; i--)
1510
- if (d = decorators[i])
1511
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1512
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1513
- }
1514
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
1515
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
1516
- var CONTROL_PIPELINE_SNAPSHOT_DELAY = 1e4;
1517
- var USE_SNAPSHOTS = true;
1518
- var ControlPipeline = class ControlPipeline2 {
1519
- constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
1520
- this._ctx = new import_context5.Context();
1521
- this._lastTimeframeSaveTime = Date.now();
1522
- this.onFeedAdmitted = new import_util5.Callback();
1523
- this._usage = new import_tracing.TimeUsageCounter();
1524
- this._mutations = new import_tracing.TimeSeriesCounter();
1525
- this._snapshotTask = new import_async6.DeferredTask(this._ctx, async () => {
1526
- await (0, import_async6.sleepWithContext)(this._ctx, CONTROL_PIPELINE_SNAPSHOT_DELAY);
1527
- await this._saveSnapshot();
1528
- });
1529
- this._spaceKey = spaceKey;
1530
- this._metadata = metadataStore;
1531
- this._pipeline = new Pipeline();
1532
- void this._pipeline.addFeed(genesisFeed);
1533
- this._spaceStateMachine = new import_credentials.SpaceStateMachine(spaceKey);
1534
- this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
1535
- (0, import_log8.log)("feed admitted", {
1536
- key: info.key
1537
- }, {
1538
- F: __dxlog_file9,
1539
- L: 74,
1540
- S: this,
1541
- C: (f, a) => f(...a)
1542
- });
1543
- if (info.assertion.designation === import_credentials2.AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
1544
- queueMicrotask(async () => {
1545
- try {
1546
- const feed = await feedProvider(info.key);
1547
- await this._pipeline.addFeed(feed);
1548
- } catch (err) {
1549
- import_log8.log.catch(err, void 0, {
1550
- F: __dxlog_file9,
1551
- L: 83,
1552
- S: this,
1553
- C: (f, a) => f(...a)
1554
- });
1555
- }
1556
- });
1557
- }
1558
- await this.onFeedAdmitted.callIfSet(info);
1559
- });
1560
- this.onMemberAdmitted = this._spaceStateMachine.onMemberAdmitted;
1561
- this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
1562
- }
1563
- get spaceState() {
1564
- return this._spaceStateMachine;
1565
- }
1566
- get pipeline() {
1567
- return this._pipeline;
1568
- }
1569
- setWriteFeed(feed) {
1570
- this._pipeline.setWriteFeed(feed);
1571
- }
1572
- async start() {
1573
- const snapshot = this._metadata.getSpaceControlPipelineSnapshot(this._spaceKey);
1574
- (0, import_log8.log)("load snapshot", {
1575
- key: this._spaceKey,
1576
- present: !!snapshot,
1577
- tf: snapshot?.timeframe
1578
- }, {
1579
- F: __dxlog_file9,
1580
- L: 110,
1581
- S: this,
1582
- C: (f, a) => f(...a)
1583
- });
1584
- if (USE_SNAPSHOTS && snapshot) {
1585
- await this._processSnapshot(snapshot);
1586
- }
1587
- (0, import_log8.log)("starting...", void 0, {
1588
- F: __dxlog_file9,
1589
- L: 115,
1590
- S: this,
1591
- C: (f, a) => f(...a)
1592
- });
1593
- setTimeout(async () => {
1594
- void this._consumePipeline(new import_context5.Context());
1595
- });
1596
- await this._pipeline.start();
1597
- (0, import_log8.log)("started", void 0, {
1598
- F: __dxlog_file9,
1599
- L: 121,
1600
- S: this,
1601
- C: (f, a) => f(...a)
1602
- });
1603
- }
1604
- async _processSnapshot(snapshot) {
1605
- await this._pipeline.setCursor(snapshot.timeframe);
1606
- for (const message of snapshot.messages ?? []) {
1607
- const result = await this._spaceStateMachine.process(message.credential, {
1608
- sourceFeed: message.feedKey,
1609
- skipVerification: true
1610
- });
1611
- if (!result) {
1612
- import_log8.log.warn("credential processing failed from snapshot", {
1613
- message
1614
- }, {
1615
- F: __dxlog_file9,
1616
- L: 134,
1617
- S: this,
1618
- C: (f, a) => f(...a)
1619
- });
1620
- }
1621
- }
1622
- }
1623
- async _saveSnapshot() {
1624
- await this._pipeline.pause();
1625
- const snapshot = {
1626
- timeframe: this._pipeline.state.timeframe,
1627
- messages: this._spaceStateMachine.credentialEntries.map((entry) => ({
1628
- feedKey: entry.sourceFeed,
1629
- credential: entry.credential
1630
- }))
1631
- };
1632
- await this._pipeline.unpause();
1633
- (0, import_log8.log)("save snapshot", {
1634
- key: this._spaceKey,
1635
- snapshot
1636
- }, {
1637
- F: __dxlog_file9,
1638
- L: 150,
1639
- S: this,
1640
- C: (f, a) => f(...a)
1641
- });
1642
- await this._metadata.setSpaceControlPipelineSnapshot(this._spaceKey, snapshot);
1643
- }
1644
- async _consumePipeline(ctx) {
1645
- for await (const msg of this._pipeline.consume()) {
1646
- const span = this._usage.beginRecording();
1647
- this._mutations.inc();
1648
- try {
1649
- await this._processMessage(ctx, msg);
1650
- } catch (err) {
1651
- import_log8.log.catch(err, void 0, {
1652
- F: __dxlog_file9,
1653
- L: 163,
1654
- S: this,
1655
- C: (f, a) => f(...a)
1656
- });
1657
- }
1658
- span.end();
1659
- }
1660
- }
1661
- async _processMessage(ctx, msg) {
1662
- (0, import_log8.log)("processing", {
1663
- key: msg.feedKey,
1664
- seq: msg.seq
1665
- }, {
1666
- F: __dxlog_file9,
1667
- L: 173,
1668
- S: this,
1669
- C: (f, a) => f(...a)
1670
- });
1671
- if (msg.data.payload.credential) {
1672
- const timer = import_util5.tracer.mark("dxos.echo.pipeline.control");
1673
- const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, {
1674
- sourceFeed: import_keys5.PublicKey.from(msg.feedKey)
1675
- });
1676
- timer.end();
1677
- if (!result) {
1678
- import_log8.log.warn("processing failed", {
1679
- msg
1680
- }, {
1681
- F: __dxlog_file9,
1682
- L: 182,
1683
- S: this,
1684
- C: (f, a) => f(...a)
1685
- });
1686
- } else {
1687
- await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1688
- }
1689
- this._snapshotTask.schedule();
1690
- }
1691
- }
1692
- async _noteTargetStateIfNeeded(timeframe) {
1693
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
1694
- this._lastTimeframeSaveTime = Date.now();
1695
- await this._saveTargetTimeframe(timeframe);
1696
- }
1697
- }
1698
- async stop() {
1699
- (0, import_log8.log)("stopping...", void 0, {
1700
- F: __dxlog_file9,
1701
- L: 202,
1702
- S: this,
1703
- C: (f, a) => f(...a)
1704
- });
1705
- await this._ctx.dispose();
1706
- await this._pipeline.stop();
1707
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1708
- (0, import_log8.log)("stopped", void 0, {
1709
- F: __dxlog_file9,
1710
- L: 206,
1711
- S: this,
1712
- C: (f, a) => f(...a)
1713
- });
1714
- }
1715
- async _saveTargetTimeframe(timeframe) {
1716
- try {
1717
- const newTimeframe = import_timeframe3.Timeframe.merge(this._targetTimeframe ?? new import_timeframe3.Timeframe(), timeframe);
1718
- await this._metadata.setSpaceControlLatestTimeframe(this._spaceKey, newTimeframe);
1719
- this._targetTimeframe = newTimeframe;
1720
- } catch (err) {
1721
- (0, import_log8.log)(err, void 0, {
1722
- F: __dxlog_file9,
1723
- L: 215,
1724
- S: this,
1725
- C: (f, a) => f(...a)
1726
- });
1727
- }
1728
- }
1729
- };
1730
- _ts_decorate4([
1731
- import_tracing.trace.metricsCounter()
1732
- ], ControlPipeline.prototype, "_usage", void 0);
1733
- _ts_decorate4([
1734
- import_tracing.trace.metricsCounter()
1735
- ], ControlPipeline.prototype, "_mutations", void 0);
1736
- _ts_decorate4([
1737
- import_tracing.trace.span({
1738
- showInBrowserTimeline: true
1739
- })
1740
- ], ControlPipeline.prototype, "start", null);
1741
- _ts_decorate4([
1742
- import_tracing.trace.span()
1743
- ], ControlPipeline.prototype, "_consumePipeline", null);
1744
- _ts_decorate4([
1745
- import_tracing.trace.span()
1746
- ], ControlPipeline.prototype, "_processMessage", null);
1747
- ControlPipeline = _ts_decorate4([
1748
- import_tracing.trace.resource(),
1749
- (0, import_async6.trackLeaks)("start", "stop")
1750
- ], ControlPipeline);
1751
-
1752
- // packages/core/echo/echo-pipeline/src/space/data-pipeline.ts
1753
- var import_async7 = require("@dxos/async");
1754
- var import_context6 = require("@dxos/context");
1755
- var import_credentials3 = require("@dxos/credentials");
1756
- var import_echo_db3 = require("@dxos/echo-db");
1757
- var import_invariant8 = require("@dxos/invariant");
1758
- var import_log9 = require("@dxos/log");
1759
- var import_protocols6 = require("@dxos/protocols");
1760
- var import_timeframe4 = require("@dxos/timeframe");
1761
- var import_tracing2 = require("@dxos/tracing");
1762
- var import_util6 = require("@dxos/util");
1763
- function _ts_decorate5(decorators, target, key, desc) {
1764
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1765
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
1766
- r = Reflect.decorate(decorators, target, key, desc);
1767
- else
1768
- for (var i = decorators.length - 1; i >= 0; i--)
1769
- if (d = decorators[i])
1770
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1771
- return c > 3 && r && Object.defineProperty(target, key, r), r;
1772
- }
1773
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
1774
- var MESSAGES_PER_SNAPSHOT = 10;
1775
- var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
1776
- var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 5e3;
1777
- var DataPipeline = class DataPipeline2 {
1778
- constructor(_params) {
1779
- this._params = _params;
1780
- this._ctx = new import_context6.Context();
1781
- this._pipeline = void 0;
1782
- this._targetTimeframe = void 0;
1783
- this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
1784
- this._isOpen = false;
1785
- this._lastTimeframeSaveTime = 0;
1786
- this._lastSnapshotSaveTime = 0;
1787
- this._lastProcessedEpoch = -1;
1788
- this._usage = new import_tracing2.TimeUsageCounter();
1789
- this._mutations = new import_tracing2.TimeSeriesCounter();
1790
- this.currentEpoch = void 0;
1791
- this.appliedEpoch = void 0;
1792
- this.onNewEpoch = new import_async7.Event();
1793
- }
1794
- get isOpen() {
1795
- return this._isOpen;
1796
- }
1797
- get pipeline() {
1798
- return this._pipeline;
1799
- }
1800
- get pipelineState() {
1801
- return this._pipeline?.state;
1802
- }
1803
- setTargetTimeframe(timeframe) {
1804
- this._targetTimeframe = timeframe;
1805
- this._pipeline?.state.setTargetTimeframe(timeframe);
1806
- }
1807
- async processCredential(credential) {
1808
- if (!(0, import_credentials3.checkCredentialType)(credential, "dxos.halo.credentials.Epoch")) {
1809
- return;
1810
- }
1811
- this.currentEpoch = credential;
1812
- if (this._isOpen) {
1813
- await this._processEpochInSeparateTask(credential);
1814
- }
1815
- }
1816
- async open() {
1817
- if (this._isOpen) {
1818
- return;
1819
- }
1820
- this._pipeline = new Pipeline();
1821
- await this._params.onPipelineCreated(this._pipeline);
1822
- await this._pipeline.pause();
1823
- await this._pipeline.start();
1824
- if (this._targetTimeframe) {
1825
- this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
1826
- }
1827
- const feedWriter = {
1828
- write: (data, options) => {
1829
- (0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
1830
- F: __dxlog_file10,
1831
- L: 159,
1832
- S: this,
1833
- A: [
1834
- "this._pipeline",
1835
- "'Pipeline is not initialized.'"
1836
- ]
1837
- });
1838
- (0, import_invariant8.invariant)(this.currentEpoch, "Epoch is not initialized.", {
1839
- F: __dxlog_file10,
1840
- L: 160,
1841
- S: this,
1842
- A: [
1843
- "this.currentEpoch",
1844
- "'Epoch is not initialized.'"
1845
- ]
1846
- });
1847
- return this._pipeline.writer.write({
1848
- data
1849
- }, options);
1850
- }
1851
- };
1852
- this.databaseHost = new DatabaseHost(feedWriter, () => this._flush());
1853
- this.itemManager = new import_echo_db3.ItemManager(this._params.modelFactory);
1854
- await this.databaseHost.open(this.itemManager, this._params.modelFactory);
1855
- (0, import_async7.scheduleTask)(this._ctx, async () => {
1856
- await this._consumePipeline();
1857
- });
1858
- this._isOpen = true;
1859
- }
1860
- async close() {
1861
- if (!this._isOpen) {
1862
- return;
1863
- }
1864
- (0, import_log9.log)("close", void 0, {
1865
- F: __dxlog_file10,
1866
- L: 184,
1867
- S: this,
1868
- C: (f, a) => f(...a)
1869
- });
1870
- this._isOpen = false;
1871
- await this._ctx.dispose();
1872
- await this._pipeline?.stop();
1873
- try {
1874
- await this._saveCache();
1875
- if (this._pipeline) {
1876
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1877
- }
1878
- } catch (err) {
1879
- import_log9.log.catch(err, void 0, {
1880
- F: __dxlog_file10,
1881
- L: 197,
1882
- S: this,
1883
- C: (f, a) => f(...a)
1884
- });
1885
- }
1886
- await this.databaseHost?.close();
1887
- await this.itemManager?.destroy();
1888
- this._ctx = new import_context6.Context();
1889
- this._pipeline = void 0;
1890
- this._targetTimeframe = void 0;
1891
- this._lastAutomaticSnapshotTimeframe = new import_timeframe4.Timeframe();
1892
- this.currentEpoch = void 0;
1893
- this.appliedEpoch = void 0;
1894
- this._lastProcessedEpoch = -1;
1895
- this._epochCtx = void 0;
1896
- }
1897
- async _consumePipeline() {
1898
- if (this.currentEpoch) {
1899
- const waitForOneEpoch = this.onNewEpoch.waitForCount(1);
1900
- await this._processEpochInSeparateTask(this.currentEpoch);
1901
- await waitForOneEpoch;
1902
- }
1903
- let messageCounter = 0;
1904
- (0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
1905
- F: __dxlog_file10,
1906
- L: 222,
1907
- S: this,
1908
- A: [
1909
- "this._pipeline",
1910
- "'Pipeline is not initialized.'"
1911
- ]
1912
- });
1913
- for await (const msg of this._pipeline.consume()) {
1914
- const span = this._usage.beginRecording();
1915
- this._mutations.inc();
1916
- const { feedKey, seq, data } = msg;
1917
- (0, import_log9.log)("processing message", {
1918
- feedKey,
1919
- seq
1920
- }, {
1921
- F: __dxlog_file10,
1922
- L: 228,
1923
- S: this,
1924
- C: (f, a) => f(...a)
1925
- });
1926
- try {
1927
- if (data.payload.data) {
1928
- const feedInfo = this._params.feedInfoProvider(feedKey);
1929
- if (!feedInfo) {
1930
- import_log9.log.warn("Could not find feed", {
1931
- feedKey
1932
- }, {
1933
- F: __dxlog_file10,
1934
- L: 234,
1935
- S: this,
1936
- C: (f, a) => f(...a)
1937
- });
1938
- continue;
1939
- }
1940
- const timer = import_util6.tracer.mark("dxos.echo.pipeline.data");
1941
- this.databaseHost.echoProcessor({
1942
- batch: data.payload.data.batch,
1943
- meta: {
1944
- feedKey,
1945
- seq,
1946
- timeframe: data.timeframe,
1947
- memberKey: feedInfo.assertion.identityKey
1948
- }
1949
- });
1950
- timer.end();
1951
- import_log9.log.trace("dxos.echo.data-pipeline.processed", {
1952
- feedKey: feedKey.toHex(),
1953
- seq,
1954
- spaceKey: this._params.spaceKey.toHex()
1955
- }, {
1956
- F: __dxlog_file10,
1957
- L: 251,
1958
- S: this,
1959
- C: (f, a) => f(...a)
1960
- });
1961
- await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
1962
- }
1963
- } catch (err) {
1964
- import_log9.log.catch(err, void 0, {
1965
- F: __dxlog_file10,
1966
- L: 261,
1967
- S: this,
1968
- C: (f, a) => f(...a)
1969
- });
1970
- }
1971
- span.end();
1972
- if (++messageCounter > 1e3) {
1973
- messageCounter = 0;
1974
- await (0, import_async7.sleep)(1);
1975
- }
1976
- }
1977
- }
1978
- _createSnapshot() {
1979
- (0, import_invariant8.invariant)(this.databaseHost, "Database backend is not initialized.", {
1980
- F: __dxlog_file10,
1981
- L: 275,
1982
- S: this,
1983
- A: [
1984
- "this.databaseHost",
1985
- "'Database backend is not initialized.'"
1986
- ]
1987
- });
1988
- return {
1989
- spaceKey: this._params.spaceKey.asUint8Array(),
1990
- timeframe: this._pipeline.state.timeframe,
1991
- database: this.databaseHost.createSnapshot()
1992
- };
1993
- }
1994
- async _saveTargetTimeframe(timeframe) {
1995
- const newTimeframe = import_timeframe4.Timeframe.merge(this._targetTimeframe ?? new import_timeframe4.Timeframe(), timeframe);
1996
- await this._params.metadataStore.setSpaceDataLatestTimeframe(this._params.spaceKey, newTimeframe);
1997
- this._targetTimeframe = newTimeframe;
1998
- }
1999
- async _saveCache() {
2000
- const cache = {};
2001
- try {
2002
- const propertiesItem = this.itemManager.items.find((item) => item.modelMeta?.type === "dxos.org/model/document" && // TODO(burdon): Document?
2003
- ((0, import_echo_db3.getStateMachineFromItem)(item)?.snapshot()).type === import_echo_db3.TYPE_PROPERTIES);
2004
- if (propertiesItem) {
2005
- cache.properties = (0, import_echo_db3.getStateMachineFromItem)(propertiesItem)?.snapshot();
2006
- }
2007
- } catch (err) {
2008
- import_log9.log.warn("Failed to cache properties", err, {
2009
- F: __dxlog_file10,
2010
- L: 304,
2011
- S: this,
2012
- C: (f, a) => f(...a)
2013
- });
2014
- }
2015
- await this._params.metadataStore.setCache(this._params.spaceKey, cache);
2016
- }
2017
- async _noteTargetStateIfNeeded(timeframe) {
2018
- if (!this._pipeline?.state.reachedTarget) {
2019
- return;
2020
- }
2021
- if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2) {
2022
- this._lastTimeframeSaveTime = Date.now();
2023
- await this._saveTargetTimeframe(timeframe);
2024
- }
2025
- if (Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL && timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT) {
2026
- await this._saveCache();
2027
- }
2028
- }
2029
- async _processEpochInSeparateTask(epoch) {
2030
- if (epoch.subject.assertion.number <= this._lastProcessedEpoch) {
2031
- return;
2032
- }
2033
- await this._epochCtx?.dispose();
2034
- const ctx = new import_context6.Context({
2035
- onError: (err) => {
2036
- if (err instanceof import_protocols6.CancelledError) {
2037
- (0, import_log9.log)("Epoch processing cancelled.", void 0, {
2038
- F: __dxlog_file10,
2039
- L: 340,
2040
- S: this,
2041
- C: (f, a) => f(...a)
2042
- });
2043
- } else {
2044
- import_log9.log.catch(err, void 0, {
2045
- F: __dxlog_file10,
2046
- L: 342,
2047
- S: this,
2048
- C: (f, a) => f(...a)
2049
- });
2050
- }
2051
- }
2052
- });
2053
- this._epochCtx = ctx;
2054
- (0, import_async7.scheduleTask)(ctx, async () => {
2055
- if (!this._isOpen) {
2056
- return;
2057
- }
2058
- await this._processEpoch(ctx, epoch.subject.assertion);
2059
- this.appliedEpoch = epoch;
2060
- this.onNewEpoch.emit(epoch);
2061
- });
2062
- }
2063
- async _processEpoch(ctx, epoch) {
2064
- (0, import_invariant8.invariant)(this._isOpen, "Space is closed.", {
2065
- F: __dxlog_file10,
2066
- L: 361,
2067
- S: this,
2068
- A: [
2069
- "this._isOpen",
2070
- "'Space is closed.'"
2071
- ]
2072
- });
2073
- (0, import_invariant8.invariant)(this._pipeline, void 0, {
2074
- F: __dxlog_file10,
2075
- L: 362,
2076
- S: this,
2077
- A: [
2078
- "this._pipeline",
2079
- ""
2080
- ]
2081
- });
2082
- this._lastProcessedEpoch = epoch.number;
2083
- (0, import_log9.log)("processing", {
2084
- epoch: (0, import_log9.omit)(epoch, "proof")
2085
- }, {
2086
- F: __dxlog_file10,
2087
- L: 365,
2088
- S: this,
2089
- C: (f, a) => f(...a)
2090
- });
2091
- if (epoch.snapshotCid) {
2092
- const snapshot = await this._params.snapshotManager.load(ctx, epoch.snapshotCid);
2093
- this.databaseHost._itemDemuxer.restoreFromSnapshot(snapshot.database);
2094
- }
2095
- (0, import_log9.log)("restarting pipeline from epoch", void 0, {
2096
- F: __dxlog_file10,
2097
- L: 371,
2098
- S: this,
2099
- C: (f, a) => f(...a)
2100
- });
2101
- await this._pipeline.pause();
2102
- await this._pipeline.setCursor(epoch.timeframe);
2103
- await this._pipeline.unpause();
2104
- }
2105
- async waitUntilTimeframe(timeframe) {
2106
- (0, import_invariant8.invariant)(this._pipeline, "Pipeline is not initialized.", {
2107
- F: __dxlog_file10,
2108
- L: 378,
2109
- S: this,
2110
- A: [
2111
- "this._pipeline",
2112
- "'Pipeline is not initialized.'"
2113
- ]
2114
- });
2115
- await this._pipeline.state.waitUntilTimeframe(timeframe);
2116
- }
2117
- async createEpoch() {
2118
- (0, import_invariant8.invariant)(this._pipeline, void 0, {
2119
- F: __dxlog_file10,
2120
- L: 384,
2121
- S: this,
2122
- A: [
2123
- "this._pipeline",
2124
- ""
2125
- ]
2126
- });
2127
- (0, import_invariant8.invariant)(this.currentEpoch, void 0, {
2128
- F: __dxlog_file10,
2129
- L: 385,
2130
- S: this,
2131
- A: [
2132
- "this.currentEpoch",
2133
- ""
2134
- ]
2135
- });
2136
- await this._pipeline.pause();
2137
- const snapshot = await this._createSnapshot();
2138
- const snapshotCid = await this._params.snapshotManager.store(snapshot);
2139
- const epoch = {
2140
- previousId: this.currentEpoch.id,
2141
- timeframe: this._pipeline.state.timeframe,
2142
- number: this.currentEpoch.subject.assertion.number + 1,
2143
- snapshotCid
2144
- };
2145
- await this._pipeline.unpause();
2146
- return epoch;
2147
- }
2148
- async ensureEpochInitialized() {
2149
- await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
2150
- }
2151
- async _flush() {
2152
- try {
2153
- await this._saveCache();
2154
- if (this._pipeline) {
2155
- await this._saveTargetTimeframe(this._pipeline.state.timeframe);
2156
- }
2157
- } catch (err) {
2158
- import_log9.log.catch(err, void 0, {
2159
- F: __dxlog_file10,
2160
- L: 415,
2161
- S: this,
2162
- C: (f, a) => f(...a)
2163
- });
2164
- }
2165
- await this._params.metadataStore.flush();
2166
- }
2167
- };
2168
- _ts_decorate5([
2169
- import_tracing2.trace.metricsCounter()
2170
- ], DataPipeline.prototype, "_usage", void 0);
2171
- _ts_decorate5([
2172
- import_tracing2.trace.metricsCounter()
2173
- ], DataPipeline.prototype, "_mutations", void 0);
2174
- _ts_decorate5([
2175
- import_async7.synchronized
2176
- ], DataPipeline.prototype, "open", null);
2177
- _ts_decorate5([
2178
- import_async7.synchronized
2179
- ], DataPipeline.prototype, "close", null);
2180
- _ts_decorate5([
2181
- import_async7.synchronized
2182
- ], DataPipeline.prototype, "_processEpoch", null);
2183
- _ts_decorate5([
2184
- import_async7.synchronized
2185
- ], DataPipeline.prototype, "createEpoch", null);
2186
- DataPipeline = _ts_decorate5([
2187
- (0, import_async7.trackLeaks)("open", "close"),
2188
- import_tracing2.trace.resource()
2189
- ], DataPipeline);
2190
-
2191
- // packages/core/echo/echo-pipeline/src/space/space.ts
2192
- function _ts_decorate6(decorators, target, key, desc) {
2193
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2194
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2195
- r = Reflect.decorate(decorators, target, key, desc);
2196
- else
2197
- for (var i = decorators.length - 1; i >= 0; i--)
2198
- if (d = decorators[i])
2199
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2200
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2201
- }
2202
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
2203
- var Space = class Space2 {
2204
- constructor(params) {
2205
- this._addFeedLock = new import_async8.Lock();
2206
- this.onCredentialProcessed = new import_util7.Callback();
2207
- this.stateUpdate = new import_async8.Event();
2208
- this._isOpen = false;
2209
- (0, import_invariant9.invariant)(params.spaceKey && params.feedProvider, void 0, {
2210
- F: __dxlog_file11,
2211
- L: 73,
2212
- S: this,
2213
- A: [
2214
- "params.spaceKey && params.feedProvider",
2215
- ""
2216
- ]
2217
- });
2218
- this._key = params.spaceKey;
2219
- this._genesisFeedKey = params.genesisFeed.key;
2220
- this._feedProvider = params.feedProvider;
2221
- this._snapshotManager = params.snapshotManager;
2222
- this._controlPipeline = new ControlPipeline({
2223
- spaceKey: params.spaceKey,
2224
- genesisFeed: params.genesisFeed,
2225
- feedProvider: params.feedProvider,
2226
- metadataStore: params.metadataStore
2227
- });
2228
- this._controlPipeline.onFeedAdmitted.set(async (info) => {
2229
- const sparse = info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA;
2230
- if (info.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA) {
2231
- queueMicrotask(async () => {
2232
- if (this._dataPipeline.pipeline) {
2233
- if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
2234
- return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key, {
2235
- sparse
2236
- }));
2237
- }
2238
- }
2239
- });
2240
- }
2241
- if (!info.key.equals(params.genesisFeed.key)) {
2242
- queueMicrotask(async () => {
2243
- this.protocol.addFeed(await params.feedProvider(info.key, {
2244
- sparse
2245
- }));
2246
- });
2247
- }
2248
- });
2249
- this._controlPipeline.onCredentialProcessed.set(async (credential) => {
2250
- await this.onCredentialProcessed.callIfSet(credential);
2251
- (0, import_log10.log)("onCredentialProcessed", {
2252
- credential
2253
- }, {
2254
- F: __dxlog_file11,
2255
- L: 111,
2256
- S: this,
2257
- C: (f, a) => f(...a)
2258
- });
2259
- this.stateUpdate.emit();
2260
- });
2261
- this.protocol = params.protocol;
2262
- this.protocol.addFeed(params.genesisFeed);
2263
- this._dataPipeline = new DataPipeline({
2264
- modelFactory: params.modelFactory,
2265
- metadataStore: params.metadataStore,
2266
- snapshotManager: params.snapshotManager,
2267
- memberKey: params.memberKey,
2268
- spaceKey: this._key,
2269
- feedInfoProvider: (feedKey) => this._controlPipeline.spaceState.feeds.get(feedKey),
2270
- snapshotId: params.snapshotId,
2271
- onPipelineCreated: async (pipeline) => {
2272
- if (this._dataFeed) {
2273
- pipeline.setWriteFeed(this._dataFeed);
2274
- }
2275
- await this._addFeedLock.executeSynchronized(async () => {
2276
- for (const feed of this._controlPipeline.spaceState.feeds.values()) {
2277
- if (feed.assertion.designation === import_credentials4.AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
2278
- await pipeline.addFeed(await this._feedProvider(feed.key, {
2279
- sparse: true
2280
- }));
2281
- }
2282
- }
2283
- });
2284
- }
2285
- });
2286
- }
2287
- get key() {
2288
- return this._key;
2289
- }
2290
- get isOpen() {
2291
- return this._isOpen;
2292
- }
2293
- get genesisFeedKey() {
2294
- return this._genesisFeedKey;
2295
- }
2296
- get controlFeedKey() {
2297
- return this._controlFeed?.key;
2298
- }
2299
- get dataFeedKey() {
2300
- return this._dataFeed?.key;
2301
- }
2302
- get spaceState() {
2303
- return this._controlPipeline.spaceState;
2304
- }
2305
- /**
2306
- * @test-only
2307
- */
2308
- get controlPipeline() {
2309
- return this._controlPipeline.pipeline;
2310
- }
2311
- get dataPipeline() {
2312
- return this._dataPipeline;
2313
- }
2314
- get snapshotManager() {
2315
- return this._snapshotManager;
2316
- }
2317
- setControlFeed(feed) {
2318
- (0, import_invariant9.invariant)(!this._controlFeed, "Control feed already set.", {
2319
- F: __dxlog_file11,
2320
- L: 186,
2321
- S: this,
2322
- A: [
2323
- "!this._controlFeed",
2324
- "'Control feed already set.'"
2325
- ]
2326
- });
2327
- this._controlFeed = feed;
2328
- this._controlPipeline.setWriteFeed(feed);
2329
- return this;
2330
- }
2331
- setDataFeed(feed) {
2332
- (0, import_invariant9.invariant)(!this._dataFeed, "Data feed already set.", {
2333
- F: __dxlog_file11,
2334
- L: 193,
2335
- S: this,
2336
- A: [
2337
- "!this._dataFeed",
2338
- "'Data feed already set.'"
2339
- ]
2340
- });
2341
- this._dataFeed = feed;
2342
- this._dataPipeline.pipeline?.setWriteFeed(feed);
2343
- return this;
2344
- }
2345
- /**
2346
- * Use for diagnostics.
2347
- */
2348
- getControlFeeds() {
2349
- return Array.from(this._controlPipeline.spaceState.feeds.values());
2350
- }
2351
- /**
2352
- * Use for diagnostics.
2353
- */
2354
- // getDataFeeds(): FeedInfo[] {
2355
- // return this._dataPipeline?.getFeeds();
2356
- // }
2357
- async open(ctx) {
2358
- (0, import_log10.log)("opening...", void 0, {
2359
- F: __dxlog_file11,
2360
- L: 215,
2361
- S: this,
2362
- C: (f, a) => f(...a)
2363
- });
2364
- if (this._isOpen) {
2365
- return;
2366
- }
2367
- await this._controlPipeline.start();
2368
- await this.protocol.start();
2369
- await this._controlPipeline.spaceState.addCredentialProcessor(this._dataPipeline);
2370
- this._isOpen = true;
2371
- (0, import_log10.log)("opened", void 0, {
2372
- F: __dxlog_file11,
2373
- L: 226,
2374
- S: this,
2375
- C: (f, a) => f(...a)
2376
- });
2377
- }
2378
- async close() {
2379
- (0, import_log10.log)("closing...", {
2380
- key: this._key
2381
- }, {
2382
- F: __dxlog_file11,
2383
- L: 231,
2384
- S: this,
2385
- C: (f, a) => f(...a)
2386
- });
2387
- if (!this._isOpen) {
2388
- return;
2389
- }
2390
- await this._controlPipeline.spaceState.removeCredentialProcessor(this._dataPipeline);
2391
- await this._dataPipeline.close();
2392
- await this.protocol.stop();
2393
- await this._controlPipeline.stop();
2394
- this._isOpen = false;
2395
- (0, import_log10.log)("closed", void 0, {
2396
- F: __dxlog_file11,
2397
- L: 244,
2398
- S: this,
2399
- C: (f, a) => f(...a)
2400
- });
2401
- }
2402
- async initializeDataPipeline() {
2403
- (0, import_log10.log)("initializeDataPipeline", void 0, {
2404
- F: __dxlog_file11,
2405
- L: 249,
2406
- S: this,
2407
- C: (f, a) => f(...a)
2408
- });
2409
- (0, import_invariant9.invariant)(this._isOpen, "Space must be open to initialize data pipeline.", {
2410
- F: __dxlog_file11,
2411
- L: 250,
2412
- S: this,
2413
- A: [
2414
- "this._isOpen",
2415
- "'Space must be open to initialize data pipeline.'"
2416
- ]
2417
- });
2418
- await this._dataPipeline.open();
2419
- }
2420
- };
2421
- _ts_decorate6([
2422
- import_log10.logInfo,
2423
- import_tracing3.trace.info()
2424
- ], Space.prototype, "key", null);
2425
- _ts_decorate6([
2426
- import_async8.synchronized,
2427
- import_tracing3.trace.span()
2428
- ], Space.prototype, "open", null);
2429
- _ts_decorate6([
2430
- import_async8.synchronized
2431
- ], Space.prototype, "close", null);
2432
- _ts_decorate6([
2433
- import_async8.synchronized
2434
- ], Space.prototype, "initializeDataPipeline", null);
2435
- Space = _ts_decorate6([
2436
- (0, import_async8.trackLeaks)("open", "close"),
2437
- import_tracing3.trace.resource()
2438
- ], Space);
2439
-
2440
- // packages/core/echo/echo-pipeline/src/space/space-manager.ts
2441
- var import_async9 = require("@dxos/async");
2442
- var import_debug4 = require("@dxos/debug");
2443
- var import_keys7 = require("@dxos/keys");
2444
- var import_log12 = require("@dxos/log");
2445
- var import_protocols7 = require("@dxos/protocols");
2446
- var import_util9 = require("@dxos/util");
2447
-
2448
- // packages/core/echo/echo-pipeline/src/space/space-protocol.ts
2449
- var import_crypto3 = require("@dxos/crypto");
2450
- var import_keys6 = require("@dxos/keys");
2451
- var import_log11 = require("@dxos/log");
2452
- var import_network_manager = require("@dxos/network-manager");
2453
- var import_teleport2 = require("@dxos/teleport");
2454
- var import_teleport_extension_object_sync = require("@dxos/teleport-extension-object-sync");
2455
- var import_teleport_extension_replicator = require("@dxos/teleport-extension-replicator");
2456
- var import_util8 = require("@dxos/util");
2457
- function _ts_decorate7(decorators, target, key, desc) {
2458
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2459
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2460
- r = Reflect.decorate(decorators, target, key, desc);
2461
- else
2462
- for (var i = decorators.length - 1; i >= 0; i--)
2463
- if (d = decorators[i])
2464
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2465
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2466
- }
2467
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
2468
- var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
2469
- var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
2470
- var SpaceProtocol = class {
2471
- constructor({ topic, swarmIdentity, networkManager, onSessionAuth, onAuthFailure, blobStore }) {
2472
- this._feeds = /* @__PURE__ */ new Set();
2473
- this._sessions = new import_util8.ComplexMap(import_keys6.PublicKey.hash);
2474
- this._networkManager = networkManager;
2475
- this._swarmIdentity = swarmIdentity;
2476
- this._onSessionAuth = onSessionAuth;
2477
- this._onAuthFailure = onAuthFailure;
2478
- this.blobSync = new import_teleport_extension_object_sync.BlobSync({
2479
- blobStore
2480
- });
2481
- this._topic = import_crypto3.subtleCrypto.digest("SHA-256", topic.asBuffer()).then(import_crypto3.discoveryKey).then(import_keys6.PublicKey.from);
2482
- }
2483
- get sessions() {
2484
- return this._sessions;
2485
- }
2486
- get feeds() {
2487
- return this._feeds;
2488
- }
2489
- get _ownPeerKey() {
2490
- return this._swarmIdentity.peerKey;
2491
- }
2492
- // TODO(burdon): Create abstraction for Space (e.g., add keys and have provider).
2493
- addFeed(feed) {
2494
- (0, import_log11.log)("addFeed", {
2495
- key: feed.key
2496
- }, {
2497
- F: __dxlog_file12,
2498
- L: 96,
2499
- S: this,
2500
- C: (f, a) => f(...a)
2501
- });
2502
- this._feeds.add(feed);
2503
- for (const session of this._sessions.values()) {
2504
- session.replicator.addFeed(feed);
2505
- }
2506
- }
2507
- // TODO(burdon): Rename open? Common open/close interfaces for all services?
2508
- async start() {
2509
- if (this._connection) {
2510
- return;
2511
- }
2512
- const credentials = await this._swarmIdentity.credentialProvider(Buffer.from(""));
2513
- const topologyConfig = {
2514
- originateConnections: 4,
2515
- maxPeers: 10,
2516
- sampleSize: 20
2517
- };
2518
- await this.blobSync.open();
2519
- (0, import_log11.log)("starting...", void 0, {
2520
- F: __dxlog_file12,
2521
- L: 122,
2522
- S: this,
2523
- C: (f, a) => f(...a)
2524
- });
2525
- const topic = await this._topic;
2526
- this._connection = await this._networkManager.joinSwarm({
2527
- protocolProvider: this._createProtocolProvider(credentials),
2528
- peerId: this._swarmIdentity.peerKey,
2529
- topic,
2530
- topology: new import_network_manager.MMSTTopology(topologyConfig),
2531
- label: `space swarm ${topic.truncate()}`
2532
- });
2533
- (0, import_log11.log)("started", void 0, {
2534
- F: __dxlog_file12,
2535
- L: 132,
2536
- S: this,
2537
- C: (f, a) => f(...a)
2538
- });
2539
- }
2540
- async stop() {
2541
- await this.blobSync.close();
2542
- if (this._connection) {
2543
- (0, import_log11.log)("stopping...", void 0, {
2544
- F: __dxlog_file12,
2545
- L: 139,
2546
- S: this,
2547
- C: (f, a) => f(...a)
2548
- });
2549
- await this._connection.close();
2550
- (0, import_log11.log)("stopped", void 0, {
2551
- F: __dxlog_file12,
2552
- L: 141,
2553
- S: this,
2554
- C: (f, a) => f(...a)
2555
- });
2556
- }
2557
- }
2558
- _createProtocolProvider(credentials) {
2559
- return (wireParams) => {
2560
- const session = new SpaceProtocolSession({
2561
- wireParams,
2562
- swarmIdentity: this._swarmIdentity,
2563
- onSessionAuth: this._onSessionAuth,
2564
- onAuthFailure: this._onAuthFailure,
2565
- blobSync: this.blobSync
2566
- });
2567
- this._sessions.set(wireParams.remotePeerId, session);
2568
- for (const feed of this._feeds) {
2569
- session.replicator.addFeed(feed);
2570
- }
2571
- return session;
2572
- };
2573
- }
2574
- };
2575
- _ts_decorate7([
2576
- import_log11.logInfo
2577
- ], SpaceProtocol.prototype, "_topic", void 0);
2578
- _ts_decorate7([
2579
- import_log11.logInfo
2580
- ], SpaceProtocol.prototype, "_ownPeerKey", null);
2581
- var AuthStatus;
2582
- (function(AuthStatus2) {
2583
- AuthStatus2["INITIAL"] = "INITIAL";
2584
- AuthStatus2["SUCCESS"] = "SUCCESS";
2585
- AuthStatus2["FAILURE"] = "FAILURE";
2586
- })(AuthStatus || (AuthStatus = {}));
2587
- var SpaceProtocolSession = class {
2588
- // TODO(dmaretskyi): Allow to pass in extra extensions.
2589
- constructor({ wireParams, swarmIdentity, onSessionAuth, onAuthFailure, blobSync }) {
2590
- // TODO(dmaretskyi): Start with upload=false when switching it on the fly works.
2591
- this.replicator = new import_teleport_extension_replicator.ReplicatorExtension().setOptions({
2592
- upload: true
2593
- });
2594
- this._authStatus = AuthStatus.INITIAL;
2595
- this._wireParams = wireParams;
2596
- this._swarmIdentity = swarmIdentity;
2597
- this._onSessionAuth = onSessionAuth;
2598
- this._onAuthFailure = onAuthFailure;
2599
- this._blobSync = blobSync;
2600
- this._teleport = new import_teleport2.Teleport(wireParams);
2601
- }
2602
- get authStatus() {
2603
- return this._authStatus;
2604
- }
2605
- get stats() {
2606
- return this._teleport.stats;
2607
- }
2608
- get stream() {
2609
- return this._teleport.stream;
2610
- }
2611
- async open() {
2612
- await this._teleport.open();
2613
- this._teleport.addExtension("dxos.mesh.teleport.auth", new AuthExtension({
2614
- provider: this._swarmIdentity.credentialProvider,
2615
- verifier: this._swarmIdentity.credentialAuthenticator,
2616
- onAuthSuccess: () => {
2617
- (0, import_log11.log)("Peer authenticated", void 0, {
2618
- F: __dxlog_file12,
2619
- L: 238,
2620
- S: this,
2621
- C: (f, a) => f(...a)
2622
- });
2623
- this._authStatus = AuthStatus.SUCCESS;
2624
- this._onSessionAuth?.(this._teleport);
2625
- },
2626
- onAuthFailure: () => {
2627
- this._authStatus = AuthStatus.FAILURE;
2628
- this._onAuthFailure?.(this._teleport);
2629
- }
2630
- }));
2631
- this._teleport.addExtension("dxos.mesh.teleport.replicator", this.replicator);
2632
- this._teleport.addExtension("dxos.mesh.teleport.blobsync", this._blobSync.createExtension());
2633
- }
2634
- async close() {
2635
- (0, import_log11.log)("close", void 0, {
2636
- F: __dxlog_file12,
2637
- L: 254,
2638
- S: this,
2639
- C: (f, a) => f(...a)
2640
- });
2641
- await this._teleport.close();
2642
- }
2643
- async abort() {
2644
- await this._teleport.abort();
2645
- }
2646
- };
2647
- _ts_decorate7([
2648
- import_log11.logInfo
2649
- ], SpaceProtocolSession.prototype, "_wireParams", void 0);
2650
- _ts_decorate7([
2651
- import_log11.logInfo
2652
- ], SpaceProtocolSession.prototype, "authStatus", null);
2653
-
2654
- // packages/core/echo/echo-pipeline/src/space/space-manager.ts
2655
- function _ts_decorate8(decorators, target, key, desc) {
2656
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
2657
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
2658
- r = Reflect.decorate(decorators, target, key, desc);
2659
- else
2660
- for (var i = decorators.length - 1; i >= 0; i--)
2661
- if (d = decorators[i])
2662
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
2663
- return c > 3 && r && Object.defineProperty(target, key, r), r;
2664
- }
2665
- var __dxlog_file13 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
2666
- var SpaceManager = class SpaceManager2 {
2667
- constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
2668
- this._spaces = new import_util9.ComplexMap(import_keys7.PublicKey.hash);
2669
- this._instanceId = import_keys7.PublicKey.random().toHex();
2670
- this._feedStore = feedStore;
2671
- this._networkManager = networkManager;
2672
- this._modelFactory = modelFactory;
2673
- this._metadataStore = metadataStore;
2674
- this._snapshotStore = snapshotStore;
2675
- this._blobStore = blobStore;
2676
- }
2677
- // TODO(burdon): Remove.
2678
- get spaces() {
2679
- return this._spaces;
2680
- }
2681
- async open() {
2682
- }
2683
- async close() {
2684
- await Promise.all([
2685
- ...this._spaces.values()
2686
- ].map((space) => space.close()));
2687
- }
2688
- async constructSpace({ metadata, swarmIdentity, onNetworkConnection, onAuthFailure, memberKey }) {
2689
- import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.begin({
2690
- id: this._instanceId
2691
- }), {
2692
- F: __dxlog_file13,
2693
- L: 97,
2694
- S: this,
2695
- C: (f, a) => f(...a)
2696
- });
2697
- (0, import_log12.log)("constructing space...", {
2698
- spaceKey: metadata.genesisFeedKey
2699
- }, {
2700
- F: __dxlog_file13,
2701
- L: 98,
2702
- S: this,
2703
- C: (f, a) => f(...a)
2704
- });
2705
- const genesisFeed = await this._feedStore.openFeed(metadata.genesisFeedKey ?? (0, import_debug4.failUndefined)());
2706
- const spaceKey = metadata.key;
2707
- const protocol = new SpaceProtocol({
2708
- topic: spaceKey,
2709
- swarmIdentity,
2710
- networkManager: this._networkManager,
2711
- onSessionAuth: onNetworkConnection,
2712
- onAuthFailure,
2713
- blobStore: this._blobStore
2714
- });
2715
- const snapshotManager = new SnapshotManager(this._snapshotStore, this._blobStore, protocol.blobSync);
2716
- const space = new Space({
2717
- spaceKey,
2718
- protocol,
2719
- genesisFeed,
2720
- feedProvider: (feedKey, opts) => this._feedStore.openFeed(feedKey, opts),
2721
- modelFactory: this._modelFactory,
2722
- metadataStore: this._metadataStore,
2723
- snapshotManager,
2724
- memberKey
2725
- });
2726
- this._spaces.set(space.key, space);
2727
- import_log12.log.trace("dxos.echo.space-manager.construct-space", import_protocols7.trace.end({
2728
- id: this._instanceId
2729
- }), {
2730
- F: __dxlog_file13,
2731
- L: 126,
2732
- S: this,
2733
- C: (f, a) => f(...a)
2734
- });
2735
- return space;
2736
- }
2737
- };
2738
- _ts_decorate8([
2739
- import_async9.synchronized
2740
- ], SpaceManager.prototype, "open", null);
2741
- _ts_decorate8([
2742
- import_async9.synchronized
2743
- ], SpaceManager.prototype, "close", null);
2744
- SpaceManager = _ts_decorate8([
2745
- (0, import_async9.trackLeaks)("open", "close")
2746
- ], SpaceManager);
46
+ module.exports = __toCommonJS(node_exports);
47
+ var import_chunk_YJAADRTG = require("./chunk-YJAADRTG.cjs");
2747
48
  // Annotate the CommonJS export names for ESM import in node:
2748
49
  0 && (module.exports = {
2749
50
  AuthExtension,