@dxos/echo-pipeline 0.1.55-main.ff36828 → 0.1.55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/lib/browser/{chunk-OJGL5427.mjs → chunk-4VJRJ4SY.mjs} +877 -872
  2. package/dist/lib/browser/chunk-4VJRJ4SY.mjs.map +7 -0
  3. package/dist/lib/browser/index.mjs +1 -1
  4. package/dist/lib/browser/meta.json +1 -1
  5. package/dist/lib/browser/testing/index.mjs +1 -1
  6. package/dist/lib/browser/testing/index.mjs.map +1 -1
  7. package/dist/lib/node/index.cjs +787 -784
  8. package/dist/lib/node/index.cjs.map +4 -4
  9. package/dist/lib/node/meta.json +1 -1
  10. package/dist/lib/node/testing/index.cjs +78 -73
  11. package/dist/lib/node/testing/index.cjs.map +4 -4
  12. package/dist/types/src/db-host/data-service-host.d.ts.map +1 -0
  13. package/dist/types/src/db-host/data-service.d.ts.map +1 -0
  14. package/dist/types/src/db-host/database-host.d.ts.map +1 -0
  15. package/dist/types/src/db-host/index.d.ts.map +1 -0
  16. package/dist/types/src/db-host/snapshot-manager.d.ts.map +1 -0
  17. package/dist/types/src/db-host/snapshot-store.d.ts.map +1 -0
  18. package/dist/types/src/db-host/snapshot-store.test.d.ts.map +1 -0
  19. package/dist/types/src/index.d.ts +2 -2
  20. package/dist/types/src/index.d.ts.map +1 -1
  21. package/dist/types/src/pipeline/pipeline.d.ts +3 -2
  22. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  23. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  24. package/dist/types/src/space/data-pipeline.d.ts +4 -4
  25. package/dist/types/src/space/data-pipeline.d.ts.map +1 -1
  26. package/dist/types/src/space/space-manager.d.ts +1 -1
  27. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  28. package/dist/types/src/space/space-protocol.d.ts +2 -3
  29. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  30. package/dist/types/src/space/space.d.ts +1 -1
  31. package/dist/types/src/space/space.d.ts.map +1 -1
  32. package/dist/types/src/testing/database-test-rig.d.ts +1 -1
  33. package/dist/types/src/testing/database-test-rig.d.ts.map +1 -1
  34. package/dist/types/src/testing/test-agent-builder.d.ts +1 -1
  35. package/dist/types/src/testing/test-agent-builder.d.ts.map +1 -1
  36. package/dist/types/src/testing/util.d.ts +1 -1
  37. package/dist/types/src/testing/util.d.ts.map +1 -1
  38. package/package.json +30 -30
  39. package/src/{dbhost → db-host}/database-host.ts +0 -2
  40. package/src/index.ts +2 -2
  41. package/src/pipeline/pipeline.ts +19 -26
  42. package/src/space/control-pipeline.ts +4 -1
  43. package/src/space/data-pipeline.ts +12 -9
  44. package/src/space/space-manager.ts +1 -1
  45. package/src/space/space-protocol.ts +2 -3
  46. package/src/space/space.ts +1 -1
  47. package/src/testing/database-test-rig.ts +1 -1
  48. package/src/testing/test-agent-builder.ts +1 -1
  49. package/src/testing/util.ts +1 -1
  50. package/src/tests/database.test.ts +1 -1
  51. package/dist/lib/browser/chunk-OJGL5427.mjs.map +0 -7
  52. package/dist/types/src/dbhost/data-service-host.d.ts.map +0 -1
  53. package/dist/types/src/dbhost/data-service.d.ts.map +0 -1
  54. package/dist/types/src/dbhost/database-host.d.ts.map +0 -1
  55. package/dist/types/src/dbhost/index.d.ts.map +0 -1
  56. package/dist/types/src/dbhost/snapshot-manager.d.ts.map +0 -1
  57. package/dist/types/src/dbhost/snapshot-store.d.ts.map +0 -1
  58. package/dist/types/src/dbhost/snapshot-store.test.d.ts.map +0 -1
  59. /package/dist/types/src/{dbhost → db-host}/data-service-host.d.ts +0 -0
  60. /package/dist/types/src/{dbhost → db-host}/data-service.d.ts +0 -0
  61. /package/dist/types/src/{dbhost → db-host}/database-host.d.ts +0 -0
  62. /package/dist/types/src/{dbhost → db-host}/index.d.ts +0 -0
  63. /package/dist/types/src/{dbhost → db-host}/snapshot-manager.d.ts +0 -0
  64. /package/dist/types/src/{dbhost → db-host}/snapshot-store.d.ts +0 -0
  65. /package/dist/types/src/{dbhost → db-host}/snapshot-store.test.d.ts +0 -0
  66. /package/src/{dbhost → db-host}/data-service-host.ts +0 -0
  67. /package/src/{dbhost → db-host}/data-service.ts +0 -0
  68. /package/src/{dbhost → db-host}/index.ts +0 -0
  69. /package/src/{dbhost → db-host}/snapshot-manager.ts +0 -0
  70. /package/src/{dbhost → db-host}/snapshot-store.test.ts +0 -0
  71. /package/src/{dbhost → db-host}/snapshot-store.ts +0 -0
@@ -1,14 +1,183 @@
1
1
  import "@dxos/node-std/globals";
2
2
 
3
- // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
4
- import CRC32 from "crc-32";
3
+ // packages/core/echo/echo-pipeline/src/common/codec.ts
4
+ import { createCodecEncoding } from "@dxos/hypercore";
5
+ import { schema } from "@dxos/protocols";
6
+ var codec = schema.getCodecForType("dxos.echo.feed.FeedMessage");
7
+ var valueEncoding = createCodecEncoding(codec);
8
+
9
+ // packages/core/echo/echo-pipeline/src/common/feeds.ts
5
10
  import invariant from "tiny-invariant";
6
- import { synchronized, Event } from "@dxos/async";
7
- import { DataCorruptionError } from "@dxos/errors";
11
+ var createMappedFeedWriter = (mapper, writer) => {
12
+ invariant(mapper);
13
+ invariant(writer);
14
+ return {
15
+ write: async (data, options) => await writer.write(await mapper(data), options)
16
+ };
17
+ };
18
+
19
+ // packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts
20
+ import invariant2 from "tiny-invariant";
21
+ import { Stream } from "@dxos/codec-protobuf";
22
+ import { Context } from "@dxos/context";
23
+ import { tagMutationsInBatch, setMetadataOnObject } from "@dxos/echo-db";
8
24
  import { log } from "@dxos/log";
9
- import { STORAGE_VERSION, schema } from "@dxos/protocols";
10
- import { SpaceState } from "@dxos/protocols/proto/dxos/client/services";
11
- import { arrayToBuffer } from "@dxos/util";
25
+ import { ComplexMap } from "@dxos/util";
26
+ var __dxlog_file = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service-host.ts";
27
+ var DataServiceHost = class {
28
+ constructor(_itemManager, _itemDemuxer, _writeStream) {
29
+ this._itemManager = _itemManager;
30
+ this._itemDemuxer = _itemDemuxer;
31
+ this._writeStream = _writeStream;
32
+ this._ctx = new Context();
33
+ this._clientTagMap = new ComplexMap(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
34
+ }
35
+ async open() {
36
+ }
37
+ async close() {
38
+ await this._ctx.dispose();
39
+ }
40
+ /**
41
+ * Real-time subscription to data objects in a space.
42
+ */
43
+ subscribe() {
44
+ return new Stream(({ next, close, ctx }) => {
45
+ ctx.onDispose(this._ctx.onDispose(close));
46
+ const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
47
+ next({
48
+ batch: {
49
+ objects
50
+ }
51
+ });
52
+ this._itemDemuxer.mutation.on(ctx, (message) => {
53
+ var _a;
54
+ const { batch, meta } = message;
55
+ invariant2(!meta.clientTag, "Unexpected client tag in mutation message");
56
+ log("message", {
57
+ batch,
58
+ meta
59
+ }, {
60
+ F: __dxlog_file,
61
+ L: 63,
62
+ S: this,
63
+ C: (f, a) => f(...a)
64
+ });
65
+ const clientTag = this._clientTagMap.get([
66
+ message.meta.feedKey,
67
+ message.meta.seq
68
+ ]);
69
+ (_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
70
+ setMetadataOnObject(object, {
71
+ ...meta
72
+ });
73
+ });
74
+ if (clientTag) {
75
+ tagMutationsInBatch(batch, clientTag, 0);
76
+ }
77
+ next({
78
+ clientTag,
79
+ feedKey: message.meta.feedKey,
80
+ seq: message.meta.seq,
81
+ batch
82
+ });
83
+ });
84
+ });
85
+ }
86
+ async write(request) {
87
+ var _a, _b;
88
+ invariant2(!this._ctx.disposed, "Cannot write to closed DataServiceHost");
89
+ invariant2(this._writeStream, "Cannot write mutations in readonly mode");
90
+ log("write", {
91
+ clientTag: request.clientTag,
92
+ objectCount: (_b = (_a = request.batch.objects) == null ? void 0 : _a.length) != null ? _b : 0
93
+ }, {
94
+ F: __dxlog_file,
95
+ L: 92,
96
+ S: this,
97
+ C: (f, a) => f(...a)
98
+ });
99
+ const message = createDataMessage(request.batch);
100
+ const receipt = await this._writeStream.write(message, {
101
+ afterWrite: async (receipt2) => {
102
+ if (request.clientTag) {
103
+ log("tag", {
104
+ clientTag: request.clientTag,
105
+ feedKey: receipt2.feedKey,
106
+ seq: receipt2.seq
107
+ }, {
108
+ F: __dxlog_file,
109
+ L: 101,
110
+ S: this,
111
+ C: (f, a) => f(...a)
112
+ });
113
+ this._clientTagMap.set([
114
+ receipt2.feedKey,
115
+ receipt2.seq
116
+ ], request.clientTag);
117
+ }
118
+ }
119
+ });
120
+ return receipt;
121
+ }
122
+ };
123
+ var createDataMessage = (batch) => {
124
+ var _a;
125
+ return {
126
+ batch: {
127
+ objects: (_a = batch.objects) == null ? void 0 : _a.map((object) => {
128
+ var _a2;
129
+ return {
130
+ ...object,
131
+ mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((mutation) => ({
132
+ ...mutation,
133
+ meta: void 0
134
+ })),
135
+ meta: void 0
136
+ };
137
+ })
138
+ }
139
+ };
140
+ };
141
+
142
+ // packages/core/echo/echo-pipeline/src/db-host/database-host.ts
143
+ import { ItemDemuxer } from "@dxos/echo-db";
144
+ var DatabaseHost = class {
145
+ constructor(_outboundStream) {
146
+ this._outboundStream = _outboundStream;
147
+ }
148
+ get isReadOnly() {
149
+ return !!this._outboundStream;
150
+ }
151
+ get echoProcessor() {
152
+ return this._echoProcessor;
153
+ }
154
+ async open(itemManager, modelFactory) {
155
+ this._itemManager = itemManager;
156
+ this._itemManager._debugLabel = "host";
157
+ this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
158
+ this._echoProcessor = this._itemDemuxer.open();
159
+ }
160
+ async close() {
161
+ }
162
+ getWriteStream() {
163
+ return this._outboundStream;
164
+ }
165
+ createSnapshot() {
166
+ return this._itemDemuxer.createSnapshot();
167
+ }
168
+ createDataServiceHost() {
169
+ var _a;
170
+ return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
171
+ }
172
+ };
173
+
174
+ // packages/core/echo/echo-pipeline/src/db-host/snapshot-manager.ts
175
+ import { trackLeaks } from "@dxos/async";
176
+ import { cancelWithContext } from "@dxos/context";
177
+ import { timed } from "@dxos/debug";
178
+ import { PublicKey } from "@dxos/keys";
179
+ import { schema as schema2 } from "@dxos/protocols";
180
+ import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
12
181
  function _ts_decorate(decorators, target, key, desc) {
13
182
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
14
183
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -19,57 +188,223 @@ function _ts_decorate(decorators, target, key, desc) {
19
188
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
20
189
  return c > 3 && r && Object.defineProperty(target, key, r), r;
21
190
  }
22
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
23
- var emptyEchoMetadata = () => ({
24
- version: STORAGE_VERSION,
25
- spaces: [],
26
- created: new Date(),
27
- updated: new Date()
28
- });
29
- var EchoMetadata = schema.getCodecForType("dxos.echo.metadata.EchoMetadata");
30
- var MetadataStore = class {
191
+ var SpaceSnapshot = schema2.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
192
+ var SnapshotManager = class SnapshotManager2 {
31
193
  // prettier-ignore
32
- constructor(_directory) {
33
- this._directory = _directory;
34
- this._metadata = emptyEchoMetadata();
35
- this.update = new Event();
194
+ constructor(_snapshotStore, _blobStore, _blobSync) {
195
+ this._snapshotStore = _snapshotStore;
196
+ this._blobStore = _blobStore;
197
+ this._blobSync = _blobSync;
36
198
  }
37
- get metadata() {
38
- return this._metadata;
199
+ async _getBlob(blobId) {
200
+ const blob = await this._blobStore.get(blobId);
201
+ return SpaceSnapshot.decode(blob);
39
202
  }
40
- get version() {
41
- var _a;
42
- return (_a = this._metadata.version) != null ? _a : 0;
203
+ async load(ctx, id) {
204
+ const blobId = PublicKey.fromHex(id).asUint8Array();
205
+ const blobMeta = await this._blobStore.getMeta(blobId);
206
+ if (blobMeta && blobMeta.state === BlobMeta.State.FULLY_PRESENT) {
207
+ return this._getBlob(blobId);
208
+ }
209
+ const fallbackStore = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
210
+ if (fallbackStore) {
211
+ return fallbackStore;
212
+ }
213
+ await this._blobSync.download(ctx, blobId);
214
+ return this._getBlob(blobId);
43
215
  }
44
- /**
45
- * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
46
- * addSpaceFeed functions.
47
- */
48
- get spaces() {
49
- var _a;
50
- return (_a = this._metadata.spaces) != null ? _a : [];
216
+ async store(snapshot) {
217
+ const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
218
+ await this._blobSync.notifyBlobAdded(id);
219
+ return PublicKey.from(id).toHex();
51
220
  }
52
- /**
53
- * Loads metadata from persistent storage.
54
- */
55
- async load() {
56
- var _a;
57
- const file = this._directory.getOrCreateFile("EchoMetadata");
221
+ };
222
+ _ts_decorate([
223
+ timed(1e4)
224
+ ], SnapshotManager.prototype, "load", null);
225
+ SnapshotManager = _ts_decorate([
226
+ trackLeaks("open", "close")
227
+ ], SnapshotManager);
228
+
229
+ // packages/core/echo/echo-pipeline/src/db-host/snapshot-store.ts
230
+ import { subtleCrypto } from "@dxos/crypto";
231
+ import { schema as schema3 } from "@dxos/protocols";
232
+ var SpaceSnapshot2 = schema3.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
233
+ var SnapshotStore = class {
234
+ // prettier-ignore
235
+ constructor(_directory) {
236
+ this._directory = _directory;
237
+ }
238
+ async saveSnapshot(snapshot) {
239
+ const encoded = SpaceSnapshot2.encode(snapshot);
240
+ const key = await subtleCrypto.digest("SHA-256", encoded);
241
+ const keyString = Buffer.from(key).toString("hex");
242
+ const file = await this._directory.getOrCreateFile(keyString);
58
243
  try {
59
- const { size: fileLength } = await file.stat();
60
- if (fileLength < 8) {
61
- return;
244
+ await file.write(0, Buffer.from(encoded));
245
+ } finally {
246
+ await file.close();
247
+ }
248
+ return keyString;
249
+ }
250
+ async loadSnapshot(key) {
251
+ const file = await this._directory.getOrCreateFile(key);
252
+ try {
253
+ const { size } = await file.stat();
254
+ if (size === 0) {
255
+ return void 0;
62
256
  }
63
- const dataSize = fromBytesInt32(await file.read(0, 4));
64
- const checksum = fromBytesInt32(await file.read(4, 4));
65
- log("loaded", {
66
- size: dataSize,
67
- checksum
68
- }, {
69
- F: __dxlog_file,
70
- L: 72,
71
- S: this,
72
- C: (f, a) => f(...a)
257
+ const buffer = await file.read(0, size);
258
+ return SpaceSnapshot2.decode(buffer);
259
+ } finally {
260
+ await file.close();
261
+ }
262
+ }
263
+ async listSnapshots() {
264
+ const entries = await this._directory.list();
265
+ return await Promise.all(entries.map(async (key) => {
266
+ const { size } = await this._directory.getOrCreateFile(key).stat();
267
+ return {
268
+ key,
269
+ size
270
+ };
271
+ }));
272
+ }
273
+ };
274
+
275
+ // packages/core/echo/echo-pipeline/src/db-host/data-service.ts
276
+ import invariant3 from "tiny-invariant";
277
+ import { raise } from "@dxos/debug";
278
+ import { PublicKey as PublicKey2 } from "@dxos/keys";
279
+ import { log as log2 } from "@dxos/log";
280
+ import { ComplexMap as ComplexMap2 } from "@dxos/util";
281
+ var __dxlog_file2 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/db-host/data-service.ts";
282
+ var DataServiceSubscriptions = class {
283
+ constructor() {
284
+ this._spaces = new ComplexMap2(PublicKey2.hash);
285
+ }
286
+ clear() {
287
+ this._spaces.clear();
288
+ }
289
+ async registerSpace(spaceKey, host) {
290
+ log2("Registering space", {
291
+ spaceKey
292
+ }, {
293
+ F: __dxlog_file2,
294
+ L: 31,
295
+ S: this,
296
+ C: (f, a) => f(...a)
297
+ });
298
+ invariant3(!this._spaces.has(spaceKey));
299
+ await host.open();
300
+ this._spaces.set(spaceKey, host);
301
+ }
302
+ async unregisterSpace(spaceKey) {
303
+ log2("Unregistering space", {
304
+ spaceKey
305
+ }, {
306
+ F: __dxlog_file2,
307
+ L: 38,
308
+ S: this,
309
+ C: (f, a) => f(...a)
310
+ });
311
+ const host = this._spaces.get(spaceKey);
312
+ await (host == null ? void 0 : host.close());
313
+ this._spaces.delete(spaceKey);
314
+ }
315
+ getDataService(spaceKey) {
316
+ return this._spaces.get(spaceKey);
317
+ }
318
+ };
319
+ var DataServiceImpl = class {
320
+ constructor(_subscriptions) {
321
+ this._subscriptions = _subscriptions;
322
+ }
323
+ subscribe(request) {
324
+ var _a;
325
+ invariant3(request.spaceKey);
326
+ const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
327
+ return host.subscribe();
328
+ }
329
+ write(request) {
330
+ var _a;
331
+ invariant3(request.spaceKey);
332
+ invariant3(request.batch);
333
+ const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
334
+ return host.write(request);
335
+ }
336
+ };
337
+
338
+ // packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts
339
+ import CRC32 from "crc-32";
340
+ import invariant4 from "tiny-invariant";
341
+ import { synchronized, Event } from "@dxos/async";
342
+ import { DataCorruptionError } from "@dxos/errors";
343
+ import { log as log3 } from "@dxos/log";
344
+ import { STORAGE_VERSION, schema as schema4 } from "@dxos/protocols";
345
+ import { SpaceState } from "@dxos/protocols/proto/dxos/client/services";
346
+ import { arrayToBuffer } from "@dxos/util";
347
+ function _ts_decorate2(decorators, target, key, desc) {
348
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
349
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
350
+ r = Reflect.decorate(decorators, target, key, desc);
351
+ else
352
+ for (var i = decorators.length - 1; i >= 0; i--)
353
+ if (d = decorators[i])
354
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
355
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
356
+ }
357
+ var __dxlog_file3 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/metadata/metadata-store.ts";
358
+ var emptyEchoMetadata = () => ({
359
+ version: STORAGE_VERSION,
360
+ spaces: [],
361
+ created: new Date(),
362
+ updated: new Date()
363
+ });
364
+ var EchoMetadata = schema4.getCodecForType("dxos.echo.metadata.EchoMetadata");
365
+ var MetadataStore = class {
366
+ // prettier-ignore
367
+ constructor(_directory) {
368
+ this._directory = _directory;
369
+ this._metadata = emptyEchoMetadata();
370
+ this.update = new Event();
371
+ }
372
+ get metadata() {
373
+ return this._metadata;
374
+ }
375
+ get version() {
376
+ var _a;
377
+ return (_a = this._metadata.version) != null ? _a : 0;
378
+ }
379
+ /**
380
+ * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
381
+ * addSpaceFeed functions.
382
+ */
383
+ get spaces() {
384
+ var _a;
385
+ return (_a = this._metadata.spaces) != null ? _a : [];
386
+ }
387
+ /**
388
+ * Loads metadata from persistent storage.
389
+ */
390
+ async load() {
391
+ var _a;
392
+ const file = this._directory.getOrCreateFile("EchoMetadata");
393
+ try {
394
+ const { size: fileLength } = await file.stat();
395
+ if (fileLength < 8) {
396
+ return;
397
+ }
398
+ const dataSize = fromBytesInt32(await file.read(0, 4));
399
+ const checksum = fromBytesInt32(await file.read(4, 4));
400
+ log3("loaded", {
401
+ size: dataSize,
402
+ checksum
403
+ }, {
404
+ F: __dxlog_file3,
405
+ L: 72,
406
+ S: this,
407
+ C: (f, a) => f(...a)
73
408
  });
74
409
  if (fileLength < dataSize + 8) {
75
410
  throw new DataCorruptionError("Metadata size is smaller than expected.");
@@ -85,10 +420,10 @@ var MetadataStore = class {
85
420
  (_a2 = space.state) != null ? _a2 : space.state = SpaceState.ACTIVE;
86
421
  });
87
422
  } catch (err) {
88
- log.error("failed to load metadata", {
423
+ log3.error("failed to load metadata", {
89
424
  err
90
425
  }, {
91
- F: __dxlog_file,
426
+ F: __dxlog_file3,
92
427
  L: 92,
93
428
  S: this,
94
429
  C: (f, a) => f(...a)
@@ -116,11 +451,11 @@ var MetadataStore = class {
116
451
  result.writeInt32LE(checksum, 4);
117
452
  encoded.copy(result, 8);
118
453
  await file.write(0, result);
119
- log("saved", {
454
+ log3("saved", {
120
455
  size: encoded.length,
121
456
  checksum
122
457
  }, {
123
- F: __dxlog_file,
458
+ F: __dxlog_file3,
124
459
  L: 124,
125
460
  S: this,
126
461
  C: (f, a) => f(...a)
@@ -135,15 +470,15 @@ var MetadataStore = class {
135
470
  return this._metadata.identity.haloSpace;
136
471
  }
137
472
  const space = this.spaces.find((space2) => space2.key === spaceKey);
138
- invariant(space, "Space not found");
473
+ invariant4(space, "Space not found");
139
474
  return space;
140
475
  }
141
476
  /**
142
477
  * Clears storage - doesn't work for now.
143
478
  */
144
479
  async clear() {
145
- log("clearing all metadata", void 0, {
146
- F: __dxlog_file,
480
+ log3("clearing all metadata", void 0, {
481
+ F: __dxlog_file3,
147
482
  L: 145,
148
483
  S: this,
149
484
  C: (f, a) => f(...a)
@@ -155,13 +490,13 @@ var MetadataStore = class {
155
490
  return this._metadata.identity;
156
491
  }
157
492
  async setIdentityRecord(record) {
158
- invariant(!this._metadata.identity, "Cannot overwrite existing identity in metadata");
493
+ invariant4(!this._metadata.identity, "Cannot overwrite existing identity in metadata");
159
494
  this._metadata.identity = record;
160
495
  await this._save();
161
496
  }
162
497
  async addSpace(record) {
163
498
  var _a, _b, _c;
164
- invariant(!((_a = this._metadata.spaces) != null ? _a : []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata");
499
+ invariant4(!((_a = this._metadata.spaces) != null ? _a : []).find((space) => space.key === record.key), "Cannot overwrite existing space in metadata");
165
500
  ((_c = (_b = this._metadata).spaces) != null ? _c : _b.spaces = []).push(record);
166
501
  await this._save();
167
502
  }
@@ -188,36 +523,20 @@ var MetadataStore = class {
188
523
  await this._save();
189
524
  }
190
525
  };
191
- _ts_decorate([
526
+ _ts_decorate2([
192
527
  synchronized
193
528
  ], MetadataStore.prototype, "load", null);
194
- _ts_decorate([
529
+ _ts_decorate2([
195
530
  synchronized
196
531
  ], MetadataStore.prototype, "_save", null);
197
532
  var fromBytesInt32 = (buf) => buf.readInt32LE(0);
198
533
 
199
- // packages/core/echo/echo-pipeline/src/common/codec.ts
200
- import { createCodecEncoding } from "@dxos/hypercore";
201
- import { schema as schema2 } from "@dxos/protocols";
202
- var codec = schema2.getCodecForType("dxos.echo.feed.FeedMessage");
203
- var valueEncoding = createCodecEncoding(codec);
204
-
205
- // packages/core/echo/echo-pipeline/src/common/feeds.ts
206
- import invariant2 from "tiny-invariant";
207
- var createMappedFeedWriter = (mapper, writer) => {
208
- invariant2(mapper);
209
- invariant2(writer);
210
- return {
211
- write: async (data, options) => await writer.write(await mapper(data), options)
212
- };
213
- };
214
-
215
534
  // packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts
216
535
  import { Event as Event2 } from "@dxos/async";
217
- import { timed } from "@dxos/debug";
218
- import { log as log2 } from "@dxos/log";
536
+ import { timed as timed2 } from "@dxos/debug";
537
+ import { log as log4 } from "@dxos/log";
219
538
  import { Timeframe } from "@dxos/timeframe";
220
- function _ts_decorate2(decorators, target, key, desc) {
539
+ function _ts_decorate3(decorators, target, key, desc) {
221
540
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
222
541
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
223
542
  r = Reflect.decorate(decorators, target, key, desc);
@@ -227,7 +546,7 @@ function _ts_decorate2(decorators, target, key, desc) {
227
546
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
228
547
  return c > 3 && r && Object.defineProperty(target, key, r), r;
229
548
  }
230
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
549
+ var __dxlog_file4 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/timeframe-clock.ts";
231
550
  var mapTimeframeToFeedIndexes = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
232
551
  feedKey,
233
552
  index
@@ -242,829 +561,510 @@ var startAfter = (timeframe) => timeframe.frames().map(([feedKey, index]) => ({
242
561
  }));
243
562
  var TimeframeClock = class {
244
563
  // prettier-ignore
245
- constructor(_timeframe = new Timeframe()) {
246
- this._timeframe = _timeframe;
247
- this.update = new Event2();
248
- this._pendingTimeframe = _timeframe;
249
- }
250
- /**
251
- * Timeframe that was processed by ECHO.
252
- */
253
- get timeframe() {
254
- return this._timeframe;
255
- }
256
- /**
257
- * Timeframe that is currently being processed by ECHO.
258
- * Will be equal to `timeframe` after the processing is complete.
259
- */
260
- get pendingTimeframe() {
261
- return this._pendingTimeframe;
262
- }
263
- setTimeframe(timeframe) {
264
- this._timeframe = timeframe;
265
- this._pendingTimeframe = timeframe;
266
- this.update.emit(this._timeframe);
267
- }
268
- updatePendingTimeframe(key, seq) {
269
- this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([
270
- [
271
- key,
272
- seq
273
- ]
274
- ]));
275
- }
276
- updateTimeframe() {
277
- this._timeframe = this._pendingTimeframe;
278
- this.update.emit(this._timeframe);
279
- }
280
- hasGaps(timeframe) {
281
- const gaps = Timeframe.dependencies(timeframe, this._timeframe);
282
- return !gaps.isEmpty();
283
- }
284
- async waitUntilReached(target) {
285
- log2("waitUntilReached", {
286
- target,
287
- current: this._timeframe
288
- }, {
289
- F: __dxlog_file2,
290
- L: 73,
291
- S: this,
292
- C: (f, a) => f(...a)
293
- });
294
- await this.update.waitForCondition(() => {
295
- log2("check if reached", {
296
- target,
297
- current: this._timeframe,
298
- deps: Timeframe.dependencies(target, this._timeframe)
299
- }, {
300
- F: __dxlog_file2,
301
- L: 75,
302
- S: this,
303
- C: (f, a) => f(...a)
304
- });
305
- return Timeframe.dependencies(target, this._timeframe).isEmpty();
306
- });
307
- }
308
- };
309
- _ts_decorate2([
310
- timed(5e3)
311
- ], TimeframeClock.prototype, "waitUntilReached", null);
312
-
313
- // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
314
- import invariant4 from "tiny-invariant";
315
- import { Event as Event3, sleep, synchronized as synchronized2, Trigger } from "@dxos/async";
316
- import { Context, rejectOnDispose } from "@dxos/context";
317
- import { failUndefined } from "@dxos/debug";
318
- import { FeedSetIterator } from "@dxos/feed-store";
319
- import { PublicKey } from "@dxos/keys";
320
- import { log as log4 } from "@dxos/log";
321
- import { Timeframe as Timeframe2 } from "@dxos/timeframe";
322
- import { ComplexMap } from "@dxos/util";
323
-
324
- // packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
325
- import invariant3 from "tiny-invariant";
326
- import { log as log3 } from "@dxos/log";
327
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
328
- var createMessageSelector = (timeframeClock) => {
329
- return (messages) => {
330
- for (let i = 0; i < messages.length; i++) {
331
- const { data: { timeframe } } = messages[i];
332
- invariant3(timeframe);
333
- if (!timeframeClock.hasGaps(timeframe)) {
334
- return i;
335
- }
336
- }
337
- log3("Skipping...", void 0, {
338
- F: __dxlog_file3,
339
- L: 34,
340
- S: void 0,
341
- C: (f, a) => f(...a)
342
- });
343
- };
344
- };
345
-
346
- // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
347
- function _ts_decorate3(decorators, target, key, desc) {
348
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
349
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
350
- r = Reflect.decorate(decorators, target, key, desc);
351
- else
352
- for (var i = decorators.length - 1; i >= 0; i--)
353
- if (d = decorators[i])
354
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
355
- return c > 3 && r && Object.defineProperty(target, key, r), r;
356
- }
357
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
358
- var PipelineState = class {
359
- // prettier-ignore
360
- constructor(_feeds, _timeframeClock) {
361
- this._feeds = _feeds;
362
- this._timeframeClock = _timeframeClock;
363
- this._ctx = new Context();
364
- this.timeframeUpdate = this._timeframeClock.update;
365
- this.stalled = new Event3();
366
- this._startTimeframe = new Timeframe2();
367
- }
368
- /**
369
- * Latest theoretical timeframe based on the last mutation in each feed.
370
- * NOTE: This might never be reached if the mutation dependencies
371
- */
372
- // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
373
- get endTimeframe() {
374
- return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
375
- feedKey: feed.key,
376
- index: feed.length - 1
377
- })));
378
- }
379
- get startTimeframe() {
380
- return this._startTimeframe;
381
- }
382
- get timeframe() {
383
- return this._timeframeClock.timeframe;
384
- }
385
- get pendingTimeframe() {
386
- return this._timeframeClock.pendingTimeframe;
387
- }
388
- get targetTimeframe() {
389
- return this._targetTimeframe ? this._targetTimeframe : new Timeframe2();
390
- }
391
- get feeds() {
392
- return Array.from(this._feeds.values());
393
- }
394
- async waitUntilTimeframe(target) {
395
- await this._timeframeClock.waitUntilReached(target);
396
- }
397
- setTargetTimeframe(target) {
398
- this._targetTimeframe = target;
399
- }
400
- /**
401
- * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
402
- *
403
- * This function will resolve immediately if the pipeline is stalled.
404
- *
405
- * @param timeout Timeout in milliseconds to specify the maximum wait time.
406
- */
407
- async waitUntilReachedTargetTimeframe({ ctx = new Context(), timeout, breakOnStall = true } = {}) {
408
- var _a;
409
- log4("waitUntilReachedTargetTimeframe", {
410
- timeout,
411
- current: this.timeframe,
412
- target: this.targetTimeframe
413
- }, {
414
- F: __dxlog_file4,
415
- L: 126,
416
- S: this,
417
- C: (f, a) => f(...a)
418
- });
419
- (_a = this._reachedTargetPromise) != null ? _a : this._reachedTargetPromise = Promise.race([
420
- this._timeframeClock.update.waitForCondition(() => {
421
- return Timeframe2.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
422
- }),
423
- ...breakOnStall ? [
424
- this.stalled.discardParameter().waitForCount(1)
425
- ] : []
426
- ]);
427
- let done = false;
428
- if (timeout) {
429
- return Promise.race([
430
- rejectOnDispose(ctx),
431
- rejectOnDispose(this._ctx),
432
- this._reachedTargetPromise.then(() => {
433
- done = true;
434
- }),
435
- sleep(timeout).then(() => {
436
- if (done) {
437
- return;
438
- }
439
- log4.warn("waitUntilReachedTargetTimeframe timed out", {
440
- timeout,
441
- current: this.timeframe,
442
- target: this.targetTimeframe,
443
- dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
444
- }, {
445
- F: __dxlog_file4,
446
- L: 153,
447
- S: this,
448
- C: (f, a) => f(...a)
449
- });
450
- })
451
- ]);
452
- } else {
453
- return this._reachedTargetPromise;
454
- }
455
- }
456
- };
457
- var Pipeline = class {
458
- constructor() {
459
- this._timeframeClock = new TimeframeClock(new Timeframe2());
460
- this._feeds = new ComplexMap(PublicKey.hash);
461
- // External state accessor.
462
- this._state = new PipelineState(this._feeds, this._timeframeClock);
463
- // Waits for the message consumer to process the message and yield control back to the pipeline.
464
- this._processingTrigger = new Trigger().wake();
465
- this._pauseTrigger = new Trigger().wake();
466
- this._isStopping = false;
467
- this._isStarted = false;
468
- this._isOpen = false;
469
- this._isPaused = false;
470
- }
471
- get state() {
472
- return this._state;
473
- }
474
- get writer() {
475
- invariant4(this._writer, "Writer not set.");
476
- return this._writer;
477
- }
478
- getFeeds() {
479
- return this._feedSetIterator.feeds;
480
- }
481
- // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
482
- // which might be opening feeds during the mutation processing, which w
483
- async addFeed(feed) {
484
- this._feeds.set(feed.key, feed);
485
- if (this._feedSetIterator) {
486
- await this._feedSetIterator.addFeed(feed);
487
- }
488
- this._setFeedDownloadState(feed);
489
- }
490
- hasFeed(feedKey) {
491
- return this._feeds.has(feedKey);
492
- }
493
- setWriteFeed(feed) {
494
- invariant4(!this._writer, "Writer already set.");
495
- invariant4(feed.properties.writable, "Feed must be writable.");
496
- this._writer = createMappedFeedWriter((payload) => ({
497
- timeframe: this._timeframeClock.timeframe,
498
- payload
499
- }), feed.createFeedWriter());
500
- }
501
- async start() {
502
- log4("starting...", {}, {
503
- F: __dxlog_file4,
504
- L: 273,
505
- S: this,
506
- C: (f, a) => f(...a)
507
- });
508
- await this._initIterator();
509
- await this._feedSetIterator.open();
510
- log4("started", void 0, {
511
- F: __dxlog_file4,
512
- L: 276,
513
- S: this,
514
- C: (f, a) => f(...a)
515
- });
516
- this._isStarted = true;
517
- }
518
- async stop() {
519
- var _a;
520
- log4("stopping...", {}, {
521
- F: __dxlog_file4,
522
- L: 282,
523
- S: this,
524
- C: (f, a) => f(...a)
525
- });
526
- this._isStopping = true;
527
- await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
528
- await this._processingTrigger.wait();
529
- await this._state._ctx.dispose();
530
- this._state._ctx = new Context();
531
- this._state._reachedTargetPromise = void 0;
532
- log4("stopped", void 0, {
533
- F: __dxlog_file4,
534
- L: 289,
535
- S: this,
536
- C: (f, a) => f(...a)
537
- });
538
- this._isStarted = false;
539
- }
540
- /**
541
- * @param timeframe Timeframe of already processed messages. The pipeline will start processing messages AFTER this timeframe.
542
- */
543
- async setCursor(timeframe) {
544
- invariant4(!this._isStarted || this._isPaused, "Invalid state.");
545
- this._state._startTimeframe = timeframe;
546
- this._timeframeClock.setTimeframe(timeframe);
547
- for (const feed of this._feeds.values()) {
548
- this._setFeedDownloadState(feed);
549
- }
550
- if (this._feedSetIterator) {
551
- await this._feedSetIterator.close();
552
- await this._initIterator();
553
- await this._feedSetIterator.open();
554
- }
564
+ constructor(_timeframe = new Timeframe()) {
565
+ this._timeframe = _timeframe;
566
+ this.update = new Event2();
567
+ this._pendingTimeframe = _timeframe;
555
568
  }
556
569
  /**
557
- * Calling pause while processing will cause a deadlock.
570
+ * Timeframe that was processed by ECHO.
558
571
  */
559
- async pause() {
560
- invariant4(this._isStarted, "Pipeline is not open.");
561
- if (this._isPaused) {
562
- return;
563
- }
564
- this._pauseTrigger.reset();
565
- await this._processingTrigger.wait();
566
- this._isPaused = true;
567
- }
568
- async unpause() {
569
- invariant4(this._isStarted, "Pipeline is not open.");
570
- invariant4(this._isPaused, "Pipeline is not paused.");
571
- this._pauseTrigger.wake();
572
- this._isPaused = false;
572
+ get timeframe() {
573
+ return this._timeframe;
573
574
  }
574
575
  /**
575
- * Starts to iterate over the ordered messages from the added feeds.
576
- * Updates the timeframe clock after the message has bee processed.
576
+ * Timeframe that is currently being processed by ECHO.
577
+ * Will be equal to `timeframe` after the processing is complete.
577
578
  */
578
- async *consume() {
579
- invariant4(!this._isOpen, "Pipeline is already being consumed.");
580
- this._isOpen = true;
581
- invariant4(this._feedSetIterator, "Iterator not initialized.");
582
- let lastFeedSetIterator = this._feedSetIterator;
583
- let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
584
- while (!this._isStopping) {
585
- await this._pauseTrigger.wait();
586
- if (lastFeedSetIterator !== this._feedSetIterator) {
587
- invariant4(this._feedSetIterator, "Iterator not initialized.");
588
- lastFeedSetIterator = this._feedSetIterator;
589
- iterable = lastFeedSetIterator[Symbol.asyncIterator]();
590
- }
591
- const { done, value } = await iterable.next();
592
- if (done) {
593
- continue;
594
- }
595
- const block = value != null ? value : failUndefined();
596
- this._processingTrigger.reset();
597
- this._timeframeClock.updatePendingTimeframe(PublicKey.from(block.feedKey), block.seq);
598
- yield block;
599
- this._processingTrigger.wake();
600
- this._timeframeClock.updateTimeframe();
601
- }
602
- this._isOpen = false;
579
+ get pendingTimeframe() {
580
+ return this._pendingTimeframe;
603
581
  }
604
- _setFeedDownloadState(feed) {
605
- var _a;
606
- const timeframe = this._state._startTimeframe;
607
- const seq = (_a = timeframe.get(feed.key)) != null ? _a : 0;
608
- feed.undownload({
609
- callback: () => log4("Undownloaded", void 0, {
610
- F: __dxlog_file4,
611
- L: 383,
612
- S: this,
613
- C: (f, a) => f(...a)
614
- })
615
- });
616
- feed.download({
617
- start: seq + 1,
618
- linear: true
619
- }).catch((err) => {
620
- log4("failed to download feed", {
621
- err
622
- }, {
623
- F: __dxlog_file4,
624
- L: 386,
625
- S: this,
626
- C: (f, a) => f(...a)
627
- });
628
- });
582
+ setTimeframe(timeframe) {
583
+ this._timeframe = timeframe;
584
+ this._pendingTimeframe = timeframe;
585
+ this.update.emit(this._timeframe);
629
586
  }
630
- async _initIterator() {
631
- this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
632
- start: startAfter(this._timeframeClock.timeframe),
633
- stallTimeout: 1e3
587
+ updatePendingTimeframe(key, seq) {
588
+ this._pendingTimeframe = Timeframe.merge(this._pendingTimeframe, new Timeframe([
589
+ [
590
+ key,
591
+ seq
592
+ ]
593
+ ]));
594
+ }
595
+ updateTimeframe() {
596
+ this._timeframe = this._pendingTimeframe;
597
+ this.update.emit(this._timeframe);
598
+ }
599
+ hasGaps(timeframe) {
600
+ const gaps = Timeframe.dependencies(timeframe, this._timeframe);
601
+ return !gaps.isEmpty();
602
+ }
603
+ async waitUntilReached(target) {
604
+ log4("waitUntilReached", {
605
+ target,
606
+ current: this._timeframe
607
+ }, {
608
+ F: __dxlog_file4,
609
+ L: 73,
610
+ S: this,
611
+ C: (f, a) => f(...a)
634
612
  });
635
- this._feedSetIterator.stalled.on((iterator) => {
636
- log4.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
613
+ await this.update.waitForCondition(() => {
614
+ log4("check if reached", {
615
+ target,
616
+ current: this._timeframe,
617
+ deps: Timeframe.dependencies(target, this._timeframe)
618
+ }, {
637
619
  F: __dxlog_file4,
638
- L: 396,
620
+ L: 75,
639
621
  S: this,
640
622
  C: (f, a) => f(...a)
641
623
  });
642
- this._state.stalled.emit();
624
+ return Timeframe.dependencies(target, this._timeframe).isEmpty();
643
625
  });
644
- for (const feed of this._feeds.values()) {
645
- await this._feedSetIterator.addFeed(feed);
646
- }
647
626
  }
648
627
  };
649
628
  _ts_decorate3([
650
- synchronized2
651
- ], Pipeline.prototype, "start", null);
652
- _ts_decorate3([
653
- synchronized2
654
- ], Pipeline.prototype, "stop", null);
655
- _ts_decorate3([
656
- synchronized2
657
- ], Pipeline.prototype, "setCursor", null);
658
- _ts_decorate3([
659
- synchronized2
660
- ], Pipeline.prototype, "pause", null);
661
- _ts_decorate3([
662
- synchronized2
663
- ], Pipeline.prototype, "unpause", null);
629
+ timed2(5e3)
630
+ ], TimeframeClock.prototype, "waitUntilReached", null);
664
631
 
665
- // packages/core/echo/echo-pipeline/src/space/auth.ts
632
+ // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
633
+ import invariant6 from "tiny-invariant";
634
+ import { Event as Event3, sleep, synchronized as synchronized2, Trigger } from "@dxos/async";
635
+ import { Context as Context2, rejectOnDispose } from "@dxos/context";
636
+ import { failUndefined } from "@dxos/debug";
637
+ import { FeedSetIterator } from "@dxos/feed-store";
638
+ import { PublicKey as PublicKey3 } from "@dxos/keys";
639
+ import { log as log6 } from "@dxos/log";
640
+ import { Timeframe as Timeframe2 } from "@dxos/timeframe";
641
+ import { ComplexMap as ComplexMap3 } from "@dxos/util";
642
+
643
+ // packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts
666
644
  import invariant5 from "tiny-invariant";
667
- import { runInContext, scheduleTask } from "@dxos/async";
668
- import { Context as Context2 } from "@dxos/context";
669
- import { randomBytes } from "@dxos/crypto";
670
645
  import { log as log5 } from "@dxos/log";
671
- import { schema as schema3 } from "@dxos/protocols";
672
- import { RpcExtension } from "@dxos/teleport";
673
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/auth.ts";
674
- var AuthExtension = class extends RpcExtension {
675
- constructor(_authParams) {
676
- super({
677
- requested: {
678
- AuthService: schema3.getService("dxos.mesh.teleport.auth.AuthService")
679
- },
680
- exposed: {
681
- AuthService: schema3.getService("dxos.mesh.teleport.auth.AuthService")
682
- },
683
- timeout: 60 * 1e3
684
- });
685
- this._authParams = _authParams;
686
- this._ctx = new Context2({
687
- onError: (err) => {
688
- log5.catch(err, void 0, {
689
- F: __dxlog_file5,
690
- L: 29,
691
- S: this,
692
- C: (f, a) => f(...a)
693
- });
646
+ var __dxlog_file5 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/message-selector.ts";
647
+ var createMessageSelector = (timeframeClock) => {
648
+ return (messages) => {
649
+ for (let i = 0; i < messages.length; i++) {
650
+ const { data: { timeframe } } = messages[i];
651
+ invariant5(timeframe);
652
+ if (!timeframeClock.hasGaps(timeframe)) {
653
+ return i;
694
654
  }
655
+ }
656
+ log5("Skipping...", void 0, {
657
+ F: __dxlog_file5,
658
+ L: 34,
659
+ S: void 0,
660
+ C: (f, a) => f(...a)
695
661
  });
662
+ };
663
+ };
664
+
665
+ // packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts
666
+ function _ts_decorate4(decorators, target, key, desc) {
667
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
668
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
669
+ r = Reflect.decorate(decorators, target, key, desc);
670
+ else
671
+ for (var i = decorators.length - 1; i >= 0; i--)
672
+ if (d = decorators[i])
673
+ r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
674
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
675
+ }
676
+ var __dxlog_file6 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/pipeline/pipeline.ts";
677
+ var PipelineState = class {
678
+ // prettier-ignore
679
+ constructor(_feeds, _timeframeClock) {
680
+ this._feeds = _feeds;
681
+ this._timeframeClock = _timeframeClock;
682
+ this._ctx = new Context2();
683
+ this.timeframeUpdate = this._timeframeClock.update;
684
+ this.stalled = new Event3();
685
+ this._startTimeframe = new Timeframe2();
696
686
  }
697
- async getHandlers() {
698
- return {
699
- AuthService: {
700
- authenticate: async ({ challenge }) => {
701
- try {
702
- const credential = await this._authParams.provider(challenge);
703
- if (!credential) {
704
- throw new Error("auth rejected");
705
- }
706
- return {
707
- credential
708
- };
709
- } catch (err) {
710
- log5.error("failed to generate auth credentials", err, {
711
- F: __dxlog_file5,
712
- L: 56,
713
- S: this,
714
- C: (f, a) => f(...a)
715
- });
716
- throw new Error("auth rejected");
717
- }
718
- }
719
- }
720
- };
687
+ /**
688
+ * Latest theoretical timeframe based on the last mutation in each feed.
689
+ * NOTE: This might never be reached if the mutation dependencies
690
+ */
691
+ // TODO(dmaretskyi): Rename `totalTimeframe`? or `lastTimeframe`.
692
+ get endTimeframe() {
693
+ return mapFeedIndexesToTimeframe(Array.from(this._feeds.values()).filter((feed) => feed.length > 0).map((feed) => ({
694
+ feedKey: feed.key,
695
+ index: feed.length - 1
696
+ })));
721
697
  }
722
- async onOpen(context) {
723
- await super.onOpen(context);
724
- scheduleTask(this._ctx, async () => {
725
- try {
726
- const challenge = randomBytes(32);
727
- const { credential } = await this.rpc.AuthService.authenticate({
728
- challenge
729
- });
730
- invariant5((credential == null ? void 0 : credential.length) > 0, "invalid credential");
731
- const success = await this._authParams.verifier(challenge, credential);
732
- invariant5(success, "credential not verified");
733
- runInContext(this._ctx, () => this._authParams.onAuthSuccess());
734
- } catch (err) {
735
- log5("auth failed", err, {
736
- F: __dxlog_file5,
737
- L: 75,
738
- S: this,
739
- C: (f, a) => f(...a)
740
- });
741
- this.close();
742
- this._authParams.onAuthFailure();
743
- }
744
- });
698
+ get startTimeframe() {
699
+ return this._startTimeframe;
745
700
  }
746
- async onClose() {
747
- await this._ctx.dispose();
748
- await super.onClose();
701
+ get timeframe() {
702
+ return this._timeframeClock.timeframe;
749
703
  }
750
- };
751
-
752
- // packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts
753
- import invariant6 from "tiny-invariant";
754
- import { Stream } from "@dxos/codec-protobuf";
755
- import { Context as Context3 } from "@dxos/context";
756
- import { tagMutationsInBatch, setMetadataOnObject } from "@dxos/echo-db";
757
- import { log as log6 } from "@dxos/log";
758
- import { ComplexMap as ComplexMap2 } from "@dxos/util";
759
- var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/dbhost/data-service-host.ts";
760
- var DataServiceHost = class {
761
- constructor(_itemManager, _itemDemuxer, _writeStream) {
762
- this._itemManager = _itemManager;
763
- this._itemDemuxer = _itemDemuxer;
764
- this._writeStream = _writeStream;
765
- this._ctx = new Context3();
766
- this._clientTagMap = new ComplexMap2(([feedKey, seq]) => `${feedKey.toHex()}:${seq}`);
704
+ get pendingTimeframe() {
705
+ return this._timeframeClock.pendingTimeframe;
706
+ }
707
+ get targetTimeframe() {
708
+ return this._targetTimeframe ? this._targetTimeframe : new Timeframe2();
767
709
  }
768
- async open() {
710
+ get feeds() {
711
+ return Array.from(this._feeds.values());
769
712
  }
770
- async close() {
771
- await this._ctx.dispose();
713
+ async waitUntilTimeframe(target) {
714
+ await this._timeframeClock.waitUntilReached(target);
715
+ }
716
+ setTargetTimeframe(target) {
717
+ this._targetTimeframe = target;
772
718
  }
773
719
  /**
774
- * Real-time subscription to data objects in a space.
720
+ * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
721
+ *
722
+ * This function will resolve immediately if the pipeline is stalled.
723
+ *
724
+ * @param timeout Timeout in milliseconds to specify the maximum wait time.
775
725
  */
776
- subscribe() {
777
- return new Stream(({ next, close, ctx }) => {
778
- ctx.onDispose(this._ctx.onDispose(close));
779
- const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
780
- next({
781
- batch: {
782
- objects
783
- }
784
- });
785
- this._itemDemuxer.mutation.on(ctx, (message) => {
786
- var _a;
787
- const { batch, meta } = message;
788
- invariant6(!meta.clientTag, "Unexpected client tag in mutation message");
789
- log6("message", {
790
- batch,
791
- meta
792
- }, {
793
- F: __dxlog_file6,
794
- L: 63,
795
- S: this,
796
- C: (f, a) => f(...a)
797
- });
798
- const clientTag = this._clientTagMap.get([
799
- message.meta.feedKey,
800
- message.meta.seq
801
- ]);
802
- (_a = batch.objects) == null ? void 0 : _a.forEach((object) => {
803
- setMetadataOnObject(object, {
804
- ...meta
805
- });
806
- });
807
- if (clientTag) {
808
- tagMutationsInBatch(batch, clientTag, 0);
809
- }
810
- next({
811
- clientTag,
812
- feedKey: message.meta.feedKey,
813
- seq: message.meta.seq,
814
- batch
815
- });
816
- });
817
- });
818
- }
819
- async write(request) {
820
- var _a, _b;
821
- invariant6(!this._ctx.disposed, "Cannot write to closed DataServiceHost");
822
- invariant6(this._writeStream, "Cannot write mutations in readonly mode");
823
- log6("write", {
824
- clientTag: request.clientTag,
825
- objectCount: (_b = (_a = request.batch.objects) == null ? void 0 : _a.length) != null ? _b : 0
726
+ async waitUntilReachedTargetTimeframe({ ctx = new Context2(), timeout, breakOnStall = true } = {}) {
727
+ var _a;
728
+ log6("waitUntilReachedTargetTimeframe", {
729
+ timeout,
730
+ current: this.timeframe,
731
+ target: this.targetTimeframe
826
732
  }, {
827
733
  F: __dxlog_file6,
828
- L: 92,
734
+ L: 126,
829
735
  S: this,
830
736
  C: (f, a) => f(...a)
831
737
  });
832
- const message = createDataMessage(request.batch);
833
- const receipt = await this._writeStream.write(message, {
834
- afterWrite: async (receipt2) => {
835
- if (request.clientTag) {
836
- log6("tag", {
837
- clientTag: request.clientTag,
838
- feedKey: receipt2.feedKey,
839
- seq: receipt2.seq
738
+ (_a = this._reachedTargetPromise) != null ? _a : this._reachedTargetPromise = Promise.race([
739
+ this._timeframeClock.update.waitForCondition(() => {
740
+ return Timeframe2.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
741
+ }),
742
+ ...breakOnStall ? [
743
+ this.stalled.discardParameter().waitForCount(1)
744
+ ] : []
745
+ ]);
746
+ let done = false;
747
+ if (timeout) {
748
+ return Promise.race([
749
+ rejectOnDispose(ctx),
750
+ rejectOnDispose(this._ctx),
751
+ this._reachedTargetPromise.then(() => {
752
+ done = true;
753
+ }),
754
+ sleep(timeout).then(() => {
755
+ if (done) {
756
+ return;
757
+ }
758
+ log6.warn("waitUntilReachedTargetTimeframe timed out", {
759
+ timeout,
760
+ current: this.timeframe,
761
+ target: this.targetTimeframe,
762
+ dependencies: Timeframe2.dependencies(this.targetTimeframe, this.timeframe)
840
763
  }, {
841
764
  F: __dxlog_file6,
842
- L: 101,
765
+ L: 153,
843
766
  S: this,
844
767
  C: (f, a) => f(...a)
845
768
  });
846
- this._clientTagMap.set([
847
- receipt2.feedKey,
848
- receipt2.seq
849
- ], request.clientTag);
850
- }
851
- }
852
- });
853
- return receipt;
854
- }
855
- };
856
- var createDataMessage = (batch) => {
857
- var _a;
858
- return {
859
- batch: {
860
- objects: (_a = batch.objects) == null ? void 0 : _a.map((object) => {
861
- var _a2;
862
- return {
863
- ...object,
864
- mutations: (_a2 = object.mutations) == null ? void 0 : _a2.map((mutation) => ({
865
- ...mutation,
866
- meta: void 0
867
- })),
868
- meta: void 0
869
- };
870
- })
769
+ })
770
+ ]);
771
+ } else {
772
+ return this._reachedTargetPromise;
871
773
  }
872
- };
873
- };
874
-
875
- // packages/core/echo/echo-pipeline/src/dbhost/database-host.ts
876
- import { ItemDemuxer } from "@dxos/echo-db";
877
- var DatabaseHost = class {
878
- constructor(_outboundStream) {
879
- this._outboundStream = _outboundStream;
880
774
  }
881
- get isReadOnly() {
882
- return !!this._outboundStream;
775
+ };
776
+ var Pipeline = class {
777
+ constructor() {
778
+ this._timeframeClock = new TimeframeClock(new Timeframe2());
779
+ this._feeds = new ComplexMap3(PublicKey3.hash);
780
+ // External state accessor.
781
+ this._state = new PipelineState(this._feeds, this._timeframeClock);
782
+ // Waits for the message consumer to process the message and yield control back to the pipeline.
783
+ this._processingTrigger = new Trigger().wake();
784
+ this._pauseTrigger = new Trigger().wake();
785
+ this._isStopping = false;
786
+ this._isStarted = false;
787
+ this._isOpen = false;
788
+ this._isPaused = false;
883
789
  }
884
- get echoProcessor() {
885
- return this._echoProcessor;
790
+ get state() {
791
+ return this._state;
886
792
  }
887
- async open(itemManager, modelFactory) {
888
- this._itemManager = itemManager;
889
- this._itemManager._debugLabel = "host";
890
- this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory);
891
- this._echoProcessor = this._itemDemuxer.open();
793
+ get writer() {
794
+ invariant6(this._writer, "Writer not set.");
795
+ return this._writer;
892
796
  }
893
- async close() {
797
+ hasFeed(feedKey) {
798
+ return this._feeds.has(feedKey);
894
799
  }
895
- getWriteStream() {
896
- return this._outboundStream;
800
+ getFeeds() {
801
+ return this._feedSetIterator.feeds;
897
802
  }
898
- createSnapshot() {
899
- return this._itemDemuxer.createSnapshot();
803
+ // NOTE: This cannot be synchronized with `stop` because stop waits for the mutation processing to complete,
804
+ // which might be opening feeds during the mutation processing, which w
805
+ async addFeed(feed) {
806
+ this._feeds.set(feed.key, feed);
807
+ if (this._feedSetIterator) {
808
+ await this._feedSetIterator.addFeed(feed);
809
+ }
810
+ this._setFeedDownloadState(feed);
900
811
  }
901
- createDataServiceHost() {
902
- var _a;
903
- return new DataServiceHost(this._itemManager, this._itemDemuxer, (_a = this._outboundStream) != null ? _a : void 0);
812
+ setWriteFeed(feed) {
813
+ invariant6(!this._writer, "Writer already set.");
814
+ invariant6(feed.properties.writable, "Feed must be writable.");
815
+ this._writer = createMappedFeedWriter((payload) => ({
816
+ timeframe: this._timeframeClock.timeframe,
817
+ payload
818
+ }), feed.createFeedWriter());
904
819
  }
905
- };
906
-
907
- // packages/core/echo/echo-pipeline/src/dbhost/snapshot-manager.ts
908
- import { trackLeaks } from "@dxos/async";
909
- import { cancelWithContext } from "@dxos/context";
910
- import { timed as timed2 } from "@dxos/debug";
911
- import { PublicKey as PublicKey2 } from "@dxos/keys";
912
- import { schema as schema4 } from "@dxos/protocols";
913
- import { BlobMeta } from "@dxos/protocols/proto/dxos/echo/blob";
914
- function _ts_decorate4(decorators, target, key, desc) {
915
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
916
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
917
- r = Reflect.decorate(decorators, target, key, desc);
918
- else
919
- for (var i = decorators.length - 1; i >= 0; i--)
920
- if (d = decorators[i])
921
- r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
922
- return c > 3 && r && Object.defineProperty(target, key, r), r;
923
- }
924
- var SpaceSnapshot = schema4.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
925
- var SnapshotManager = class SnapshotManager2 {
926
- // prettier-ignore
927
- constructor(_snapshotStore, _blobStore, _blobSync) {
928
- this._snapshotStore = _snapshotStore;
929
- this._blobStore = _blobStore;
930
- this._blobSync = _blobSync;
820
+ async start() {
821
+ log6("starting...", void 0, {
822
+ F: __dxlog_file6,
823
+ L: 268,
824
+ S: this,
825
+ C: (f, a) => f(...a)
826
+ });
827
+ await this._initIterator();
828
+ await this._feedSetIterator.open();
829
+ this._isStarted = true;
830
+ log6("started", void 0, {
831
+ F: __dxlog_file6,
832
+ L: 272,
833
+ S: this,
834
+ C: (f, a) => f(...a)
835
+ });
931
836
  }
932
- async _getBlob(blobId) {
933
- const blob = await this._blobStore.get(blobId);
934
- return SpaceSnapshot.decode(blob);
837
+ async stop() {
838
+ var _a;
839
+ log6("stopping...", void 0, {
840
+ F: __dxlog_file6,
841
+ L: 277,
842
+ S: this,
843
+ C: (f, a) => f(...a)
844
+ });
845
+ this._isStopping = true;
846
+ await ((_a = this._feedSetIterator) == null ? void 0 : _a.close());
847
+ await this._processingTrigger.wait();
848
+ await this._state._ctx.dispose();
849
+ this._state._ctx = new Context2();
850
+ this._state._reachedTargetPromise = void 0;
851
+ this._isStarted = false;
852
+ log6("stopped", void 0, {
853
+ F: __dxlog_file6,
854
+ L: 285,
855
+ S: this,
856
+ C: (f, a) => f(...a)
857
+ });
935
858
  }
936
- async load(ctx, id) {
937
- const blobId = PublicKey2.fromHex(id).asUint8Array();
938
- const blobMeta = await this._blobStore.getMeta(blobId);
939
- if (blobMeta && blobMeta.state === BlobMeta.State.FULLY_PRESENT) {
940
- return this._getBlob(blobId);
859
+ /**
860
+ * @param timeframe Timeframe of already processed messages.
861
+ * The pipeline will start processing messages AFTER this timeframe.
862
+ */
863
+ async setCursor(timeframe) {
864
+ invariant6(!this._isStarted || this._isPaused, "Invalid state.");
865
+ this._state._startTimeframe = timeframe;
866
+ this._timeframeClock.setTimeframe(timeframe);
867
+ for (const feed of this._feeds.values()) {
868
+ this._setFeedDownloadState(feed);
941
869
  }
942
- const fallbackStore = await cancelWithContext(ctx, this._snapshotStore.loadSnapshot(id));
943
- if (fallbackStore) {
944
- return fallbackStore;
870
+ if (this._feedSetIterator) {
871
+ await this._feedSetIterator.close();
872
+ await this._initIterator();
873
+ await this._feedSetIterator.open();
945
874
  }
946
- await this._blobSync.download(ctx, blobId);
947
- return this._getBlob(blobId);
948
- }
949
- async store(snapshot) {
950
- const { id } = await this._blobStore.set(SpaceSnapshot.encode(snapshot));
951
- await this._blobSync.notifyBlobAdded(id);
952
- return PublicKey2.from(id).toHex();
953
- }
954
- };
955
- _ts_decorate4([
956
- timed2(1e4)
957
- ], SnapshotManager.prototype, "load", null);
958
- SnapshotManager = _ts_decorate4([
959
- trackLeaks("open", "close")
960
- ], SnapshotManager);
961
-
962
- // packages/core/echo/echo-pipeline/src/dbhost/snapshot-store.ts
963
- import { subtleCrypto } from "@dxos/crypto";
964
- import { schema as schema5 } from "@dxos/protocols";
965
- var SpaceSnapshot2 = schema5.getCodecForType("dxos.echo.snapshot.SpaceSnapshot");
966
- var SnapshotStore = class {
967
- // prettier-ignore
968
- constructor(_directory) {
969
- this._directory = _directory;
970
875
  }
971
- async saveSnapshot(snapshot) {
972
- const encoded = SpaceSnapshot2.encode(snapshot);
973
- const key = await subtleCrypto.digest("SHA-256", encoded);
974
- const keyString = Buffer.from(key).toString("hex");
975
- const file = await this._directory.getOrCreateFile(keyString);
976
- try {
977
- await file.write(0, Buffer.from(encoded));
978
- } finally {
979
- await file.close();
876
+ /**
877
+ * Calling pause while processing will cause a deadlock.
878
+ */
879
+ async pause() {
880
+ invariant6(this._isStarted, "Pipeline is not open.");
881
+ if (this._isPaused) {
882
+ return;
980
883
  }
981
- return keyString;
884
+ this._pauseTrigger.reset();
885
+ await this._processingTrigger.wait();
886
+ this._isPaused = true;
982
887
  }
983
- async loadSnapshot(key) {
984
- const file = await this._directory.getOrCreateFile(key);
985
- try {
986
- const { size } = await file.stat();
987
- if (size === 0) {
988
- return void 0;
888
+ async unpause() {
889
+ invariant6(this._isStarted, "Pipeline is not open.");
890
+ invariant6(this._isPaused, "Pipeline is not paused.");
891
+ this._pauseTrigger.wake();
892
+ this._isPaused = false;
893
+ }
894
+ /**
895
+ * Starts to iterate over the ordered messages from the added feeds.
896
+ * Updates the timeframe clock after the message has bee processed.
897
+ */
898
+ async *consume() {
899
+ invariant6(!this._isOpen, "Pipeline is already being consumed.");
900
+ this._isOpen = true;
901
+ invariant6(this._feedSetIterator, "Iterator not initialized.");
902
+ let lastFeedSetIterator = this._feedSetIterator;
903
+ let iterable = lastFeedSetIterator[Symbol.asyncIterator]();
904
+ while (!this._isStopping) {
905
+ await this._pauseTrigger.wait();
906
+ if (lastFeedSetIterator !== this._feedSetIterator) {
907
+ invariant6(this._feedSetIterator, "Iterator not initialized.");
908
+ lastFeedSetIterator = this._feedSetIterator;
909
+ iterable = lastFeedSetIterator[Symbol.asyncIterator]();
910
+ }
911
+ const { done, value } = await iterable.next();
912
+ if (!done) {
913
+ const block = value != null ? value : failUndefined();
914
+ this._processingTrigger.reset();
915
+ this._timeframeClock.updatePendingTimeframe(PublicKey3.from(block.feedKey), block.seq);
916
+ yield block;
917
+ this._processingTrigger.wake();
918
+ this._timeframeClock.updateTimeframe();
989
919
  }
990
- const buffer = await file.read(0, size);
991
- return SpaceSnapshot2.decode(buffer);
992
- } finally {
993
- await file.close();
994
920
  }
921
+ this._isOpen = false;
995
922
  }
996
- async listSnapshots() {
997
- const entries = await this._directory.list();
998
- return await Promise.all(entries.map(async (key) => {
999
- const { size } = await this._directory.getOrCreateFile(key).stat();
1000
- return {
1001
- key,
1002
- size
1003
- };
1004
- }));
923
+ _setFeedDownloadState(feed) {
924
+ var _a;
925
+ const timeframe = this._state._startTimeframe;
926
+ const seq = (_a = timeframe.get(feed.key)) != null ? _a : 0;
927
+ feed.undownload({
928
+ callback: () => log6("undownload", void 0, {
929
+ F: __dxlog_file6,
930
+ L: 376,
931
+ S: this,
932
+ C: (f, a) => f(...a)
933
+ })
934
+ });
935
+ feed.download({
936
+ start: seq + 1,
937
+ linear: true
938
+ }).catch((err) => {
939
+ log6("failed to download feed", {
940
+ err
941
+ }, {
942
+ F: __dxlog_file6,
943
+ L: 378,
944
+ S: this,
945
+ C: (f, a) => f(...a)
946
+ });
947
+ });
948
+ }
949
+ async _initIterator() {
950
+ this._feedSetIterator = new FeedSetIterator(createMessageSelector(this._timeframeClock), {
951
+ start: startAfter(this._timeframeClock.timeframe),
952
+ stallTimeout: 1e3
953
+ });
954
+ this._feedSetIterator.stalled.on((iterator) => {
955
+ log6.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`, void 0, {
956
+ F: __dxlog_file6,
957
+ L: 389,
958
+ S: this,
959
+ C: (f, a) => f(...a)
960
+ });
961
+ this._state.stalled.emit();
962
+ });
963
+ for (const feed of this._feeds.values()) {
964
+ await this._feedSetIterator.addFeed(feed);
965
+ }
1005
966
  }
1006
967
  };
968
+ _ts_decorate4([
969
+ synchronized2
970
+ ], Pipeline.prototype, "start", null);
971
+ _ts_decorate4([
972
+ synchronized2
973
+ ], Pipeline.prototype, "stop", null);
974
+ _ts_decorate4([
975
+ synchronized2
976
+ ], Pipeline.prototype, "setCursor", null);
977
+ _ts_decorate4([
978
+ synchronized2
979
+ ], Pipeline.prototype, "pause", null);
980
+ _ts_decorate4([
981
+ synchronized2
982
+ ], Pipeline.prototype, "unpause", null);
1007
983
 
1008
- // packages/core/echo/echo-pipeline/src/dbhost/data-service.ts
984
+ // packages/core/echo/echo-pipeline/src/space/auth.ts
1009
985
  import invariant7 from "tiny-invariant";
1010
- import { raise } from "@dxos/debug";
1011
- import { PublicKey as PublicKey3 } from "@dxos/keys";
986
+ import { runInContext, scheduleTask } from "@dxos/async";
987
+ import { Context as Context3 } from "@dxos/context";
988
+ import { randomBytes } from "@dxos/crypto";
1012
989
  import { log as log7 } from "@dxos/log";
1013
- import { ComplexMap as ComplexMap3 } from "@dxos/util";
1014
- var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/dbhost/data-service.ts";
1015
- var DataServiceSubscriptions = class {
1016
- constructor() {
1017
- this._spaces = new ComplexMap3(PublicKey3.hash);
1018
- }
1019
- clear() {
1020
- this._spaces.clear();
1021
- }
1022
- async registerSpace(spaceKey, host) {
1023
- log7("Registering space", {
1024
- spaceKey
1025
- }, {
1026
- F: __dxlog_file7,
1027
- L: 31,
1028
- S: this,
1029
- C: (f, a) => f(...a)
990
+ import { schema as schema5 } from "@dxos/protocols";
991
+ import { RpcExtension } from "@dxos/teleport";
992
+ var __dxlog_file7 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/auth.ts";
993
+ var AuthExtension = class extends RpcExtension {
994
+ constructor(_authParams) {
995
+ super({
996
+ requested: {
997
+ AuthService: schema5.getService("dxos.mesh.teleport.auth.AuthService")
998
+ },
999
+ exposed: {
1000
+ AuthService: schema5.getService("dxos.mesh.teleport.auth.AuthService")
1001
+ },
1002
+ timeout: 60 * 1e3
1030
1003
  });
1031
- invariant7(!this._spaces.has(spaceKey));
1032
- await host.open();
1033
- this._spaces.set(spaceKey, host);
1034
- }
1035
- async unregisterSpace(spaceKey) {
1036
- log7("Unregistering space", {
1037
- spaceKey
1038
- }, {
1039
- F: __dxlog_file7,
1040
- L: 38,
1041
- S: this,
1042
- C: (f, a) => f(...a)
1004
+ this._authParams = _authParams;
1005
+ this._ctx = new Context3({
1006
+ onError: (err) => {
1007
+ log7.catch(err, void 0, {
1008
+ F: __dxlog_file7,
1009
+ L: 29,
1010
+ S: this,
1011
+ C: (f, a) => f(...a)
1012
+ });
1013
+ }
1043
1014
  });
1044
- const host = this._spaces.get(spaceKey);
1045
- await (host == null ? void 0 : host.close());
1046
- this._spaces.delete(spaceKey);
1047
- }
1048
- getDataService(spaceKey) {
1049
- return this._spaces.get(spaceKey);
1050
1015
  }
1051
- };
1052
- var DataServiceImpl = class {
1053
- constructor(_subscriptions) {
1054
- this._subscriptions = _subscriptions;
1016
+ async getHandlers() {
1017
+ return {
1018
+ AuthService: {
1019
+ authenticate: async ({ challenge }) => {
1020
+ try {
1021
+ const credential = await this._authParams.provider(challenge);
1022
+ if (!credential) {
1023
+ throw new Error("auth rejected");
1024
+ }
1025
+ return {
1026
+ credential
1027
+ };
1028
+ } catch (err) {
1029
+ log7.error("failed to generate auth credentials", err, {
1030
+ F: __dxlog_file7,
1031
+ L: 56,
1032
+ S: this,
1033
+ C: (f, a) => f(...a)
1034
+ });
1035
+ throw new Error("auth rejected");
1036
+ }
1037
+ }
1038
+ }
1039
+ };
1055
1040
  }
1056
- subscribe(request) {
1057
- var _a;
1058
- invariant7(request.spaceKey);
1059
- const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
1060
- return host.subscribe();
1041
+ async onOpen(context) {
1042
+ await super.onOpen(context);
1043
+ scheduleTask(this._ctx, async () => {
1044
+ try {
1045
+ const challenge = randomBytes(32);
1046
+ const { credential } = await this.rpc.AuthService.authenticate({
1047
+ challenge
1048
+ });
1049
+ invariant7((credential == null ? void 0 : credential.length) > 0, "invalid credential");
1050
+ const success = await this._authParams.verifier(challenge, credential);
1051
+ invariant7(success, "credential not verified");
1052
+ runInContext(this._ctx, () => this._authParams.onAuthSuccess());
1053
+ } catch (err) {
1054
+ log7("auth failed", err, {
1055
+ F: __dxlog_file7,
1056
+ L: 75,
1057
+ S: this,
1058
+ C: (f, a) => f(...a)
1059
+ });
1060
+ this.close();
1061
+ this._authParams.onAuthFailure();
1062
+ }
1063
+ });
1061
1064
  }
1062
- write(request) {
1063
- var _a;
1064
- invariant7(request.spaceKey);
1065
- invariant7(request.batch);
1066
- const host = (_a = this._subscriptions.getDataService(request.spaceKey)) != null ? _a : raise(new Error(`space not found: ${request.spaceKey}`));
1067
- return host.write(request);
1065
+ async onClose() {
1066
+ await this._ctx.dispose();
1067
+ await super.onClose();
1068
1068
  }
1069
1069
  };
1070
1070
 
@@ -1077,6 +1077,7 @@ import { getStateMachineFromItem, ItemManager } from "@dxos/echo-db";
1077
1077
  import { CancelledError } from "@dxos/errors";
1078
1078
  import { log as log8 } from "@dxos/log";
1079
1079
  import { Timeframe as Timeframe3 } from "@dxos/timeframe";
1080
+ import { tracer } from "@dxos/util";
1080
1081
  function _ts_decorate5(decorators, target, key, desc) {
1081
1082
  var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
1082
1083
  if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1087,7 +1088,7 @@ function _ts_decorate5(decorators, target, key, desc) {
1087
1088
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1088
1089
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1089
1090
  }
1090
- var __dxlog_file8 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
1091
+ var __dxlog_file8 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/data-pipeline.ts";
1091
1092
  var MESSAGES_PER_SNAPSHOT = 10;
1092
1093
  var AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5e3;
1093
1094
  var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
@@ -1101,9 +1102,9 @@ var DataPipeline = class DataPipeline2 {
1101
1102
  this._isOpen = false;
1102
1103
  this._lastTimeframeSaveTime = 0;
1103
1104
  this._lastSnapshotSaveTime = 0;
1105
+ this._lastProcessedEpoch = -1;
1104
1106
  this.currentEpoch = void 0;
1105
1107
  this.appliedEpoch = void 0;
1106
- this._lastProcessedEpoch = -1;
1107
1108
  this.onNewEpoch = new Event4();
1108
1109
  }
1109
1110
  get isOpen() {
@@ -1218,7 +1219,7 @@ var DataPipeline = class DataPipeline2 {
1218
1219
  if (data.payload.data) {
1219
1220
  const feedInfo = this._params.feedInfoProvider(feedKey);
1220
1221
  if (!feedInfo) {
1221
- log8.error("Could not find feed.", {
1222
+ log8.warn("Could not find feed", {
1222
1223
  feedKey
1223
1224
  }, {
1224
1225
  F: __dxlog_file8,
@@ -1228,7 +1229,8 @@ var DataPipeline = class DataPipeline2 {
1228
1229
  });
1229
1230
  continue;
1230
1231
  }
1231
- await this.databaseHost.echoProcessor({
1232
+ const timer = tracer.mark("dxos.echo.pipeline.data");
1233
+ this.databaseHost.echoProcessor({
1232
1234
  batch: data.payload.data.batch,
1233
1235
  meta: {
1234
1236
  feedKey,
@@ -1237,13 +1239,14 @@ var DataPipeline = class DataPipeline2 {
1237
1239
  memberKey: feedInfo.assertion.identityKey
1238
1240
  }
1239
1241
  });
1242
+ timer.end();
1240
1243
  log8.trace("dxos.echo.data-pipeline.processed", {
1241
1244
  feedKey: feedKey.toHex(),
1242
1245
  seq,
1243
1246
  spaceKey: this._params.spaceKey.toHex()
1244
1247
  }, {
1245
1248
  F: __dxlog_file8,
1246
- L: 230,
1249
+ L: 233,
1247
1250
  S: this,
1248
1251
  C: (f, a) => f(...a)
1249
1252
  });
@@ -1252,7 +1255,7 @@ var DataPipeline = class DataPipeline2 {
1252
1255
  } catch (err) {
1253
1256
  log8.catch(err, void 0, {
1254
1257
  F: __dxlog_file8,
1255
- L: 240,
1258
+ L: 243,
1256
1259
  S: this,
1257
1260
  C: (f, a) => f(...a)
1258
1261
  });
@@ -1287,7 +1290,7 @@ var DataPipeline = class DataPipeline2 {
1287
1290
  } catch (err) {
1288
1291
  log8.warn("Failed to cache properties", err, {
1289
1292
  F: __dxlog_file8,
1290
- L: 275,
1293
+ L: 278,
1291
1294
  S: this,
1292
1295
  C: (f, a) => f(...a)
1293
1296
  });
@@ -1314,14 +1317,14 @@ var DataPipeline = class DataPipeline2 {
1314
1317
  if (err instanceof CancelledError) {
1315
1318
  log8("Epoch processing cancelled.", void 0, {
1316
1319
  F: __dxlog_file8,
1317
- L: 307,
1320
+ L: 310,
1318
1321
  S: this,
1319
1322
  C: (f, a) => f(...a)
1320
1323
  });
1321
1324
  } else {
1322
1325
  log8.catch(err, void 0, {
1323
1326
  F: __dxlog_file8,
1324
- L: 309,
1327
+ L: 312,
1325
1328
  S: this,
1326
1329
  C: (f, a) => f(...a)
1327
1330
  });
@@ -1337,7 +1340,7 @@ var DataPipeline = class DataPipeline2 {
1337
1340
  epoch
1338
1341
  }, {
1339
1342
  F: __dxlog_file8,
1340
- L: 319,
1343
+ L: 322,
1341
1344
  S: this,
1342
1345
  C: (f, a) => f(...a)
1343
1346
  });
@@ -1354,7 +1357,7 @@ var DataPipeline = class DataPipeline2 {
1354
1357
  epoch
1355
1358
  }, {
1356
1359
  F: __dxlog_file8,
1357
- L: 333,
1360
+ L: 336,
1358
1361
  S: this,
1359
1362
  C: (f, a) => f(...a)
1360
1363
  });
@@ -1364,7 +1367,7 @@ var DataPipeline = class DataPipeline2 {
1364
1367
  }
1365
1368
  log8("restarting pipeline for epoch", void 0, {
1366
1369
  F: __dxlog_file8,
1367
- L: 341,
1370
+ L: 344,
1368
1371
  S: this,
1369
1372
  C: (f, a) => f(...a)
1370
1373
  });
@@ -1424,8 +1427,8 @@ import { PublicKey as PublicKey4 } from "@dxos/keys";
1424
1427
  import { log as log9 } from "@dxos/log";
1425
1428
  import { AdmittedFeed } from "@dxos/protocols/proto/dxos/halo/credentials";
1426
1429
  import { Timeframe as Timeframe4 } from "@dxos/timeframe";
1427
- import { Callback } from "@dxos/util";
1428
- var __dxlog_file9 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
1430
+ import { Callback, tracer as tracer2 } from "@dxos/util";
1431
+ var __dxlog_file9 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/control-pipeline.ts";
1429
1432
  var TIMEFRAME_SAVE_DEBOUNCE_INTERVAL2 = 500;
1430
1433
  var ControlPipeline = class {
1431
1434
  constructor({ spaceKey, genesisFeed, feedProvider, metadataStore }) {
@@ -1492,13 +1495,15 @@ var ControlPipeline = class {
1492
1495
  C: (f, a) => f(...a)
1493
1496
  });
1494
1497
  if (msg.data.payload.credential) {
1498
+ const timer = tracer2.mark("dxos.echo.pipeline.control");
1495
1499
  const result = await this._spaceStateMachine.process(msg.data.payload.credential.credential, PublicKey4.from(msg.feedKey));
1500
+ timer.end();
1496
1501
  if (!result) {
1497
1502
  log9.warn("processing failed", {
1498
1503
  msg
1499
1504
  }, {
1500
1505
  F: __dxlog_file9,
1501
- L: 95,
1506
+ L: 98,
1502
1507
  S: this,
1503
1508
  C: (f, a) => f(...a)
1504
1509
  });
@@ -1509,7 +1514,7 @@ var ControlPipeline = class {
1509
1514
  } catch (err) {
1510
1515
  log9.catch(err, void 0, {
1511
1516
  F: __dxlog_file9,
1512
- L: 101,
1517
+ L: 104,
1513
1518
  S: this,
1514
1519
  C: (f, a) => f(...a)
1515
1520
  });
@@ -1519,7 +1524,7 @@ var ControlPipeline = class {
1519
1524
  await this._pipeline.start();
1520
1525
  log9("started", void 0, {
1521
1526
  F: __dxlog_file9,
1522
- L: 107,
1527
+ L: 110,
1523
1528
  S: this,
1524
1529
  C: (f, a) => f(...a)
1525
1530
  });
@@ -1533,7 +1538,7 @@ var ControlPipeline = class {
1533
1538
  async stop() {
1534
1539
  log9("stopping...", void 0, {
1535
1540
  F: __dxlog_file9,
1536
- L: 121,
1541
+ L: 124,
1537
1542
  S: this,
1538
1543
  C: (f, a) => f(...a)
1539
1544
  });
@@ -1541,7 +1546,7 @@ var ControlPipeline = class {
1541
1546
  await this._saveTargetTimeframe(this._pipeline.state.timeframe);
1542
1547
  log9("stopped", void 0, {
1543
1548
  F: __dxlog_file9,
1544
- L: 124,
1549
+ L: 127,
1545
1550
  S: this,
1546
1551
  C: (f, a) => f(...a)
1547
1552
  });
@@ -1555,7 +1560,7 @@ var ControlPipeline = class {
1555
1560
  } catch (err) {
1556
1561
  log9(err, void 0, {
1557
1562
  F: __dxlog_file9,
1558
- L: 133,
1563
+ L: 136,
1559
1564
  S: this,
1560
1565
  C: (f, a) => f(...a)
1561
1566
  });
@@ -1574,7 +1579,7 @@ function _ts_decorate6(decorators, target, key, desc) {
1574
1579
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1575
1580
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1576
1581
  }
1577
- var __dxlog_file10 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space.ts";
1582
+ var __dxlog_file10 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space.ts";
1578
1583
  var Space = class Space2 {
1579
1584
  constructor(params) {
1580
1585
  this._addFeedLock = new Lock();
@@ -1797,7 +1802,7 @@ function _ts_decorate7(decorators, target, key, desc) {
1797
1802
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1798
1803
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1799
1804
  }
1800
- var __dxlog_file11 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
1805
+ var __dxlog_file11 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-protocol.ts";
1801
1806
  var MOCK_AUTH_PROVIDER = async (nonce) => Buffer.from("mock");
1802
1807
  var MOCK_AUTH_VERIFIER = async (nonce, credential) => true;
1803
1808
  var SpaceProtocol = class {
@@ -1828,7 +1833,7 @@ var SpaceProtocol = class {
1828
1833
  key: feed.key
1829
1834
  }, {
1830
1835
  F: __dxlog_file11,
1831
- L: 96,
1836
+ L: 95,
1832
1837
  S: this,
1833
1838
  C: (f, a) => f(...a)
1834
1839
  });
@@ -1850,7 +1855,7 @@ var SpaceProtocol = class {
1850
1855
  await this.blobSync.open();
1851
1856
  log11("starting...", void 0, {
1852
1857
  F: __dxlog_file11,
1853
- L: 121,
1858
+ L: 120,
1854
1859
  S: this,
1855
1860
  C: (f, a) => f(...a)
1856
1861
  });
@@ -1863,7 +1868,7 @@ var SpaceProtocol = class {
1863
1868
  });
1864
1869
  log11("started", void 0, {
1865
1870
  F: __dxlog_file11,
1866
- L: 130,
1871
+ L: 129,
1867
1872
  S: this,
1868
1873
  C: (f, a) => f(...a)
1869
1874
  });
@@ -1873,14 +1878,14 @@ var SpaceProtocol = class {
1873
1878
  if (this._connection) {
1874
1879
  log11("stopping...", void 0, {
1875
1880
  F: __dxlog_file11,
1876
- L: 137,
1881
+ L: 136,
1877
1882
  S: this,
1878
1883
  C: (f, a) => f(...a)
1879
1884
  });
1880
1885
  await this._connection.close();
1881
1886
  log11("stopped", void 0, {
1882
1887
  F: __dxlog_file11,
1883
- L: 139,
1888
+ L: 138,
1884
1889
  S: this,
1885
1890
  C: (f, a) => f(...a)
1886
1891
  });
@@ -1948,7 +1953,7 @@ var SpaceProtocolSession = class {
1948
1953
  var _a;
1949
1954
  log11("Peer authenticated", void 0, {
1950
1955
  F: __dxlog_file11,
1951
- L: 236,
1956
+ L: 235,
1952
1957
  S: this,
1953
1958
  C: (f, a) => f(...a)
1954
1959
  });
@@ -1992,7 +1997,7 @@ function _ts_decorate8(decorators, target, key, desc) {
1992
1997
  r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
1993
1998
  return c > 3 && r && Object.defineProperty(target, key, r), r;
1994
1999
  }
1995
- var __dxlog_file12 = "/home/runner/work/dxos/dxos/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
2000
+ var __dxlog_file12 = "/mnt/ramdisk/work/packages/core/echo/echo-pipeline/src/space/space-manager.ts";
1996
2001
  var SpaceManager = class SpaceManager2 {
1997
2002
  constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore, blobStore }) {
1998
2003
  this._spaces = new ComplexMap5(PublicKey6.hash);
@@ -2077,22 +2082,22 @@ SpaceManager = _ts_decorate8([
2077
2082
  ], SpaceManager);
2078
2083
 
2079
2084
  export {
2080
- MetadataStore,
2081
2085
  codec,
2082
2086
  valueEncoding,
2083
2087
  createMappedFeedWriter,
2084
- mapTimeframeToFeedIndexes,
2085
- mapFeedIndexesToTimeframe,
2086
- startAfter,
2087
- TimeframeClock,
2088
- Pipeline,
2089
- AuthExtension,
2090
2088
  DataServiceHost,
2091
2089
  DatabaseHost,
2092
2090
  SnapshotManager,
2093
2091
  SnapshotStore,
2094
2092
  DataServiceSubscriptions,
2095
2093
  DataServiceImpl,
2094
+ MetadataStore,
2095
+ mapTimeframeToFeedIndexes,
2096
+ mapFeedIndexesToTimeframe,
2097
+ startAfter,
2098
+ TimeframeClock,
2099
+ Pipeline,
2100
+ AuthExtension,
2096
2101
  DataPipeline,
2097
2102
  Space,
2098
2103
  MOCK_AUTH_PROVIDER,
@@ -2102,4 +2107,4 @@ export {
2102
2107
  SpaceProtocolSession,
2103
2108
  SpaceManager
2104
2109
  };
2105
- //# sourceMappingURL=chunk-OJGL5427.mjs.map
2110
+ //# sourceMappingURL=chunk-4VJRJ4SY.mjs.map