@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc674ce

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/dist/lib/browser/{chunk-WJJXJTNS.mjs → chunk-FJPXA75J.mjs} +7 -12
  2. package/dist/lib/browser/chunk-FJPXA75J.mjs.map +7 -0
  3. package/dist/lib/browser/{chunk-FR6GJ4S4.mjs → chunk-WIQUYO7H.mjs} +39 -29
  4. package/dist/lib/browser/chunk-WIQUYO7H.mjs.map +7 -0
  5. package/dist/lib/browser/filter/index.mjs +1 -2
  6. package/dist/lib/browser/index.mjs +1933 -620
  7. package/dist/lib/browser/index.mjs.map +4 -4
  8. package/dist/lib/browser/meta.json +1 -1
  9. package/dist/lib/browser/testing/index.mjs +28 -201
  10. package/dist/lib/browser/testing/index.mjs.map +4 -4
  11. package/dist/lib/node-esm/{chunk-JQCCVDLT.mjs → chunk-7PL6UK4B.mjs} +39 -29
  12. package/dist/lib/node-esm/chunk-7PL6UK4B.mjs.map +7 -0
  13. package/dist/lib/node-esm/{chunk-LEQ77KAT.mjs → chunk-S5KGVQFZ.mjs} +7 -12
  14. package/dist/lib/node-esm/chunk-S5KGVQFZ.mjs.map +7 -0
  15. package/dist/lib/node-esm/filter/index.mjs +1 -2
  16. package/dist/lib/node-esm/index.mjs +1933 -620
  17. package/dist/lib/node-esm/index.mjs.map +4 -4
  18. package/dist/lib/node-esm/meta.json +1 -1
  19. package/dist/lib/node-esm/testing/index.mjs +28 -201
  20. package/dist/lib/node-esm/testing/index.mjs.map +4 -4
  21. package/dist/types/src/automerge/automerge-host.d.ts +36 -9
  22. package/dist/types/src/automerge/automerge-host.d.ts.map +1 -1
  23. package/dist/types/src/automerge/automerge-repo.test.d.ts +1 -1
  24. package/dist/types/src/automerge/automerge-repo.test.d.ts.map +1 -1
  25. package/dist/types/src/automerge/collection-synchronizer.d.ts +3 -2
  26. package/dist/types/src/automerge/collection-synchronizer.d.ts.map +1 -1
  27. package/dist/types/src/automerge/echo-network-adapter.d.ts +11 -6
  28. package/dist/types/src/automerge/echo-network-adapter.d.ts.map +1 -1
  29. package/dist/types/src/automerge/echo-replicator.d.ts +9 -6
  30. package/dist/types/src/automerge/echo-replicator.d.ts.map +1 -1
  31. package/dist/types/src/automerge/heads-store.d.ts +9 -2
  32. package/dist/types/src/automerge/heads-store.d.ts.map +1 -1
  33. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts +4 -4
  34. package/dist/types/src/automerge/leveldb-storage-adapter.d.ts.map +1 -1
  35. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts +7 -7
  36. package/dist/types/src/automerge/mesh-echo-replicator-connection.d.ts.map +1 -1
  37. package/dist/types/src/automerge/mesh-echo-replicator.d.ts.map +1 -1
  38. package/dist/types/src/db-host/automerge-data-source.d.ts +34 -0
  39. package/dist/types/src/db-host/automerge-data-source.d.ts.map +1 -0
  40. package/dist/types/src/db-host/automerge-data-source.test.d.ts +2 -0
  41. package/dist/types/src/db-host/automerge-data-source.test.d.ts.map +1 -0
  42. package/dist/types/src/db-host/data-service.d.ts +4 -3
  43. package/dist/types/src/db-host/data-service.d.ts.map +1 -1
  44. package/dist/types/src/db-host/documents-iterator.d.ts.map +1 -1
  45. package/dist/types/src/db-host/documents-synchronizer.d.ts +6 -5
  46. package/dist/types/src/db-host/documents-synchronizer.d.ts.map +1 -1
  47. package/dist/types/src/db-host/echo-host.d.ts +43 -11
  48. package/dist/types/src/db-host/echo-host.d.ts.map +1 -1
  49. package/dist/types/src/db-host/index.d.ts +1 -0
  50. package/dist/types/src/db-host/index.d.ts.map +1 -1
  51. package/dist/types/src/db-host/local-queue-service.d.ts +15 -0
  52. package/dist/types/src/db-host/local-queue-service.d.ts.map +1 -0
  53. package/dist/types/src/db-host/query-service.d.ts +7 -2
  54. package/dist/types/src/db-host/query-service.d.ts.map +1 -1
  55. package/dist/types/src/db-host/queue-data-source.d.ts +28 -0
  56. package/dist/types/src/db-host/queue-data-source.d.ts.map +1 -0
  57. package/dist/types/src/db-host/queue-service.test.d.ts +2 -0
  58. package/dist/types/src/db-host/queue-service.test.d.ts.map +1 -0
  59. package/dist/types/src/db-host/space-state-manager.d.ts +1 -0
  60. package/dist/types/src/db-host/space-state-manager.d.ts.map +1 -1
  61. package/dist/types/src/db-host/stub.d.ts +10 -0
  62. package/dist/types/src/db-host/stub.d.ts.map +1 -0
  63. package/dist/types/src/edge/echo-edge-replicator.d.ts +2 -2
  64. package/dist/types/src/edge/echo-edge-replicator.d.ts.map +1 -1
  65. package/dist/types/src/filter/filter-match.d.ts.map +1 -1
  66. package/dist/types/src/metadata/metadata-store.d.ts.map +1 -1
  67. package/dist/types/src/pipeline/pipeline.d.ts +2 -2
  68. package/dist/types/src/pipeline/pipeline.d.ts.map +1 -1
  69. package/dist/types/src/query/errors.d.ts +11 -15
  70. package/dist/types/src/query/errors.d.ts.map +1 -1
  71. package/dist/types/src/query/plan.d.ts +37 -3
  72. package/dist/types/src/query/plan.d.ts.map +1 -1
  73. package/dist/types/src/query/query-executor.d.ts +39 -3
  74. package/dist/types/src/query/query-executor.d.ts.map +1 -1
  75. package/dist/types/src/query/query-planner.d.ts +6 -0
  76. package/dist/types/src/query/query-planner.d.ts.map +1 -1
  77. package/dist/types/src/space/auth.d.ts +3 -3
  78. package/dist/types/src/space/auth.d.ts.map +1 -1
  79. package/dist/types/src/space/control-pipeline.d.ts +2 -2
  80. package/dist/types/src/space/control-pipeline.d.ts.map +1 -1
  81. package/dist/types/src/space/space-manager.d.ts +6 -6
  82. package/dist/types/src/space/space-manager.d.ts.map +1 -1
  83. package/dist/types/src/space/space-protocol.d.ts +5 -5
  84. package/dist/types/src/space/space-protocol.d.ts.map +1 -1
  85. package/dist/types/src/space/space.d.ts +3 -3
  86. package/dist/types/src/space/space.d.ts.map +1 -1
  87. package/dist/types/src/testing/index.d.ts +0 -2
  88. package/dist/types/src/testing/index.d.ts.map +1 -1
  89. package/dist/types/src/testing/test-network-adapter.d.ts +1 -1
  90. package/dist/types/src/testing/test-network-adapter.d.ts.map +1 -1
  91. package/dist/types/src/testing/test-replicator.d.ts +6 -6
  92. package/dist/types/src/testing/test-replicator.d.ts.map +1 -1
  93. package/dist/types/src/util.d.ts.map +1 -1
  94. package/dist/types/tsconfig.tsbuildinfo +1 -1
  95. package/package.json +48 -38
  96. package/src/automerge/automerge-host.test.ts +37 -14
  97. package/src/automerge/automerge-host.ts +130 -36
  98. package/src/automerge/automerge-repo.test.ts +147 -30
  99. package/src/automerge/collection-synchronizer.test.ts +16 -10
  100. package/src/automerge/collection-synchronizer.ts +41 -11
  101. package/src/automerge/echo-network-adapter.ts +23 -11
  102. package/src/automerge/echo-replicator.ts +10 -6
  103. package/src/automerge/heads-store.ts +14 -2
  104. package/src/automerge/leveldb-storage-adapter.ts +4 -4
  105. package/src/automerge/mesh-echo-replicator-connection.ts +7 -7
  106. package/src/automerge/mesh-echo-replicator.ts +2 -2
  107. package/src/db-host/automerge-data-source.test.ts +294 -0
  108. package/src/db-host/automerge-data-source.ts +144 -0
  109. package/src/db-host/data-service.ts +10 -3
  110. package/src/db-host/documents-iterator.ts +3 -1
  111. package/src/db-host/documents-synchronizer.test.ts +77 -13
  112. package/src/db-host/documents-synchronizer.ts +51 -50
  113. package/src/db-host/echo-host.ts +191 -28
  114. package/src/db-host/index.ts +1 -0
  115. package/src/db-host/local-queue-service.ts +127 -0
  116. package/src/db-host/query-service.ts +17 -4
  117. package/src/db-host/queue-data-source.ts +132 -0
  118. package/src/db-host/queue-service.test.ts +141 -0
  119. package/src/db-host/space-state-manager.ts +4 -0
  120. package/src/db-host/stub.ts +28 -0
  121. package/src/edge/echo-edge-replicator.ts +12 -8
  122. package/src/filter/filter-match.test.ts +31 -25
  123. package/src/filter/filter-match.ts +8 -14
  124. package/src/metadata/metadata-store.ts +5 -2
  125. package/src/pipeline/pipeline.ts +2 -2
  126. package/src/query/errors.ts +2 -2
  127. package/src/query/plan.ts +48 -3
  128. package/src/query/query-executor.ts +612 -131
  129. package/src/query/query-planner.test.ts +509 -35
  130. package/src/query/query-planner.ts +187 -13
  131. package/src/space/admission-discovery-extension.ts +1 -1
  132. package/src/space/auth.ts +6 -6
  133. package/src/space/control-pipeline.ts +2 -2
  134. package/src/space/space-manager.ts +6 -6
  135. package/src/space/space-protocol.ts +11 -11
  136. package/src/space/space.ts +3 -3
  137. package/src/testing/index.ts +0 -2
  138. package/src/testing/test-network-adapter.ts +16 -3
  139. package/src/testing/test-replicator.ts +8 -7
  140. package/src/util.ts +7 -3
  141. package/dist/lib/browser/chunk-CGS2ULMK.mjs +0 -11
  142. package/dist/lib/browser/chunk-CGS2ULMK.mjs.map +0 -7
  143. package/dist/lib/browser/chunk-FR6GJ4S4.mjs.map +0 -7
  144. package/dist/lib/browser/chunk-WJJXJTNS.mjs.map +0 -7
  145. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
  146. package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +0 -7
  147. package/dist/lib/node-esm/chunk-JQCCVDLT.mjs.map +0 -7
  148. package/dist/lib/node-esm/chunk-LEQ77KAT.mjs.map +0 -7
  149. package/dist/types/src/testing/test-data.d.ts +0 -18
  150. package/dist/types/src/testing/test-data.d.ts.map +0 -1
  151. package/dist/types/src/testing/test-schema.d.ts +0 -37
  152. package/dist/types/src/testing/test-schema.d.ts.map +0 -1
  153. package/src/testing/test-data.ts +0 -127
  154. package/src/testing/test-schema.ts +0 -53
@@ -4,6 +4,7 @@
4
4
 
5
5
  import { getHeads } from '@automerge/automerge';
6
6
  import { type DocHandle, type DocumentId } from '@automerge/automerge-repo';
7
+ import type * as SqlClient from '@effect/sql/SqlClient';
7
8
  import * as Schema from 'effect/Schema';
8
9
 
9
10
  import { DeferredTask, scheduleMicroTask, synchronized } from '@dxos/async';
@@ -11,6 +12,8 @@ import { Stream } from '@dxos/codec-protobuf/stream';
11
12
  import { Context, Resource } from '@dxos/context';
12
13
  import { raise } from '@dxos/debug';
13
14
  import { DatabaseDirectory, QueryAST } from '@dxos/echo-protocol';
15
+ import { type RuntimeProvider } from '@dxos/effect';
16
+ import { type IndexEngine } from '@dxos/index-core';
14
17
  import { type IdToHeads, type Indexer, type ObjectSnapshot } from '@dxos/indexing';
15
18
  import { log } from '@dxos/log';
16
19
  import { objectPointerCodec } from '@dxos/protocols';
@@ -28,8 +31,10 @@ import { QueryExecutor } from '../query';
28
31
 
29
32
  import type { SpaceStateManager } from './space-state-manager';
30
33
 
31
- export type QueryServiceParams = {
34
+ export type QueryServiceProps = {
32
35
  indexer: Indexer;
36
+ indexer2?: IndexEngine;
37
+ runtime?: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient>;
33
38
  automergeHost: AutomergeHost;
34
39
  spaceStateManager: SpaceStateManager;
35
40
  };
@@ -62,7 +67,7 @@ export class QueryServiceImpl extends Resource implements QueryService {
62
67
  private _updateQueries!: DeferredTask;
63
68
 
64
69
  // TODO(burdon): OK for options, but not params. Pass separately and type readonly here.
65
- constructor(private readonly _params: QueryServiceParams) {
70
+ constructor(private readonly _params: QueryServiceProps) {
66
71
  super();
67
72
 
68
73
  trace.diagnostic({
@@ -155,6 +160,8 @@ export class QueryServiceImpl extends Resource implements QueryService {
155
160
  const queryEntry: ActiveQuery = {
156
161
  executor: new QueryExecutor({
157
162
  indexer: this._params.indexer,
163
+ indexer2: this._params.indexer2,
164
+ runtime: this._params.runtime,
158
165
  automergeHost: this._params.automergeHost,
159
166
  queryId: request.queryId ?? raise(new Error('query id required')),
160
167
  query: parsedQuery,
@@ -245,7 +252,13 @@ const createDocumentsIterator = (automergeHost: AutomergeHost) =>
245
252
  if (visited.has(urlString)) {
246
253
  continue;
247
254
  }
248
- const linkHandle = await automergeHost.loadDoc<DatabaseDirectory>(Context.default(), urlString as DocumentId);
255
+ const linkHandle = await automergeHost.loadDoc<DatabaseDirectory>(
256
+ Context.default(),
257
+ urlString as DocumentId,
258
+ {
259
+ fetchFromNetwork: true,
260
+ },
261
+ );
249
262
  for await (const result of getObjectsFromHandle(linkHandle)) {
250
263
  yield result;
251
264
  }
@@ -256,7 +269,7 @@ const createDocumentsIterator = (automergeHost: AutomergeHost) =>
256
269
  }
257
270
 
258
271
  // TODO(mykola): Use list of roots instead of iterating over all handles.
259
- for (const handle of Object.values(automergeHost.repo.handles)) {
272
+ for (const handle of Object.values(automergeHost.handles)) {
260
273
  if (visited.has(handle.documentId)) {
261
274
  continue;
262
275
  }
@@ -0,0 +1,132 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import type * as SqlClient from '@effect/sql/SqlClient';
6
+ import * as Effect from 'effect/Effect';
7
+
8
+ import { RuntimeProvider } from '@dxos/effect';
9
+ import { FeedCursor, type FeedStore } from '@dxos/feed';
10
+ import { type DataSourceCursor, type IndexDataSource, type IndexerObject } from '@dxos/index-core';
11
+ import { failedInvariant } from '@dxos/invariant';
12
+ import type { SpaceId } from '@dxos/keys';
13
+ import { log } from '@dxos/log';
14
+
15
+ export type QueueDataSourceOptions = {
16
+ feedStore: FeedStore;
17
+ runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient>;
18
+ /**
19
+ * Callback to get the list of space IDs that should be indexed.
20
+ */
21
+ getSpaceIds: () => SpaceId[];
22
+ };
23
+
24
+ export class QueueDataSource implements IndexDataSource {
25
+ readonly sourceName = 'queue';
26
+
27
+ private readonly _feedStore: FeedStore;
28
+ private readonly _runtime: RuntimeProvider.RuntimeProvider<SqlClient.SqlClient>;
29
+ private readonly _getSpaceIds: () => SpaceId[];
30
+
31
+ constructor(options: QueueDataSourceOptions) {
32
+ this._feedStore = options.feedStore;
33
+ this._runtime = options.runtime;
34
+ this._getSpaceIds = options.getSpaceIds;
35
+ }
36
+
37
+ getChangedObjects(
38
+ cursors: DataSourceCursor[],
39
+ opts?: { limit?: number },
40
+ ): Effect.Effect<{ objects: IndexerObject[]; cursors: DataSourceCursor[] }> {
41
+ // For queue, the cursor is assumed to have:
42
+ // spaceId = set
43
+ // resourceId = null
44
+ // cursor = feed cursor
45
+
46
+ // We also add new cursors from all previously unindexed spaces.
47
+
48
+ return Effect.gen(this, function* () {
49
+ const objects: IndexerObject[] = [];
50
+ const updatedCursors: DataSourceCursor[] = [];
51
+
52
+ // Augment cursors with new spaces that haven't been indexed yet.
53
+ const existingSpaceIds = new Set(cursors.map((c) => c.spaceId).filter(Boolean));
54
+ const allSpaceIds = this._getSpaceIds();
55
+ const augmentedCursors = [...cursors];
56
+ for (const spaceId of allSpaceIds) {
57
+ if (!existingSpaceIds.has(spaceId)) {
58
+ // Add a new cursor for this space starting from the beginning.
59
+ // Empty string cursor means "start from beginning".
60
+ augmentedCursors.push({
61
+ spaceId,
62
+ resourceId: null,
63
+ cursor: '',
64
+ });
65
+ }
66
+ }
67
+
68
+ // Limit per call, but we might have multiple spaces.
69
+ // We should distribute limit or just fill up to limit.
70
+ let remainingLimit = opts?.limit ?? Infinity;
71
+
72
+ for (const cursor of augmentedCursors) {
73
+ if (remainingLimit <= 0) {
74
+ // Remaining cursors are not processed in this call.
75
+ updatedCursors.push(cursor);
76
+ continue;
77
+ }
78
+ if (!cursor.spaceId) {
79
+ // Ignore cursors without spaceId.
80
+ updatedCursors.push(cursor);
81
+ continue;
82
+ }
83
+
84
+ // Empty string cursor means "start from beginning".
85
+ const currentCursor =
86
+ typeof cursor.cursor === 'string' && cursor.cursor !== '' ? FeedCursor.make(cursor.cursor) : undefined;
87
+
88
+ try {
89
+ const result = yield* this._feedStore.query({
90
+ spaceId: cursor.spaceId,
91
+ cursor: currentCursor,
92
+ query: {
93
+ // Query all feeds in data namespace
94
+ feedNamespace: 'data',
95
+ },
96
+ limit: remainingLimit,
97
+ });
98
+
99
+ // Process blocks
100
+ for (const block of result.blocks) {
101
+ try {
102
+ const dataString = new TextDecoder().decode(block.data);
103
+ // TODO: Handle non-JSON data?
104
+ const data = JSON.parse(dataString);
105
+
106
+ objects.push({
107
+ spaceId: cursor.spaceId,
108
+ queueId: block.feedId ?? failedInvariant(),
109
+ documentId: null,
110
+ recordId: null,
111
+ data,
112
+ });
113
+ } catch (err) {
114
+ log.warn('Failed to parse block data for indexing', { spaceId: cursor.spaceId, err });
115
+ }
116
+ }
117
+
118
+ remainingLimit -= result.blocks.length;
119
+ updatedCursors.push({
120
+ spaceId: cursor.spaceId,
121
+ resourceId: null,
122
+ cursor: result.nextCursor,
123
+ });
124
+ } catch (error) {
125
+ log.error('Error querying queue for indexing', { spaceId: cursor.spaceId, error });
126
+ }
127
+ }
128
+
129
+ return { objects, cursors: updatedCursors };
130
+ }).pipe(RuntimeProvider.provide(this._runtime), Effect.withSpan('QueueDataSource.getChangedObjects'), Effect.orDie);
131
+ }
132
+ }
@@ -0,0 +1,141 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import type * as SqlClient from '@effect/sql/SqlClient';
6
+ import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
7
+ import { describe, expect, it } from '@effect/vitest';
8
+ import * as Effect from 'effect/Effect';
9
+
10
+ import { RuntimeProvider } from '@dxos/effect';
11
+ import { FeedStore } from '@dxos/feed';
12
+ import { ObjectId, SpaceId } from '@dxos/keys';
13
+
14
+ import { LocalQueueServiceImpl } from './local-queue-service';
15
+
16
+ const TestLayer = SqliteClient.layer({
17
+ filename: ':memory:',
18
+ });
19
+
20
+ describe('LocalQueueServiceImpl', () => {
21
+ it.effect('should insert and query items', () =>
22
+ Effect.gen(function* () {
23
+ const sql = yield* SqliteClient.SqliteClient;
24
+ const feedStore = new FeedStore({ localActorId: 'actor-id', assignPositions: true });
25
+ const runtime = yield* RuntimeProvider.currentRuntime<SqlClient.SqlClient>();
26
+ const service = new LocalQueueServiceImpl(runtime, feedStore);
27
+ yield* feedStore.migrate();
28
+
29
+ const spaceId = SpaceId.random();
30
+ const queueId = ObjectId.random();
31
+ const object1 = { id: 'obj1', data: 'test1' };
32
+ const object2 = { id: 'obj2', data: 'test2' };
33
+
34
+ yield* Effect.promise(() =>
35
+ service.insertIntoQueue({
36
+ subspaceTag: 'default',
37
+ spaceId,
38
+ queueId,
39
+ objects: [object1, object2],
40
+ }),
41
+ );
42
+
43
+ const result = yield* Effect.promise(() =>
44
+ service.queryQueue({
45
+ query: { spaceId, queueIds: [queueId] },
46
+ }),
47
+ );
48
+ expect(result.objects?.[0]).toMatchObject(object1);
49
+ expect(result.objects?.[1]).toMatchObject(object2);
50
+ }).pipe(Effect.provide(TestLayer)),
51
+ );
52
+
53
+ it.effect('should delete items', () =>
54
+ Effect.gen(function* () {
55
+ const sql = yield* SqliteClient.SqliteClient;
56
+ const runtime = Effect.succeed(yield* Effect.runtime<any>());
57
+ const feedStore = new FeedStore({ localActorId: 'actor-id', assignPositions: true });
58
+ const service = new LocalQueueServiceImpl(runtime, feedStore);
59
+ yield* feedStore.migrate();
60
+
61
+ const spaceId = SpaceId.random();
62
+ const queueId = ObjectId.random();
63
+ const object1Id = ObjectId.random();
64
+ const object1 = { id: object1Id, data: 'test1' };
65
+
66
+ yield* Effect.promise(() =>
67
+ service.insertIntoQueue({
68
+ subspaceTag: 'default',
69
+ spaceId,
70
+ queueId,
71
+ objects: [object1],
72
+ }),
73
+ );
74
+ yield* Effect.promise(() =>
75
+ service.deleteFromQueue({
76
+ subspaceTag: 'default',
77
+ spaceId,
78
+ queueId,
79
+ objectIds: [object1Id],
80
+ }),
81
+ );
82
+
83
+ const result = yield* Effect.promise(() =>
84
+ service.queryQueue({
85
+ query: { spaceId, queueIds: [queueId] },
86
+ }),
87
+ );
88
+ expect(result.objects).toHaveLength(2);
89
+ expect(result.objects?.[1]).toMatchObject({ id: object1Id, '@deleted': true });
90
+ }).pipe(Effect.provide(TestLayer)),
91
+ );
92
+
93
+ it.effect('should support pagination', () =>
94
+ Effect.gen(function* () {
95
+ const sql = yield* SqliteClient.SqliteClient;
96
+ const runtime = Effect.succeed(yield* Effect.runtime<any>());
97
+ const feedStore = new FeedStore({ localActorId: 'actor-id', assignPositions: true });
98
+ yield* feedStore.migrate();
99
+ const service = new LocalQueueServiceImpl(runtime, feedStore);
100
+ const spaceId = 'space-1' as SpaceId;
101
+ const queueId = ObjectId.random();
102
+
103
+ // Insert 10 items
104
+ const items = Array.from({ length: 10 }, (_, i) => ({ id: `obj${i}`, data: `test${i}` }));
105
+ yield* Effect.promise(() =>
106
+ service.insertIntoQueue({
107
+ subspaceTag: 'default',
108
+ spaceId,
109
+ queueId,
110
+ objects: items,
111
+ }),
112
+ );
113
+
114
+ // Query first 5
115
+ const page1 = yield* Effect.promise(() =>
116
+ service.queryQueue({
117
+ query: { spaceId, queueIds: [queueId], limit: 5 },
118
+ }),
119
+ );
120
+ expect(page1.objects).toHaveLength(5);
121
+ expect(page1.objects?.[0]).toMatchObject(items[0]);
122
+ expect(page1.objects?.[4]).toMatchObject(items[4]);
123
+ expect(page1.nextCursor).toBeDefined();
124
+
125
+ // Query next 5
126
+ const page2 = yield* Effect.promise(() =>
127
+ service.queryQueue({
128
+ query: {
129
+ spaceId,
130
+ queueIds: [queueId],
131
+ limit: 5,
132
+ after: page1.nextCursor!,
133
+ },
134
+ }),
135
+ );
136
+ expect(page2.objects).toHaveLength(5);
137
+ expect(page2.objects?.[0]).toMatchObject(items[5]);
138
+ expect(page2.objects?.[4]).toMatchObject(items[9]);
139
+ }).pipe(Effect.provide(TestLayer)),
140
+ );
141
+ });
@@ -32,6 +32,10 @@ export class SpaceStateManager extends Resource {
32
32
  return this._roots;
33
33
  }
34
34
 
35
+ get spaceIds(): SpaceId[] {
36
+ return Array.from(this._rootBySpace.keys());
37
+ }
38
+
35
39
  getRootByDocumentId(documentId: DocumentId): DatabaseRoot | undefined {
36
40
  return this._roots.get(documentId);
37
41
  }
@@ -0,0 +1,28 @@
1
+ //
2
+ // Copyright 2025 DXOS.org
3
+ //
4
+
5
+ import type {
6
+ DeleteFromQueueRequest,
7
+ InsertIntoQueueRequest,
8
+ QueryQueueRequest,
9
+ QueueQueryResult,
10
+ QueueService,
11
+ } from '@dxos/protocols/proto/dxos/client/services';
12
+
13
+ /**
14
+ * Stub implementation for when Edge is not available.
15
+ */
16
+ export class QueueServiceStub implements QueueService {
17
+ queryQueue(request: QueryQueueRequest): Promise<QueueQueryResult> {
18
+ throw new Error('Not available.');
19
+ }
20
+
21
+ insertIntoQueue(request: InsertIntoQueueRequest): Promise<void> {
22
+ throw new Error('Not available.');
23
+ }
24
+
25
+ deleteFromQueue(request: DeleteFromQueueRequest): Promise<void> {
26
+ throw new Error('Not available.');
27
+ }
28
+ }
@@ -17,6 +17,7 @@ import {
17
17
  type AutomergeProtocolMessage,
18
18
  DocumentCodec,
19
19
  EdgeService,
20
+ type ErrorProtocolMessage,
20
21
  type ExportBundleRequest,
21
22
  type ImportBundleRequest,
22
23
  type PeerId,
@@ -32,8 +33,8 @@ import {
32
33
  type EchoReplicator,
33
34
  type EchoReplicatorContext,
34
35
  type ReplicatorConnection,
35
- type ShouldAdvertiseParams,
36
- type ShouldSyncCollectionParams,
36
+ type ShouldAdvertiseProps,
37
+ type ShouldSyncCollectionProps,
37
38
  getSpaceIdFromCollectionId,
38
39
  } from '../automerge';
39
40
 
@@ -46,7 +47,7 @@ const INITIAL_RESTART_DELAY = 500;
46
47
  const RESTART_DELAY_JITTER = 250;
47
48
  const MAX_RESTART_DELAY = 5000;
48
49
 
49
- export type EchoEdgeReplicatorParams = {
50
+ export type EchoEdgeReplicatorProps = {
50
51
  edgeConnection: EdgeConnection;
51
52
  edgeHttpClient: EdgeHttpClient;
52
53
  disableSharePolicy?: boolean;
@@ -63,7 +64,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
63
64
  private _connections = new Map<SpaceId, EdgeReplicatorConnection>();
64
65
  private _sharePolicyEnabled = true;
65
66
 
66
- constructor({ edgeConnection, edgeHttpClient, disableSharePolicy }: EchoEdgeReplicatorParams) {
67
+ constructor({ edgeConnection, edgeHttpClient, disableSharePolicy }: EchoEdgeReplicatorProps) {
67
68
  this._edgeConnection = edgeConnection;
68
69
  this._edgeHttpClient = edgeHttpClient;
69
70
  this._sharePolicyEnabled = !disableSharePolicy;
@@ -107,6 +108,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
107
108
  }
108
109
 
109
110
  async connectToSpace(spaceId: SpaceId): Promise<void> {
111
+ log('connectToSpace', { spaceId });
110
112
  using _guard = await this._mutex.acquire();
111
113
 
112
114
  if (this._connectedSpaces.has(spaceId)) {
@@ -190,7 +192,7 @@ export class EchoEdgeReplicator implements EchoReplicator {
190
192
  }
191
193
  }
192
194
 
193
- type EdgeReplicatorConnectionsParams = {
195
+ type EdgeReplicatorConnectionsProps = {
194
196
  edgeConnection: EdgeConnection;
195
197
  edgeHttpClient: EdgeHttpClient;
196
198
  spaceId: SpaceId;
@@ -237,7 +239,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
237
239
  onRemoteConnected,
238
240
  onRemoteDisconnected,
239
241
  onRestartRequested,
240
- }: EdgeReplicatorConnectionsParams) {
242
+ }: EdgeReplicatorConnectionsProps) {
241
243
  super();
242
244
  this._edgeConnection = edgeConnection;
243
245
  this._edgeHttpClient = edgeHttpClient;
@@ -311,7 +313,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
311
313
  return this._remotePeerId;
312
314
  }
313
315
 
314
- async shouldAdvertise(params: ShouldAdvertiseParams): Promise<boolean> {
316
+ async shouldAdvertise(params: ShouldAdvertiseProps): Promise<boolean> {
315
317
  if (!this._sharedPolicyEnabled) {
316
318
  return true;
317
319
  }
@@ -336,7 +338,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
336
338
  return spaceId === this._spaceId;
337
339
  }
338
340
 
339
- shouldSyncCollection(params: ShouldSyncCollectionParams): boolean {
341
+ shouldSyncCollection(params: ShouldSyncCollectionProps): boolean {
340
342
  if (!this._sharedPolicyEnabled) {
341
343
  return true;
342
344
  }
@@ -387,6 +389,7 @@ class EdgeReplicatorConnection extends Resource implements ReplicatorConnection
387
389
  // AutomergeReplicator might return a Forbidden error if the credentials are not yet replicated.
388
390
  // We restart the connection with some delay to account for that.
389
391
  if (isErrorMessage(message)) {
392
+ log.verbose('stream error', { error: (message as ErrorProtocolMessage).message });
390
393
  this._onRestartRequested();
391
394
  return;
392
395
  }
@@ -442,6 +445,7 @@ const getMessageInfo = (msg: AutomergeProtocolMessage) => {
442
445
  return {
443
446
  type: msg.type,
444
447
  documentId: 'documentId' in msg ? msg.documentId : undefined,
448
+ collectionId: 'collectionId' in msg ? msg.collectionId : undefined,
445
449
  have,
446
450
  heads,
447
451
  need,
@@ -4,8 +4,8 @@
4
4
 
5
5
  import { describe, expect, test } from 'vitest';
6
6
 
7
- import { Filter } from '@dxos/echo';
8
- import { EXPANDO_TYPENAME, Expando, Ref } from '@dxos/echo/internal';
7
+ import { Filter, Ref } from '@dxos/echo';
8
+ import { TestSchema } from '@dxos/echo/testing';
9
9
  import { ObjectStructure } from '@dxos/echo-protocol';
10
10
  import { DXN, ObjectId, SpaceId } from '@dxos/keys';
11
11
 
@@ -23,32 +23,34 @@ describe('filterMatch', () => {
23
23
  });
24
24
 
25
25
  test('properties', () => {
26
- expect(filterMatchObject(Filter.type(Expando, { title: 'test' }).ast, OBJECT_1)).to.be.true;
27
- expect(filterMatchObject(Filter.type(Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
28
- expect(filterMatchObject(Filter.type(Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
29
- expect(filterMatchObject(Filter.type(Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
30
- expect(filterMatchObject(Filter.type(Expando, { properties: { subject: 'test' } }).ast, OBJECT_1)).to.be.true;
31
- expect(filterMatchObject(Filter.type(Expando, { array: Filter.contains('two') }).ast, OBJECT_1)).to.be.true;
26
+ expect(filterMatchObject(Filter.type(TestSchema.Expando, { title: 'test' }).ast, OBJECT_1)).to.be.true;
27
+ expect(filterMatchObject(Filter.type(TestSchema.Expando, { value: 100 }).ast, OBJECT_1)).to.be.true;
28
+ expect(filterMatchObject(Filter.type(TestSchema.Expando, { complete: false }).ast, OBJECT_1)).to.be.false;
29
+ expect(filterMatchObject(Filter.type(TestSchema.Expando, { missing: undefined }).ast, OBJECT_1)).to.be.true;
30
+ expect(filterMatchObject(Filter.type(TestSchema.Expando, { properties: { subject: 'test' } }).ast, OBJECT_1)).to.be
31
+ .true;
32
+ expect(filterMatchObject(Filter.type(TestSchema.Expando, { array: Filter.contains('two') }).ast, OBJECT_1)).to.be
33
+ .true;
32
34
  });
33
35
 
34
36
  test('and', () => {
35
- const filter1 = Filter.type(Expando, { title: 'test' });
36
- const filter2 = Filter.type(Expando, { value: 100 });
37
- const filter3 = Filter.type(Expando, { complete: true });
37
+ const filter1 = Filter.type(TestSchema.Expando, { title: 'test' });
38
+ const filter2 = Filter.type(TestSchema.Expando, { value: 100 });
39
+ const filter3 = Filter.type(TestSchema.Expando, { complete: true });
38
40
 
39
41
  expect(filterMatchObject(Filter.and(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
40
42
  });
41
43
 
42
44
  test('or', () => {
43
- const filter1 = Filter.type(Expando, { value: 200 });
44
- const filter2 = Filter.type(Expando, { title: 'test' });
45
- const filter3 = Filter.type(Expando, { complete: false });
45
+ const filter1 = Filter.type(TestSchema.Expando, { value: 200 });
46
+ const filter2 = Filter.type(TestSchema.Expando, { title: 'test' });
47
+ const filter3 = Filter.type(TestSchema.Expando, { complete: false });
46
48
 
47
49
  expect(filterMatchObject(Filter.or(filter1, filter2, filter3).ast, OBJECT_1)).to.be.true;
48
50
  });
49
51
 
50
52
  test('not', () => {
51
- const filter1 = Filter.type(Expando, { title: 'test' });
53
+ const filter1 = Filter.type(TestSchema.Expando, { title: 'test' });
52
54
  const filter2 = Filter.not(filter1);
53
55
 
54
56
  expect(filterMatchObject(filter1.ast, OBJECT_1)).to.be.true;
@@ -56,30 +58,34 @@ describe('filterMatch', () => {
56
58
  });
57
59
 
58
60
  test('contains', () => {
59
- expect(filterMatchObject(Filter.type(Expando, { properties: { label: Filter.contains('test') } }).ast, OBJECT_1)).to
60
- .be.true;
61
61
  expect(
62
62
  filterMatchObject(
63
- Filter.type(Expando, { fields: Filter.contains({ label: 'label', value: 'test' }) }).ast,
63
+ Filter.type(TestSchema.Expando, { properties: { label: Filter.contains('test') } }).ast,
64
+ OBJECT_1,
65
+ ),
66
+ ).to.be.true;
67
+ expect(
68
+ filterMatchObject(
69
+ Filter.type(TestSchema.Expando, { fields: Filter.contains({ label: 'label', value: 'test' }) }).ast,
64
70
  OBJECT_1,
65
71
  ),
66
72
  ).to.be.true;
67
73
  });
68
74
 
69
75
  test('complex', () => {
70
- const filter1 = Filter.type(Expando, { title: 'bad' });
71
- const filter2 = Filter.type(Expando, { value: 0 });
76
+ const filter1 = Filter.type(TestSchema.Expando, { title: 'bad' });
77
+ const filter2 = Filter.type(TestSchema.Expando, { value: 0 });
72
78
  expect(filterMatchObject(Filter.not(Filter.or(filter1, filter2)).ast, OBJECT_1)).to.be.true;
73
79
  });
74
80
 
75
81
  test('ids', () => {
76
- const filter = Filter.ids(OBJECT_1.id);
82
+ const filter = Filter.id(OBJECT_1.id);
77
83
  expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.true;
78
84
  expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
79
85
  });
80
86
 
81
87
  test('refs', () => {
82
- const filter = Filter.type(Expando, { parent: Ref.fromDXN(DXN.fromLocalObjectId(OBJECT_1.id)) });
88
+ const filter = Filter.type(TestSchema.Expando, { parent: Ref.fromDXN(DXN.fromLocalObjectId(OBJECT_1.id)) });
83
89
  expect(filterMatchObject(filter.ast, OBJECT_1)).to.be.false;
84
90
  expect(filterMatchObject(filter.ast, OBJECT_2)).to.be.false;
85
91
  expect(filterMatchObject(filter.ast, OBJECT_3)).to.be.true;
@@ -90,7 +96,7 @@ const OBJECT_1: MatchedObject = {
90
96
  id: ObjectId.make('01JVS9YYT5VMVJW0GGTM1YHCCH'),
91
97
  spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
92
98
  doc: ObjectStructure.makeObject({
93
- type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
99
+ type: DXN.fromTypenameAndVersion(TestSchema.Expando.typename, '0.1.0').toString(),
94
100
  data: {
95
101
  title: 'test',
96
102
  value: 100,
@@ -106,7 +112,7 @@ const OBJECT_2: MatchedObject = {
106
112
  id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
107
113
  spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
108
114
  doc: ObjectStructure.makeObject({
109
- type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
115
+ type: DXN.fromTypenameAndVersion(TestSchema.Expando.typename, '0.1.0').toString(),
110
116
  data: { title: 'test', value: 100, complete: true },
111
117
  deleted: true,
112
118
  }),
@@ -116,7 +122,7 @@ const OBJECT_3: MatchedObject = {
116
122
  id: ObjectId.make('01JT5TD6K9FFJ3VNM5FGMS5C0Q'),
117
123
  spaceId: SpaceId.make('B2NJDFNVZIW77OQSXUBNAD7BUMBD3G5PO'),
118
124
  doc: ObjectStructure.makeObject({
119
- type: DXN.fromTypenameAndVersion(EXPANDO_TYPENAME, '0.1.0').toString(),
125
+ type: DXN.fromTypenameAndVersion(TestSchema.Expando.typename, '0.1.0').toString(),
120
126
  data: { title: 'test', value: 100, complete: true, parent: { '/': DXN.fromLocalObjectId(OBJECT_1.id).toString() } },
121
127
  }),
122
128
  };
@@ -2,8 +2,8 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import { ATTR_META, EXPANDO_TYPENAME, type ObjectJSON } from '@dxos/echo/internal';
6
- import { ObjectStructure, type QueryAST, decodeReference, isEncodedReference } from '@dxos/echo-protocol';
5
+ import { ATTR_META, type ObjectJSON } from '@dxos/echo/internal';
6
+ import { EncodedReference, ObjectStructure, type QueryAST, isEncodedReference } from '@dxos/echo-protocol';
7
7
  import { DXN, type ObjectId, type SpaceId } from '@dxos/keys';
8
8
 
9
9
  export type MatchedObject = {
@@ -23,11 +23,8 @@ export const filterMatchObject = (filter: QueryAST.Filter, obj: MatchedObject):
23
23
  if (filter.typename !== null) {
24
24
  // TODO(dmaretskyi): `system` is missing in some cases.
25
25
  if (!obj.doc.system?.type?.['/']) {
26
- // Objects with no type are considered to be expando objects.
27
- const expectedDXN = DXN.parse(filter.typename).asTypeDXN();
28
- if (expectedDXN?.type !== EXPANDO_TYPENAME) {
29
- return false;
30
- }
26
+ // Objects with no type are deprecated.
27
+ return false;
31
28
  } else {
32
29
  const actualDXN = DXN.parse(obj.doc.system.type['/']);
33
30
  const expectedDXN = DXN.parse(filter.typename);
@@ -100,11 +97,8 @@ export const filterMatchObjectJSON = (filter: QueryAST.Filter, obj: ObjectJSON):
100
97
  if (filter.typename !== null) {
101
98
  // TODO(dmaretskyi): `system` is missing in some cases.
102
99
  if (!obj['@type']) {
103
- // Objects with no type are considered to be expando objects
104
- const expectedDXN = DXN.parse(filter.typename).asTypeDXN();
105
- if (expectedDXN?.type !== EXPANDO_TYPENAME) {
106
- return false;
107
- }
100
+ // Objects with no type are deprecated.
101
+ return false;
108
102
  } else {
109
103
  const actualDXN = DXN.parse(obj['@type']);
110
104
  const expectedDXN = DXN.parse(filter.typename);
@@ -151,8 +145,8 @@ export const filterMatchObjectJSON = (filter: QueryAST.Filter, obj: ObjectJSON):
151
145
  return tags.some((tag) => tag === filter.tag);
152
146
  }
153
147
 
148
+ // TODO: Implement text search.
154
149
  case 'text-search': {
155
- // TODO: Implement text search
156
150
  return false;
157
151
  }
158
152
 
@@ -219,7 +213,7 @@ export const filterMatchValue = (filter: QueryAST.Filter, value: unknown): boole
219
213
  if (!isEncodedReference(value)) {
220
214
  return false;
221
215
  }
222
- return DXN.equals(decodeReference(value).toDXN(), decodeReference(compareValue).toDXN());
216
+ return DXN.equals(EncodedReference.toDXN(value), EncodedReference.toDXN(compareValue));
223
217
  }
224
218
  return value === compareValue;
225
219
  case 'neq':
@@ -91,14 +91,17 @@ export class MetadataStore {
91
91
  log('loaded', { size: dataSize, checksum, name: file.filename });
92
92
 
93
93
  if (fileLength < dataSize + 8) {
94
- throw new DataCorruptionError('Metadata size is smaller than expected.', { fileLength, dataSize });
94
+ throw new DataCorruptionError({
95
+ message: 'Metadata size is smaller than expected.',
96
+ context: { fileLength, dataSize },
97
+ });
95
98
  }
96
99
 
97
100
  const data = await file.read(8, dataSize);
98
101
 
99
102
  const calculatedChecksum = CRC32.buf(data);
100
103
  if (calculatedChecksum !== checksum) {
101
- throw new DataCorruptionError('Metadata checksum is invalid.');
104
+ throw new DataCorruptionError({ message: 'Metadata checksum is invalid.' });
102
105
  }
103
106
 
104
107
  return codec.decode(data);