@dxos/feed 0.0.0 → 0.8.4-main.1c7ec43d41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/lib/neutral/chunk-YYASRWLE.mjs +880 -0
- package/dist/lib/neutral/chunk-YYASRWLE.mjs.map +7 -0
- package/dist/lib/neutral/index.mjs +13 -0
- package/dist/lib/neutral/index.mjs.map +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/testing/index.mjs +174 -0
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/errors.d.ts +39 -0
- package/dist/types/src/errors.d.ts.map +1 -0
- package/dist/types/src/feed-store.d.ts +121 -0
- package/dist/types/src/feed-store.d.ts.map +1 -0
- package/dist/types/src/feed-store.test.d.ts +2 -0
- package/dist/types/src/feed-store.test.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +5 -0
- package/dist/types/src/index.d.ts.map +1 -0
- package/dist/types/src/sync-client.d.ts +54 -0
- package/dist/types/src/sync-client.d.ts.map +1 -0
- package/dist/types/src/sync-server.d.ts +28 -0
- package/dist/types/src/sync-server.d.ts.map +1 -0
- package/dist/types/src/sync.test.d.ts +2 -0
- package/dist/types/src/sync.test.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +2 -0
- package/dist/types/src/testing/index.d.ts.map +1 -0
- package/dist/types/src/testing/test-builder.d.ts +78 -0
- package/dist/types/src/testing/test-builder.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -0
- package/package.json +23 -21
- package/src/errors.ts +20 -0
- package/src/feed-store.test.ts +397 -40
- package/src/feed-store.ts +508 -142
- package/src/index.ts +3 -1
- package/src/sync-client.ts +280 -0
- package/src/sync-server.ts +150 -0
- package/src/sync.test.ts +151 -91
- package/src/testing/index.ts +1 -1
- package/src/testing/test-builder.ts +223 -0
- package/src/protocol.ts +0 -110
package/src/index.ts
CHANGED
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import type * as SqlClient from '@effect/sql/SqlClient';
|
|
6
|
+
import * as Array from 'effect/Array';
|
|
7
|
+
import * as Deferred from 'effect/Deferred';
|
|
8
|
+
import * as Effect from 'effect/Effect';
|
|
9
|
+
|
|
10
|
+
import { Context, ContextDisposedError } from '@dxos/context';
|
|
11
|
+
import type { SpaceId } from '@dxos/keys';
|
|
12
|
+
import { log } from '@dxos/log';
|
|
13
|
+
import { type FeedProtocol } from '@dxos/protocols';
|
|
14
|
+
import type { SqlTransaction } from '@dxos/sql-sqlite';
|
|
15
|
+
|
|
16
|
+
import type { FeedStore } from './feed-store';
|
|
17
|
+
|
|
18
|
+
type AppendResponse = FeedProtocol.AppendResponse;
|
|
19
|
+
type ProtocolMessage = FeedProtocol.ProtocolMessage;
|
|
20
|
+
type QueryResponse = FeedProtocol.QueryResponse;
|
|
21
|
+
type QueryRequestMessage = Extract<ProtocolMessage, { _tag: 'QueryRequest' }>;
|
|
22
|
+
type AppendRequestMessage = Extract<ProtocolMessage, { _tag: 'AppendRequest' }>;
|
|
23
|
+
type RequestMessage = QueryRequestMessage | AppendRequestMessage;
|
|
24
|
+
type RequestPayload =
|
|
25
|
+
| Omit<QueryRequestMessage, 'senderPeerId' | 'recipientPeerId'>
|
|
26
|
+
| Omit<AppendRequestMessage, 'senderPeerId' | 'recipientPeerId'>;
|
|
27
|
+
|
|
28
|
+
export type SyncClientOptions = {
|
|
29
|
+
/** This client's peer id. Set as senderPeerId on all requests. */
|
|
30
|
+
peerId: string;
|
|
31
|
+
/** The server's peer id. Set as recipientPeerId on all requests. */
|
|
32
|
+
serverPeerId?: string;
|
|
33
|
+
feedStore: FeedStore;
|
|
34
|
+
/** Send a protocol message to the server. Returns Effect. */
|
|
35
|
+
sendMessage: (ctx: Context, message: RequestMessage) => Effect.Effect<void, unknown, never>;
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Client-side sync: pull/push by sending protocol messages and handling responses.
|
|
40
|
+
* Sets senderPeerId = peerId and recipientPeerId = serverPeerId on all requests.
|
|
41
|
+
* Uses a map of requestId -> Deferred to match responses to in-flight requests.
|
|
42
|
+
* handleMessage completes the Deferred for the requestId (if any) and removes the handler; no handler = noop.
|
|
43
|
+
*/
|
|
44
|
+
export class SyncClient {
|
|
45
|
+
readonly #peerId: string;
|
|
46
|
+
readonly #serverPeerId: string | undefined;
|
|
47
|
+
readonly #feedStore: FeedStore;
|
|
48
|
+
readonly #sendMessage: SyncClientOptions['sendMessage'];
|
|
49
|
+
readonly #handlers = new Map<string, Deferred.Deferred<ProtocolMessage, Error>>();
|
|
50
|
+
|
|
51
|
+
constructor(options: SyncClientOptions) {
|
|
52
|
+
this.#peerId = options.peerId;
|
|
53
|
+
this.#serverPeerId = options.serverPeerId;
|
|
54
|
+
this.#feedStore = options.feedStore;
|
|
55
|
+
this.#sendMessage = options.sendMessage;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
#withPeerIds(payload: RequestPayload): RequestMessage {
|
|
59
|
+
if (payload._tag === 'QueryRequest') {
|
|
60
|
+
return {
|
|
61
|
+
...payload,
|
|
62
|
+
senderPeerId: this.#peerId,
|
|
63
|
+
recipientPeerId: this.#serverPeerId,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
...payload,
|
|
68
|
+
senderPeerId: this.#peerId,
|
|
69
|
+
recipientPeerId: this.#serverPeerId,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Receive a message from the server. If a handler is registered for the message's requestId, complete it and remove; else noop.
|
|
75
|
+
*/
|
|
76
|
+
handleMessage(message: ProtocolMessage): Effect.Effect<void, never, never> {
|
|
77
|
+
const requestId = 'requestId' in message && message.requestId != null ? String(message.requestId) : undefined;
|
|
78
|
+
if (requestId == null) {
|
|
79
|
+
log.trace('feed sync client response ignored (no request id)', {
|
|
80
|
+
tag: message._tag,
|
|
81
|
+
senderPeerId: message.senderPeerId,
|
|
82
|
+
});
|
|
83
|
+
return Effect.void;
|
|
84
|
+
}
|
|
85
|
+
const deferred = this.#handlers.get(requestId);
|
|
86
|
+
if (deferred == null) {
|
|
87
|
+
log.trace('feed sync client response ignored (no pending rpc)', {
|
|
88
|
+
requestId,
|
|
89
|
+
tag: message._tag,
|
|
90
|
+
senderPeerId: message.senderPeerId,
|
|
91
|
+
});
|
|
92
|
+
return Effect.void;
|
|
93
|
+
}
|
|
94
|
+
this.#handlers.delete(requestId);
|
|
95
|
+
log('feed sync client rpc completed', {
|
|
96
|
+
requestId,
|
|
97
|
+
tag: message._tag,
|
|
98
|
+
blockCount: 'blocks' in message ? message.blocks.length : undefined,
|
|
99
|
+
positionCount: 'positions' in message ? message.positions.length : undefined,
|
|
100
|
+
error: message._tag === 'Error' ? message.message : undefined,
|
|
101
|
+
});
|
|
102
|
+
if (message._tag === 'Error') {
|
|
103
|
+
return Effect.andThen(Deferred.fail(deferred, new Error(message.message)), () => Effect.void);
|
|
104
|
+
}
|
|
105
|
+
return Effect.andThen(Deferred.succeed(deferred, message), () => Effect.void);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
pull(
|
|
109
|
+
ctx: Context,
|
|
110
|
+
opts: {
|
|
111
|
+
spaceId: SpaceId;
|
|
112
|
+
feedNamespace: string;
|
|
113
|
+
limit?: number;
|
|
114
|
+
},
|
|
115
|
+
): Effect.Effect<{ done: boolean }, unknown, SqlClient.SqlClient | SqlTransaction.SqlTransaction> {
|
|
116
|
+
const self = this;
|
|
117
|
+
return Effect.gen(function* () {
|
|
118
|
+
const lastPulledPosition = yield* self.#feedStore.getSyncState({
|
|
119
|
+
spaceId: opts.spaceId,
|
|
120
|
+
feedNamespace: opts.feedNamespace,
|
|
121
|
+
});
|
|
122
|
+
const requestId = crypto.randomUUID();
|
|
123
|
+
const deferred = yield* Deferred.make<ProtocolMessage, Error>();
|
|
124
|
+
self.#handlers.set(requestId, deferred);
|
|
125
|
+
const cleanupDispose = ctx.disposed
|
|
126
|
+
? () => {}
|
|
127
|
+
: ctx.onDispose(() => {
|
|
128
|
+
Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
129
|
+
});
|
|
130
|
+
if (ctx.disposed) {
|
|
131
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
132
|
+
}
|
|
133
|
+
const request: RequestPayload = {
|
|
134
|
+
_tag: 'QueryRequest',
|
|
135
|
+
requestId,
|
|
136
|
+
spaceId: opts.spaceId,
|
|
137
|
+
feedNamespace: opts.feedNamespace,
|
|
138
|
+
position: lastPulledPosition,
|
|
139
|
+
limit: opts.limit,
|
|
140
|
+
};
|
|
141
|
+
log('feed sync client pull rpc sending', {
|
|
142
|
+
requestId,
|
|
143
|
+
spaceId: opts.spaceId,
|
|
144
|
+
feedNamespace: opts.feedNamespace,
|
|
145
|
+
afterPosition: lastPulledPosition,
|
|
146
|
+
limit: opts.limit,
|
|
147
|
+
});
|
|
148
|
+
yield* self.#sendMessage(ctx, self.#withPeerIds(request));
|
|
149
|
+
const message = yield* Effect.ensuring(
|
|
150
|
+
Deferred.await(deferred),
|
|
151
|
+
Effect.sync(() => {
|
|
152
|
+
cleanupDispose();
|
|
153
|
+
self.#handlers.delete(requestId);
|
|
154
|
+
}),
|
|
155
|
+
);
|
|
156
|
+
const response = yield* self.#expectResponse<QueryResponse>(requestId, message, 'QueryResponse');
|
|
157
|
+
if (response.blocks.length === 0) {
|
|
158
|
+
log.trace('feed sync client pull done (empty batch)', {
|
|
159
|
+
requestId,
|
|
160
|
+
spaceId: opts.spaceId,
|
|
161
|
+
feedNamespace: opts.feedNamespace,
|
|
162
|
+
});
|
|
163
|
+
return { done: true };
|
|
164
|
+
}
|
|
165
|
+
yield* self.#feedStore.append({
|
|
166
|
+
spaceId: opts.spaceId,
|
|
167
|
+
feedNamespace: opts.feedNamespace,
|
|
168
|
+
blocks: response.blocks,
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// Update sync state with the max position from the pulled batch.
|
|
172
|
+
const maxPulledPosition = response.blocks.reduce(
|
|
173
|
+
(max, block) => (block.position != null && block.position > max ? block.position : max),
|
|
174
|
+
lastPulledPosition,
|
|
175
|
+
);
|
|
176
|
+
yield* self.#feedStore.setSyncState({
|
|
177
|
+
spaceId: opts.spaceId,
|
|
178
|
+
feedNamespace: opts.feedNamespace,
|
|
179
|
+
lastPulledPosition: maxPulledPosition,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
log('feed sync client pull applied batch', {
|
|
183
|
+
requestId,
|
|
184
|
+
spaceId: opts.spaceId,
|
|
185
|
+
feedNamespace: opts.feedNamespace,
|
|
186
|
+
batchSize: response.blocks.length,
|
|
187
|
+
hasMore: response.hasMore,
|
|
188
|
+
maxPulledPosition,
|
|
189
|
+
});
|
|
190
|
+
return { done: false };
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
push(
|
|
195
|
+
ctx: Context,
|
|
196
|
+
opts: {
|
|
197
|
+
spaceId: SpaceId;
|
|
198
|
+
feedNamespace: string;
|
|
199
|
+
limit?: number;
|
|
200
|
+
},
|
|
201
|
+
): Effect.Effect<{ done: boolean }, unknown, SqlClient.SqlClient | SqlTransaction.SqlTransaction> {
|
|
202
|
+
const self = this;
|
|
203
|
+
return Effect.gen(function* () {
|
|
204
|
+
const unpositioned = yield* self.#feedStore.query({
|
|
205
|
+
spaceId: opts.spaceId,
|
|
206
|
+
feedNamespace: opts.feedNamespace,
|
|
207
|
+
unpositionedOnly: true,
|
|
208
|
+
limit: opts.limit,
|
|
209
|
+
});
|
|
210
|
+
if (unpositioned.blocks.length === 0) {
|
|
211
|
+
log.trace('feed sync client push skipped (nothing to send)', {
|
|
212
|
+
spaceId: opts.spaceId,
|
|
213
|
+
feedNamespace: opts.feedNamespace,
|
|
214
|
+
});
|
|
215
|
+
return { done: true };
|
|
216
|
+
}
|
|
217
|
+
const requestId = crypto.randomUUID();
|
|
218
|
+
const deferred = yield* Deferred.make<ProtocolMessage, Error>();
|
|
219
|
+
self.#handlers.set(requestId, deferred);
|
|
220
|
+
const cleanupDispose = ctx.disposed
|
|
221
|
+
? () => {}
|
|
222
|
+
: ctx.onDispose(() => {
|
|
223
|
+
Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
224
|
+
});
|
|
225
|
+
if (ctx.disposed) {
|
|
226
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
227
|
+
}
|
|
228
|
+
const request: RequestPayload = {
|
|
229
|
+
_tag: 'AppendRequest',
|
|
230
|
+
requestId,
|
|
231
|
+
spaceId: opts.spaceId,
|
|
232
|
+
feedNamespace: opts.feedNamespace,
|
|
233
|
+
blocks: unpositioned.blocks,
|
|
234
|
+
};
|
|
235
|
+
log('feed sync client push rpc sending', {
|
|
236
|
+
requestId,
|
|
237
|
+
spaceId: opts.spaceId,
|
|
238
|
+
feedNamespace: opts.feedNamespace,
|
|
239
|
+
blockCount: unpositioned.blocks.length,
|
|
240
|
+
});
|
|
241
|
+
yield* self.#sendMessage(ctx, self.#withPeerIds(request));
|
|
242
|
+
const message = yield* Effect.ensuring(
|
|
243
|
+
Deferred.await(deferred),
|
|
244
|
+
Effect.sync(() => {
|
|
245
|
+
cleanupDispose();
|
|
246
|
+
self.#handlers.delete(requestId);
|
|
247
|
+
}),
|
|
248
|
+
);
|
|
249
|
+
const response = yield* self.#expectResponse<AppendResponse>(requestId, message, 'AppendResponse');
|
|
250
|
+
yield* self.#feedStore.setPosition({
|
|
251
|
+
spaceId: opts.spaceId,
|
|
252
|
+
blocks: Array.zipWith(response.positions, unpositioned.blocks, (position, block) => ({
|
|
253
|
+
feedId: block.feedId,
|
|
254
|
+
feedNamespace: opts.feedNamespace,
|
|
255
|
+
actorId: block.actorId,
|
|
256
|
+
sequence: block.sequence,
|
|
257
|
+
position,
|
|
258
|
+
})),
|
|
259
|
+
});
|
|
260
|
+
log('feed sync client push positions applied', {
|
|
261
|
+
requestId,
|
|
262
|
+
spaceId: opts.spaceId,
|
|
263
|
+
feedNamespace: opts.feedNamespace,
|
|
264
|
+
positionCount: response.positions.length,
|
|
265
|
+
});
|
|
266
|
+
return { done: false };
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
#expectResponse<T>(requestId: string, message: ProtocolMessage, expectedTag: string): Effect.Effect<T, Error, never> {
|
|
271
|
+
if (message._tag === 'Error') {
|
|
272
|
+
return Effect.fail(new Error(message.message));
|
|
273
|
+
}
|
|
274
|
+
const requestIdMsg = 'requestId' in message ? String(message.requestId) : undefined;
|
|
275
|
+
if (message._tag !== expectedTag || requestIdMsg !== requestId) {
|
|
276
|
+
return Effect.fail(new Error(`Unexpected message: expected ${expectedTag} with requestId ${requestId}`));
|
|
277
|
+
}
|
|
278
|
+
return Effect.succeed(message as T);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import type * as SqlClient from '@effect/sql/SqlClient';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
|
|
8
|
+
import { Context } from '@dxos/context';
|
|
9
|
+
import { log } from '@dxos/log';
|
|
10
|
+
import { type FeedProtocol } from '@dxos/protocols';
|
|
11
|
+
import type { SqlTransaction } from '@dxos/sql-sqlite';
|
|
12
|
+
|
|
13
|
+
import type { FeedStore } from './feed-store';
|
|
14
|
+
|
|
15
|
+
type AppendRequest = FeedProtocol.AppendRequest;
|
|
16
|
+
type ProtocolMessage = FeedProtocol.ProtocolMessage;
|
|
17
|
+
type QueryRequest = FeedProtocol.QueryRequest;
|
|
18
|
+
type QueryResponse = FeedProtocol.QueryResponse;
|
|
19
|
+
|
|
20
|
+
const inboundSummary = (message: ProtocolMessage) => ({
|
|
21
|
+
tag: message._tag,
|
|
22
|
+
senderPeerId: message.senderPeerId,
|
|
23
|
+
recipientPeerId: message.recipientPeerId,
|
|
24
|
+
requestId: 'requestId' in message ? message.requestId : undefined,
|
|
25
|
+
spaceId: 'spaceId' in message ? message.spaceId : undefined,
|
|
26
|
+
feedNamespace: 'feedNamespace' in message ? message.feedNamespace : undefined,
|
|
27
|
+
blockCount: 'blocks' in message ? message.blocks.length : undefined,
|
|
28
|
+
position: 'position' in message ? message.position : undefined,
|
|
29
|
+
limit: 'limit' in message ? message.limit : undefined,
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
export type SyncServerOptions = {
|
|
33
|
+
/** This server's peer id. Set as senderPeerId on all replies. */
|
|
34
|
+
peerId: string;
|
|
35
|
+
feedStore: FeedStore;
|
|
36
|
+
/** Send a protocol message to a client. Receives full message (recipient is set by server). Returns Effect. */
|
|
37
|
+
sendMessage: (ctx: Context, message: ProtocolMessage) => Effect.Effect<void, unknown, never>;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Server-side sync: receives protocol messages, runs query/append against the store, sends responses.
|
|
42
|
+
* Reusable for many peers: sets senderPeerId = peerId and recipientPeerId = incoming message's senderPeerId on replies.
|
|
43
|
+
*/
|
|
44
|
+
export class SyncServer {
|
|
45
|
+
readonly #peerId: string;
|
|
46
|
+
readonly #feedStore: FeedStore;
|
|
47
|
+
readonly #sendMessage: SyncServerOptions['sendMessage'];
|
|
48
|
+
|
|
49
|
+
constructor(options: SyncServerOptions) {
|
|
50
|
+
this.#peerId = options.peerId;
|
|
51
|
+
this.#feedStore = options.feedStore;
|
|
52
|
+
this.#sendMessage = options.sendMessage;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Receive a message from a client. Handles QueryRequest and AppendRequest; sends response via sendMessage with correct peer ids.
|
|
57
|
+
*/
|
|
58
|
+
handleMessage(
|
|
59
|
+
ctx: Context,
|
|
60
|
+
message: ProtocolMessage,
|
|
61
|
+
): Effect.Effect<void, unknown, SqlClient.SqlClient | SqlTransaction.SqlTransaction> {
|
|
62
|
+
const self = this;
|
|
63
|
+
log('feed sync server received message', {
|
|
64
|
+
peerId: self.#peerId,
|
|
65
|
+
...inboundSummary(message),
|
|
66
|
+
});
|
|
67
|
+
const recipientPeerId = message.senderPeerId;
|
|
68
|
+
const withPeerIds = (payload: Omit<ProtocolMessage, 'senderPeerId' | 'recipientPeerId'>): ProtocolMessage =>
|
|
69
|
+
({
|
|
70
|
+
...payload,
|
|
71
|
+
senderPeerId: self.#peerId,
|
|
72
|
+
recipientPeerId,
|
|
73
|
+
}) as ProtocolMessage;
|
|
74
|
+
switch (message._tag) {
|
|
75
|
+
case 'QueryRequest': {
|
|
76
|
+
const req = message as QueryRequest;
|
|
77
|
+
return Effect.gen(function* () {
|
|
78
|
+
const response: QueryResponse = yield* self.#feedStore.query(req);
|
|
79
|
+
log('feed sync server query completed', {
|
|
80
|
+
peerId: self.#peerId,
|
|
81
|
+
recipientPeerId,
|
|
82
|
+
requestId: req.requestId,
|
|
83
|
+
spaceId: req.spaceId,
|
|
84
|
+
feedNamespace: req.feedNamespace,
|
|
85
|
+
returnedBlocks: response.blocks.length,
|
|
86
|
+
hasMore: response.hasMore,
|
|
87
|
+
});
|
|
88
|
+
yield* self.#sendMessage(ctx, withPeerIds({ _tag: 'QueryResponse', ...response }));
|
|
89
|
+
}).pipe(
|
|
90
|
+
Effect.catchAll((err: unknown) => {
|
|
91
|
+
log('feed sync server query failed', {
|
|
92
|
+
peerId: self.#peerId,
|
|
93
|
+
recipientPeerId,
|
|
94
|
+
requestId: req.requestId,
|
|
95
|
+
spaceId: req.spaceId,
|
|
96
|
+
feedNamespace: req.feedNamespace,
|
|
97
|
+
cause: err instanceof Error ? err.message : String(err),
|
|
98
|
+
});
|
|
99
|
+
return self.#sendMessage(
|
|
100
|
+
ctx,
|
|
101
|
+
withPeerIds({
|
|
102
|
+
_tag: 'Error',
|
|
103
|
+
message: err instanceof Error ? err.message : String(err),
|
|
104
|
+
} as Omit<ProtocolMessage, 'senderPeerId' | 'recipientPeerId'>),
|
|
105
|
+
);
|
|
106
|
+
}),
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
case 'AppendRequest': {
|
|
110
|
+
const req = message as AppendRequest;
|
|
111
|
+
return Effect.gen(function* () {
|
|
112
|
+
const response = yield* self.#feedStore.append(req);
|
|
113
|
+
log('feed sync server append completed', {
|
|
114
|
+
peerId: self.#peerId,
|
|
115
|
+
recipientPeerId,
|
|
116
|
+
requestId: req.requestId,
|
|
117
|
+
spaceId: req.spaceId,
|
|
118
|
+
feedNamespace: req.feedNamespace,
|
|
119
|
+
assignedPositions: response.positions.length,
|
|
120
|
+
});
|
|
121
|
+
yield* self.#sendMessage(ctx, withPeerIds({ _tag: 'AppendResponse', ...response }));
|
|
122
|
+
}).pipe(
|
|
123
|
+
Effect.catchAll((err: unknown) => {
|
|
124
|
+
log('feed sync server append failed', {
|
|
125
|
+
peerId: self.#peerId,
|
|
126
|
+
recipientPeerId,
|
|
127
|
+
requestId: req.requestId,
|
|
128
|
+
spaceId: req.spaceId,
|
|
129
|
+
feedNamespace: req.feedNamespace,
|
|
130
|
+
cause: err instanceof Error ? err.message : String(err),
|
|
131
|
+
});
|
|
132
|
+
return self.#sendMessage(
|
|
133
|
+
ctx,
|
|
134
|
+
withPeerIds({
|
|
135
|
+
_tag: 'Error',
|
|
136
|
+
message: err instanceof Error ? err.message : String(err),
|
|
137
|
+
} as Omit<ProtocolMessage, 'senderPeerId' | 'recipientPeerId'>),
|
|
138
|
+
);
|
|
139
|
+
}),
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
default:
|
|
143
|
+
log('feed sync server ignoring unsupported message', {
|
|
144
|
+
peerId: self.#peerId,
|
|
145
|
+
tag: message._tag,
|
|
146
|
+
});
|
|
147
|
+
return Effect.void;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
package/src/sync.test.ts
CHANGED
|
@@ -2,102 +2,162 @@
|
|
|
2
2
|
// Copyright 2026 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
6
|
-
import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
|
|
7
|
-
import { describe, expect, it } from '@effect/vitest';
|
|
8
|
-
import * as Effect from 'effect/Effect';
|
|
9
|
-
import * as ManagedRuntime from 'effect/ManagedRuntime';
|
|
10
|
-
|
|
11
|
-
import { SpaceId } from '@dxos/keys';
|
|
12
|
-
|
|
13
|
-
import { FeedStore } from './feed-store';
|
|
14
|
-
import { type Block } from './protocol';
|
|
15
|
-
|
|
16
|
-
describe.skip('Feed Sync V2 (RPC)', () => {
|
|
17
|
-
const makePeer = () => {
|
|
18
|
-
const layer = SqliteClient.layer({ filename: ':memory:' });
|
|
19
|
-
const runtime = ManagedRuntime.make(layer);
|
|
20
|
-
const run = <A, E>(effect: Effect.Effect<A, E, SqlClient.SqlClient>) => runtime.runPromise(effect);
|
|
21
|
-
return { run };
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
it('should sync blocks from server to client via RPC', async () => {
|
|
25
|
-
const server = makePeer();
|
|
26
|
-
const client = makePeer();
|
|
27
|
-
|
|
28
|
-
const spaceId = SpaceId.random();
|
|
29
|
-
const feedId = '01H1V1X1X1X1X1X1X1X1X1X1X3';
|
|
30
|
-
|
|
31
|
-
// Server setup: Append blocks
|
|
32
|
-
await server.run(
|
|
33
|
-
Effect.gen(function* () {
|
|
34
|
-
const feed = new FeedStore({ localActorId: 'server', assignPositions: true });
|
|
35
|
-
const blocks: Block[] = [
|
|
36
|
-
{
|
|
37
|
-
feedId: undefined,
|
|
38
|
-
actorId: feedId,
|
|
39
|
-
sequence: 1,
|
|
40
|
-
predActorId: null,
|
|
41
|
-
predSequence: null,
|
|
42
|
-
position: null,
|
|
43
|
-
timestamp: 100,
|
|
44
|
-
data: new Uint8Array([1]),
|
|
45
|
-
},
|
|
46
|
-
{
|
|
47
|
-
feedId: undefined,
|
|
48
|
-
actorId: feedId,
|
|
49
|
-
sequence: 2,
|
|
50
|
-
predActorId: feedId,
|
|
51
|
-
predSequence: 1,
|
|
52
|
-
position: null,
|
|
53
|
-
timestamp: 200,
|
|
54
|
-
data: new Uint8Array([2]),
|
|
55
|
-
},
|
|
56
|
-
];
|
|
57
|
-
yield* feed.append({ requestId: 'req-0', blocks, spaceId, namespace: 'data', feedId });
|
|
58
|
-
}),
|
|
59
|
-
);
|
|
5
|
+
import { describe, expect, test } from '@effect/vitest';
|
|
60
6
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
7
|
+
import { RuntimeProvider } from '@dxos/effect';
|
|
8
|
+
import { ObjectId, SpaceId } from '@dxos/keys';
|
|
9
|
+
import { FeedProtocol } from '@dxos/protocols';
|
|
10
|
+
import { range } from '@dxos/util';
|
|
11
|
+
|
|
12
|
+
import { TestBuilder } from './testing';
|
|
13
|
+
|
|
14
|
+
const WellKnownNamespaces = FeedProtocol.WellKnownNamespaces;
|
|
15
|
+
|
|
16
|
+
describe('Sync', () => {
|
|
17
|
+
const LOG_SQL = false;
|
|
18
|
+
|
|
19
|
+
const spaceId = SpaceId.random();
|
|
20
|
+
const feedId = ObjectId.random();
|
|
21
|
+
|
|
22
|
+
test('pull blocks from server', async () => {
|
|
23
|
+
await using builder = await new TestBuilder({ numPeers: 2, spaceId, logSql: LOG_SQL }).open();
|
|
24
|
+
const [server, client] = builder.peers;
|
|
25
|
+
|
|
26
|
+
const testBlocks = generateTestBlocks(0, 5);
|
|
27
|
+
|
|
28
|
+
await server.feedStore
|
|
29
|
+
.appendLocal(
|
|
30
|
+
testBlocks.map((block) => ({
|
|
31
|
+
spaceId,
|
|
32
|
+
feedId,
|
|
33
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
34
|
+
data: block,
|
|
35
|
+
})),
|
|
36
|
+
)
|
|
37
|
+
.pipe(RuntimeProvider.runPromise(server.runtime.runtimeEffect));
|
|
38
|
+
|
|
39
|
+
await builder.pull(client);
|
|
40
|
+
|
|
41
|
+
const { blocks } = await client.feedStore
|
|
42
|
+
.query({
|
|
43
|
+
spaceId,
|
|
44
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
45
|
+
})
|
|
46
|
+
.pipe(RuntimeProvider.runPromise(client.runtime.runtimeEffect));
|
|
47
|
+
expect(blocks.map((block) => block.data)).toEqual(testBlocks);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test('push blocks from client to server', async () => {
|
|
51
|
+
await using builder = await new TestBuilder({ numPeers: 2, spaceId, logSql: LOG_SQL }).open();
|
|
52
|
+
const [server, client] = builder.peers;
|
|
53
|
+
|
|
54
|
+
const testBlocks = generateTestBlocks(0, 5);
|
|
55
|
+
|
|
56
|
+
await client.feedStore
|
|
57
|
+
.appendLocal(
|
|
58
|
+
testBlocks.map((block) => ({
|
|
59
|
+
spaceId,
|
|
60
|
+
feedId,
|
|
61
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
62
|
+
data: block,
|
|
63
|
+
})),
|
|
64
|
+
)
|
|
65
|
+
.pipe(RuntimeProvider.runPromise(client.runtime.runtimeEffect));
|
|
66
|
+
|
|
67
|
+
await builder.push(client);
|
|
72
68
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
69
|
+
const serverBlocks = await server.feedStore
|
|
70
|
+
.query({
|
|
71
|
+
spaceId,
|
|
72
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
73
|
+
})
|
|
74
|
+
.pipe(RuntimeProvider.runPromise(server.runtime.runtimeEffect));
|
|
75
|
+
expect(serverBlocks.blocks.map((block) => block.data)).toEqual(testBlocks);
|
|
76
|
+
expect(serverBlocks.blocks.every((block) => block.position != null)).toBe(true);
|
|
77
|
+
|
|
78
|
+
const clientBlocks = await client.feedStore
|
|
79
|
+
.query({
|
|
80
|
+
spaceId,
|
|
81
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
82
|
+
})
|
|
83
|
+
.pipe(RuntimeProvider.runPromise(client.runtime.runtimeEffect));
|
|
84
|
+
expect(clientBlocks.blocks.map((block) => block.position)).toEqual(
|
|
85
|
+
serverBlocks.blocks.map((block) => block.position),
|
|
80
86
|
);
|
|
87
|
+
});
|
|
81
88
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
// Verify
|
|
93
|
-
const myBlocks = yield* feed.query({
|
|
94
|
-
requestId: 'req-verify',
|
|
95
|
-
query: { feedIds: [feedId] },
|
|
96
|
-
position: 0,
|
|
89
|
+
test('push blocks incrementally in batches', async () => {
|
|
90
|
+
await using builder = await new TestBuilder({ numPeers: 2, spaceId, logSql: LOG_SQL }).open();
|
|
91
|
+
const [server, client] = builder.peers;
|
|
92
|
+
|
|
93
|
+
const testBlocks = generateTestBlocks(0, 5);
|
|
94
|
+
|
|
95
|
+
await client.feedStore
|
|
96
|
+
.appendLocal(
|
|
97
|
+
testBlocks.map((block) => ({
|
|
97
98
|
spaceId,
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
feedId,
|
|
100
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
101
|
+
data: block,
|
|
102
|
+
})),
|
|
103
|
+
)
|
|
104
|
+
.pipe(RuntimeProvider.runPromise(client.runtime.runtimeEffect));
|
|
105
|
+
|
|
106
|
+
let done = false,
|
|
107
|
+
numBatches = 0;
|
|
108
|
+
while (!done) {
|
|
109
|
+
numBatches++;
|
|
110
|
+
const result = await builder.push(client, { limit: 2 });
|
|
111
|
+
done = result.done;
|
|
112
|
+
}
|
|
113
|
+
await builder.push(client);
|
|
114
|
+
expect(numBatches).toBeGreaterThan(2);
|
|
115
|
+
expect(numBatches).toBeLessThan(10);
|
|
116
|
+
|
|
117
|
+
const serverBlocks = await server.feedStore
|
|
118
|
+
.query({
|
|
119
|
+
spaceId,
|
|
120
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
121
|
+
})
|
|
122
|
+
.pipe(RuntimeProvider.runPromise(server.runtime.runtimeEffect));
|
|
123
|
+
expect(serverBlocks.blocks.map((block) => block.data)).toEqual(testBlocks);
|
|
124
|
+
expect(serverBlocks.blocks.every((block) => block.position != null)).toBe(true);
|
|
125
|
+
|
|
126
|
+
const clientBlocks = await client.feedStore
|
|
127
|
+
.query({
|
|
128
|
+
spaceId,
|
|
129
|
+
feedNamespace: WellKnownNamespaces.data,
|
|
130
|
+
})
|
|
131
|
+
.pipe(RuntimeProvider.runPromise(client.runtime.runtimeEffect));
|
|
132
|
+
expect(clientBlocks.blocks.map((block) => block.position)).toEqual(
|
|
133
|
+
serverBlocks.blocks.map((block) => block.position),
|
|
101
134
|
);
|
|
102
135
|
});
|
|
136
|
+
|
|
137
|
+
test('3-way sync', async () => {
|
|
138
|
+
await using builder = await new TestBuilder({ numPeers: 3, spaceId, logSql: LOG_SQL }).open();
|
|
139
|
+
const [, client1, client2] = builder.peers;
|
|
140
|
+
|
|
141
|
+
const testBlocks = generateTestBlocks(0, 5);
|
|
142
|
+
|
|
143
|
+
await client1.feedStore
|
|
144
|
+
.appendLocal(
|
|
145
|
+
testBlocks.map((block) => ({ spaceId, feedId, feedNamespace: WellKnownNamespaces.data, data: block })),
|
|
146
|
+
)
|
|
147
|
+
.pipe(RuntimeProvider.runPromise(client1.runtime.runtimeEffect));
|
|
148
|
+
await builder.push(client1);
|
|
149
|
+
|
|
150
|
+
await builder.pull(client2);
|
|
151
|
+
|
|
152
|
+
const { blocks: client1Blocks } = await client1.feedStore
|
|
153
|
+
.query({ spaceId, feedNamespace: WellKnownNamespaces.data })
|
|
154
|
+
.pipe(RuntimeProvider.runPromise(client1.runtime.runtimeEffect));
|
|
155
|
+
const { blocks: client2Blocks } = await client2.feedStore
|
|
156
|
+
.query({ spaceId, feedNamespace: WellKnownNamespaces.data })
|
|
157
|
+
.pipe(RuntimeProvider.runPromise(client2.runtime.runtimeEffect));
|
|
158
|
+
expect(client1Blocks).toEqual(client2Blocks);
|
|
159
|
+
expect(client1Blocks.every((block) => block.position != null)).toBe(true);
|
|
160
|
+
});
|
|
103
161
|
});
|
|
162
|
+
|
|
163
|
+
const generateTestBlocks = (start: number, count: number) => range(count, (i) => new Uint8Array([start + i]));
|