@dxos/feed 0.8.4-main.abd8ff62ef → 0.8.4-main.bc2380dfbc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/dist/lib/neutral/{chunk-CJV6NY5B.mjs → chunk-YYASRWLE.mjs} +144 -11
- package/dist/lib/neutral/{chunk-CJV6NY5B.mjs.map → chunk-YYASRWLE.mjs.map} +3 -3
- package/dist/lib/neutral/index.mjs +1 -1
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/lib/neutral/testing/index.mjs +1 -1
- package/dist/types/src/sync-client.d.ts.map +1 -1
- package/dist/types/src/sync-server.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +12 -12
- package/src/sync-client.ts +84 -3
- package/src/sync-server.ts +62 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/feed",
|
|
3
|
-
"version": "0.8.4-main.
|
|
3
|
+
"version": "0.8.4-main.bc2380dfbc",
|
|
4
4
|
"description": "Append-only replicatable log",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"type": "git",
|
|
9
9
|
"url": "https://github.com/dxos/dxos"
|
|
10
10
|
},
|
|
11
|
-
"license": "
|
|
11
|
+
"license": "FSL-1.1-Apache-2.0",
|
|
12
12
|
"author": "info@dxos.org",
|
|
13
13
|
"sideEffects": true,
|
|
14
14
|
"type": "module",
|
|
@@ -33,16 +33,16 @@
|
|
|
33
33
|
"@effect/sql": "0.49.0",
|
|
34
34
|
"@effect/sql-sqlite-node": "0.50.1",
|
|
35
35
|
"effect": "3.20.0",
|
|
36
|
-
"@dxos/
|
|
37
|
-
"@dxos/
|
|
38
|
-
"@dxos/
|
|
39
|
-
"@dxos/
|
|
40
|
-
"@dxos/
|
|
41
|
-
"@dxos/
|
|
42
|
-
"@dxos/
|
|
43
|
-
"@dxos/protocols": "0.8.4-main.
|
|
44
|
-
"@dxos/sql-sqlite": "0.8.4-main.
|
|
45
|
-
"@dxos/util": "0.8.4-main.
|
|
36
|
+
"@dxos/effect": "0.8.4-main.bc2380dfbc",
|
|
37
|
+
"@dxos/errors": "0.8.4-main.bc2380dfbc",
|
|
38
|
+
"@dxos/invariant": "0.8.4-main.bc2380dfbc",
|
|
39
|
+
"@dxos/async": "0.8.4-main.bc2380dfbc",
|
|
40
|
+
"@dxos/context": "0.8.4-main.bc2380dfbc",
|
|
41
|
+
"@dxos/log": "0.8.4-main.bc2380dfbc",
|
|
42
|
+
"@dxos/keys": "0.8.4-main.bc2380dfbc",
|
|
43
|
+
"@dxos/protocols": "0.8.4-main.bc2380dfbc",
|
|
44
|
+
"@dxos/sql-sqlite": "0.8.4-main.bc2380dfbc",
|
|
45
|
+
"@dxos/util": "0.8.4-main.bc2380dfbc"
|
|
46
46
|
},
|
|
47
47
|
"devDependencies": {
|
|
48
48
|
"@effect/vitest": "0.29.0"
|
package/src/sync-client.ts
CHANGED
|
@@ -7,8 +7,9 @@ import * as Array from 'effect/Array';
|
|
|
7
7
|
import * as Deferred from 'effect/Deferred';
|
|
8
8
|
import * as Effect from 'effect/Effect';
|
|
9
9
|
|
|
10
|
-
import { Context } from '@dxos/context';
|
|
10
|
+
import { Context, ContextDisposedError } from '@dxos/context';
|
|
11
11
|
import type { SpaceId } from '@dxos/keys';
|
|
12
|
+
import { log } from '@dxos/log';
|
|
12
13
|
import { type FeedProtocol } from '@dxos/protocols';
|
|
13
14
|
import type { SqlTransaction } from '@dxos/sql-sqlite';
|
|
14
15
|
|
|
@@ -75,13 +76,29 @@ export class SyncClient {
|
|
|
75
76
|
handleMessage(message: ProtocolMessage): Effect.Effect<void, never, never> {
|
|
76
77
|
const requestId = 'requestId' in message && message.requestId != null ? String(message.requestId) : undefined;
|
|
77
78
|
if (requestId == null) {
|
|
79
|
+
log.trace('feed sync client response ignored (no request id)', {
|
|
80
|
+
tag: message._tag,
|
|
81
|
+
senderPeerId: message.senderPeerId,
|
|
82
|
+
});
|
|
78
83
|
return Effect.void;
|
|
79
84
|
}
|
|
80
85
|
const deferred = this.#handlers.get(requestId);
|
|
81
86
|
if (deferred == null) {
|
|
87
|
+
log.trace('feed sync client response ignored (no pending rpc)', {
|
|
88
|
+
requestId,
|
|
89
|
+
tag: message._tag,
|
|
90
|
+
senderPeerId: message.senderPeerId,
|
|
91
|
+
});
|
|
82
92
|
return Effect.void;
|
|
83
93
|
}
|
|
84
94
|
this.#handlers.delete(requestId);
|
|
95
|
+
log('feed sync client rpc completed', {
|
|
96
|
+
requestId,
|
|
97
|
+
tag: message._tag,
|
|
98
|
+
blockCount: 'blocks' in message ? message.blocks.length : undefined,
|
|
99
|
+
positionCount: 'positions' in message ? message.positions.length : undefined,
|
|
100
|
+
error: message._tag === 'Error' ? message.message : undefined,
|
|
101
|
+
});
|
|
85
102
|
if (message._tag === 'Error') {
|
|
86
103
|
return Effect.andThen(Deferred.fail(deferred, new Error(message.message)), () => Effect.void);
|
|
87
104
|
}
|
|
@@ -105,6 +122,14 @@ export class SyncClient {
|
|
|
105
122
|
const requestId = crypto.randomUUID();
|
|
106
123
|
const deferred = yield* Deferred.make<ProtocolMessage, Error>();
|
|
107
124
|
self.#handlers.set(requestId, deferred);
|
|
125
|
+
const cleanupDispose = ctx.disposed
|
|
126
|
+
? () => {}
|
|
127
|
+
: ctx.onDispose(() => {
|
|
128
|
+
Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
129
|
+
});
|
|
130
|
+
if (ctx.disposed) {
|
|
131
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
132
|
+
}
|
|
108
133
|
const request: RequestPayload = {
|
|
109
134
|
_tag: 'QueryRequest',
|
|
110
135
|
requestId,
|
|
@@ -113,10 +138,28 @@ export class SyncClient {
|
|
|
113
138
|
position: lastPulledPosition,
|
|
114
139
|
limit: opts.limit,
|
|
115
140
|
};
|
|
141
|
+
log('feed sync client pull rpc sending', {
|
|
142
|
+
requestId,
|
|
143
|
+
spaceId: opts.spaceId,
|
|
144
|
+
feedNamespace: opts.feedNamespace,
|
|
145
|
+
afterPosition: lastPulledPosition,
|
|
146
|
+
limit: opts.limit,
|
|
147
|
+
});
|
|
116
148
|
yield* self.#sendMessage(ctx, self.#withPeerIds(request));
|
|
117
|
-
const message = yield*
|
|
149
|
+
const message = yield* Effect.ensuring(
|
|
150
|
+
Deferred.await(deferred),
|
|
151
|
+
Effect.sync(() => {
|
|
152
|
+
cleanupDispose();
|
|
153
|
+
self.#handlers.delete(requestId);
|
|
154
|
+
}),
|
|
155
|
+
);
|
|
118
156
|
const response = yield* self.#expectResponse<QueryResponse>(requestId, message, 'QueryResponse');
|
|
119
157
|
if (response.blocks.length === 0) {
|
|
158
|
+
log.trace('feed sync client pull done (empty batch)', {
|
|
159
|
+
requestId,
|
|
160
|
+
spaceId: opts.spaceId,
|
|
161
|
+
feedNamespace: opts.feedNamespace,
|
|
162
|
+
});
|
|
120
163
|
return { done: true };
|
|
121
164
|
}
|
|
122
165
|
yield* self.#feedStore.append({
|
|
@@ -136,6 +179,14 @@ export class SyncClient {
|
|
|
136
179
|
lastPulledPosition: maxPulledPosition,
|
|
137
180
|
});
|
|
138
181
|
|
|
182
|
+
log('feed sync client pull applied batch', {
|
|
183
|
+
requestId,
|
|
184
|
+
spaceId: opts.spaceId,
|
|
185
|
+
feedNamespace: opts.feedNamespace,
|
|
186
|
+
batchSize: response.blocks.length,
|
|
187
|
+
hasMore: response.hasMore,
|
|
188
|
+
maxPulledPosition,
|
|
189
|
+
});
|
|
139
190
|
return { done: false };
|
|
140
191
|
});
|
|
141
192
|
}
|
|
@@ -157,11 +208,23 @@ export class SyncClient {
|
|
|
157
208
|
limit: opts.limit,
|
|
158
209
|
});
|
|
159
210
|
if (unpositioned.blocks.length === 0) {
|
|
211
|
+
log.trace('feed sync client push skipped (nothing to send)', {
|
|
212
|
+
spaceId: opts.spaceId,
|
|
213
|
+
feedNamespace: opts.feedNamespace,
|
|
214
|
+
});
|
|
160
215
|
return { done: true };
|
|
161
216
|
}
|
|
162
217
|
const requestId = crypto.randomUUID();
|
|
163
218
|
const deferred = yield* Deferred.make<ProtocolMessage, Error>();
|
|
164
219
|
self.#handlers.set(requestId, deferred);
|
|
220
|
+
const cleanupDispose = ctx.disposed
|
|
221
|
+
? () => {}
|
|
222
|
+
: ctx.onDispose(() => {
|
|
223
|
+
Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
224
|
+
});
|
|
225
|
+
if (ctx.disposed) {
|
|
226
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
227
|
+
}
|
|
165
228
|
const request: RequestPayload = {
|
|
166
229
|
_tag: 'AppendRequest',
|
|
167
230
|
requestId,
|
|
@@ -169,8 +232,20 @@ export class SyncClient {
|
|
|
169
232
|
feedNamespace: opts.feedNamespace,
|
|
170
233
|
blocks: unpositioned.blocks,
|
|
171
234
|
};
|
|
235
|
+
log('feed sync client push rpc sending', {
|
|
236
|
+
requestId,
|
|
237
|
+
spaceId: opts.spaceId,
|
|
238
|
+
feedNamespace: opts.feedNamespace,
|
|
239
|
+
blockCount: unpositioned.blocks.length,
|
|
240
|
+
});
|
|
172
241
|
yield* self.#sendMessage(ctx, self.#withPeerIds(request));
|
|
173
|
-
const message = yield*
|
|
242
|
+
const message = yield* Effect.ensuring(
|
|
243
|
+
Deferred.await(deferred),
|
|
244
|
+
Effect.sync(() => {
|
|
245
|
+
cleanupDispose();
|
|
246
|
+
self.#handlers.delete(requestId);
|
|
247
|
+
}),
|
|
248
|
+
);
|
|
174
249
|
const response = yield* self.#expectResponse<AppendResponse>(requestId, message, 'AppendResponse');
|
|
175
250
|
yield* self.#feedStore.setPosition({
|
|
176
251
|
spaceId: opts.spaceId,
|
|
@@ -182,6 +257,12 @@ export class SyncClient {
|
|
|
182
257
|
position,
|
|
183
258
|
})),
|
|
184
259
|
});
|
|
260
|
+
log('feed sync client push positions applied', {
|
|
261
|
+
requestId,
|
|
262
|
+
spaceId: opts.spaceId,
|
|
263
|
+
feedNamespace: opts.feedNamespace,
|
|
264
|
+
positionCount: response.positions.length,
|
|
265
|
+
});
|
|
185
266
|
return { done: false };
|
|
186
267
|
});
|
|
187
268
|
}
|
package/src/sync-server.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type * as SqlClient from '@effect/sql/SqlClient';
|
|
|
6
6
|
import * as Effect from 'effect/Effect';
|
|
7
7
|
|
|
8
8
|
import { Context } from '@dxos/context';
|
|
9
|
+
import { log } from '@dxos/log';
|
|
9
10
|
import { type FeedProtocol } from '@dxos/protocols';
|
|
10
11
|
import type { SqlTransaction } from '@dxos/sql-sqlite';
|
|
11
12
|
|
|
@@ -16,6 +17,18 @@ type ProtocolMessage = FeedProtocol.ProtocolMessage;
|
|
|
16
17
|
type QueryRequest = FeedProtocol.QueryRequest;
|
|
17
18
|
type QueryResponse = FeedProtocol.QueryResponse;
|
|
18
19
|
|
|
20
|
+
const inboundSummary = (message: ProtocolMessage) => ({
|
|
21
|
+
tag: message._tag,
|
|
22
|
+
senderPeerId: message.senderPeerId,
|
|
23
|
+
recipientPeerId: message.recipientPeerId,
|
|
24
|
+
requestId: 'requestId' in message ? message.requestId : undefined,
|
|
25
|
+
spaceId: 'spaceId' in message ? message.spaceId : undefined,
|
|
26
|
+
feedNamespace: 'feedNamespace' in message ? message.feedNamespace : undefined,
|
|
27
|
+
blockCount: 'blocks' in message ? message.blocks.length : undefined,
|
|
28
|
+
position: 'position' in message ? message.position : undefined,
|
|
29
|
+
limit: 'limit' in message ? message.limit : undefined,
|
|
30
|
+
});
|
|
31
|
+
|
|
19
32
|
export type SyncServerOptions = {
|
|
20
33
|
/** This server's peer id. Set as senderPeerId on all replies. */
|
|
21
34
|
peerId: string;
|
|
@@ -47,6 +60,10 @@ export class SyncServer {
|
|
|
47
60
|
message: ProtocolMessage,
|
|
48
61
|
): Effect.Effect<void, unknown, SqlClient.SqlClient | SqlTransaction.SqlTransaction> {
|
|
49
62
|
const self = this;
|
|
63
|
+
log('feed sync server received message', {
|
|
64
|
+
peerId: self.#peerId,
|
|
65
|
+
...inboundSummary(message),
|
|
66
|
+
});
|
|
50
67
|
const recipientPeerId = message.senderPeerId;
|
|
51
68
|
const withPeerIds = (payload: Omit<ProtocolMessage, 'senderPeerId' | 'recipientPeerId'>): ProtocolMessage =>
|
|
52
69
|
({
|
|
@@ -59,37 +76,74 @@ export class SyncServer {
|
|
|
59
76
|
const req = message as QueryRequest;
|
|
60
77
|
return Effect.gen(function* () {
|
|
61
78
|
const response: QueryResponse = yield* self.#feedStore.query(req);
|
|
79
|
+
log('feed sync server query completed', {
|
|
80
|
+
peerId: self.#peerId,
|
|
81
|
+
recipientPeerId,
|
|
82
|
+
requestId: req.requestId,
|
|
83
|
+
spaceId: req.spaceId,
|
|
84
|
+
feedNamespace: req.feedNamespace,
|
|
85
|
+
returnedBlocks: response.blocks.length,
|
|
86
|
+
hasMore: response.hasMore,
|
|
87
|
+
});
|
|
62
88
|
yield* self.#sendMessage(ctx, withPeerIds({ _tag: 'QueryResponse', ...response }));
|
|
63
89
|
}).pipe(
|
|
64
|
-
Effect.catchAll((err: unknown) =>
|
|
65
|
-
|
|
90
|
+
Effect.catchAll((err: unknown) => {
|
|
91
|
+
log('feed sync server query failed', {
|
|
92
|
+
peerId: self.#peerId,
|
|
93
|
+
recipientPeerId,
|
|
94
|
+
requestId: req.requestId,
|
|
95
|
+
spaceId: req.spaceId,
|
|
96
|
+
feedNamespace: req.feedNamespace,
|
|
97
|
+
cause: err instanceof Error ? err.message : String(err),
|
|
98
|
+
});
|
|
99
|
+
return self.#sendMessage(
|
|
66
100
|
ctx,
|
|
67
101
|
withPeerIds({
|
|
68
102
|
_tag: 'Error',
|
|
69
103
|
message: err instanceof Error ? err.message : String(err),
|
|
70
104
|
} as Omit<ProtocolMessage, 'senderPeerId' | 'recipientPeerId'>),
|
|
71
|
-
)
|
|
72
|
-
),
|
|
105
|
+
);
|
|
106
|
+
}),
|
|
73
107
|
);
|
|
74
108
|
}
|
|
75
109
|
case 'AppendRequest': {
|
|
76
110
|
const req = message as AppendRequest;
|
|
77
111
|
return Effect.gen(function* () {
|
|
78
112
|
const response = yield* self.#feedStore.append(req);
|
|
113
|
+
log('feed sync server append completed', {
|
|
114
|
+
peerId: self.#peerId,
|
|
115
|
+
recipientPeerId,
|
|
116
|
+
requestId: req.requestId,
|
|
117
|
+
spaceId: req.spaceId,
|
|
118
|
+
feedNamespace: req.feedNamespace,
|
|
119
|
+
assignedPositions: response.positions.length,
|
|
120
|
+
});
|
|
79
121
|
yield* self.#sendMessage(ctx, withPeerIds({ _tag: 'AppendResponse', ...response }));
|
|
80
122
|
}).pipe(
|
|
81
|
-
Effect.catchAll((err: unknown) =>
|
|
82
|
-
|
|
123
|
+
Effect.catchAll((err: unknown) => {
|
|
124
|
+
log('feed sync server append failed', {
|
|
125
|
+
peerId: self.#peerId,
|
|
126
|
+
recipientPeerId,
|
|
127
|
+
requestId: req.requestId,
|
|
128
|
+
spaceId: req.spaceId,
|
|
129
|
+
feedNamespace: req.feedNamespace,
|
|
130
|
+
cause: err instanceof Error ? err.message : String(err),
|
|
131
|
+
});
|
|
132
|
+
return self.#sendMessage(
|
|
83
133
|
ctx,
|
|
84
134
|
withPeerIds({
|
|
85
135
|
_tag: 'Error',
|
|
86
136
|
message: err instanceof Error ? err.message : String(err),
|
|
87
137
|
} as Omit<ProtocolMessage, 'senderPeerId' | 'recipientPeerId'>),
|
|
88
|
-
)
|
|
89
|
-
),
|
|
138
|
+
);
|
|
139
|
+
}),
|
|
90
140
|
);
|
|
91
141
|
}
|
|
92
142
|
default:
|
|
143
|
+
log('feed sync server ignoring unsupported message', {
|
|
144
|
+
peerId: self.#peerId,
|
|
145
|
+
tag: message._tag,
|
|
146
|
+
});
|
|
93
147
|
return Effect.void;
|
|
94
148
|
}
|
|
95
149
|
}
|