@dxos/feed 0.8.4-main.bc674ce → 0.8.4-main.ef1bc66f44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/lib/neutral/chunk-Q5PE2Q5P.mjs +735 -0
- package/dist/lib/neutral/chunk-Q5PE2Q5P.mjs.map +7 -0
- package/dist/lib/neutral/index.mjs +17 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/testing/index.mjs +173 -0
- package/dist/lib/neutral/testing/index.mjs.map +7 -0
- package/dist/types/src/errors.d.ts +79 -0
- package/dist/types/src/errors.d.ts.map +1 -0
- package/dist/types/src/feed-store.d.ts +68 -9
- package/dist/types/src/feed-store.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +3 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/sync-client.d.ts +53 -0
- package/dist/types/src/sync-client.d.ts.map +1 -0
- package/dist/types/src/sync-server.d.ts +27 -0
- package/dist/types/src/sync-server.d.ts.map +1 -0
- package/dist/types/src/testing/index.d.ts +1 -1
- package/dist/types/src/testing/index.d.ts.map +1 -1
- package/dist/types/src/testing/test-builder.d.ts +78 -0
- package/dist/types/src/testing/test-builder.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +16 -13
- package/src/errors.ts +33 -0
- package/src/feed-store.test.ts +283 -35
- package/src/feed-store.ts +479 -132
- package/src/index.ts +3 -1
- package/src/sync-client.ts +188 -0
- package/src/sync-server.ts +92 -0
- package/src/sync.test.ts +151 -91
- package/src/testing/index.ts +1 -1
- package/src/testing/test-builder.ts +223 -0
- package/dist/lib/browser/index.mjs +0 -342
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/browser/testing/index.mjs +0 -1
- package/dist/lib/node-esm/index.mjs +0 -344
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/testing/index.mjs +0 -2
- package/dist/lib/node-esm/testing/index.mjs.map +0 -7
- package/dist/types/src/protocol.d.ts +0 -120
- package/dist/types/src/protocol.d.ts.map +0 -1
- package/src/protocol.ts +0 -110
- /package/dist/lib/{browser/testing → neutral}/index.mjs.map +0 -0
package/src/feed-store.ts
CHANGED
|
@@ -7,73 +7,112 @@ import type * as SqlError from '@effect/sql/SqlError';
|
|
|
7
7
|
import * as Effect from 'effect/Effect';
|
|
8
8
|
|
|
9
9
|
import { Event } from '@dxos/async';
|
|
10
|
-
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
10
|
+
import { assertArgument } from '@dxos/invariant';
|
|
11
|
+
import { type SpaceId } from '@dxos/keys';
|
|
12
|
+
import { FeedProtocol } from '@dxos/protocols';
|
|
13
|
+
import { SqlTransaction } from '@dxos/sql-sqlite';
|
|
14
|
+
|
|
15
|
+
import { DuplicateBlockLamportTimestampError, DuplicateBlockPositionError, PositionConflictError } from './errors';
|
|
16
|
+
|
|
17
|
+
type AppendRequest = FeedProtocol.AppendRequest;
|
|
18
|
+
type AppendResponse = FeedProtocol.AppendResponse;
|
|
19
|
+
type Block = FeedProtocol.Block;
|
|
20
|
+
const FeedCursor = FeedProtocol.FeedCursor;
|
|
21
|
+
type FeedCursor = FeedProtocol.FeedCursor;
|
|
22
|
+
const isWellKnownNamespace = FeedProtocol.isWellKnownNamespace;
|
|
23
|
+
type QueryRequest = FeedProtocol.QueryRequest;
|
|
24
|
+
type QueryResponse = FeedProtocol.QueryResponse;
|
|
25
|
+
type SubscribeRequest = FeedProtocol.SubscribeRequest;
|
|
26
|
+
type SubscribeResponse = FeedProtocol.SubscribeResponse;
|
|
21
27
|
|
|
22
28
|
export interface FeedStoreOptions {
|
|
29
|
+
/**
|
|
30
|
+
* The actor ID of the local user.
|
|
31
|
+
*/
|
|
23
32
|
localActorId: string;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Whether to assign positions to appended blocks.
|
|
36
|
+
* Only a single peer (usually the server) can assign positions.
|
|
37
|
+
*/
|
|
24
38
|
assignPositions: boolean;
|
|
25
39
|
}
|
|
26
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Persistent storage for feed metadata, blocks, subscriptions, and sync state.
|
|
43
|
+
*
|
|
44
|
+
*/
|
|
27
45
|
export class FeedStore {
|
|
28
|
-
|
|
46
|
+
readonly #options: FeedStoreOptions;
|
|
47
|
+
|
|
48
|
+
constructor(options: FeedStoreOptions) {
|
|
49
|
+
this.#options = options;
|
|
50
|
+
}
|
|
29
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Emits after successful block append operations.
|
|
54
|
+
*/
|
|
30
55
|
readonly onNewBlocks = new Event<void>();
|
|
31
56
|
|
|
32
|
-
|
|
33
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Creates required feed store tables and indexes if they do not exist.
|
|
59
|
+
*/
|
|
60
|
+
migrate = Effect.fn('FeedStore.migrate')(() =>
|
|
61
|
+
Effect.gen(function* () {
|
|
62
|
+
const sql = yield* SqlClient.SqlClient;
|
|
34
63
|
|
|
35
|
-
|
|
36
|
-
|
|
64
|
+
// Feeds Table
|
|
65
|
+
yield* sql`CREATE TABLE IF NOT EXISTS feeds (
|
|
37
66
|
feedPrivateId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
38
67
|
spaceId TEXT NOT NULL,
|
|
39
68
|
feedId TEXT NOT NULL,
|
|
40
69
|
feedNamespace TEXT
|
|
41
70
|
)`;
|
|
42
|
-
|
|
71
|
+
yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_feeds_spaceId_feedId ON feeds(spaceId, feedId)`;
|
|
43
72
|
|
|
44
|
-
|
|
45
|
-
|
|
73
|
+
// Blocks Table
|
|
74
|
+
yield* sql`CREATE TABLE IF NOT EXISTS blocks (
|
|
46
75
|
insertionId INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
47
76
|
feedPrivateId INTEGER NOT NULL,
|
|
48
77
|
position INTEGER,
|
|
49
78
|
sequence INTEGER NOT NULL,
|
|
50
79
|
actorId TEXT NOT NULL,
|
|
51
|
-
|
|
52
|
-
|
|
80
|
+
prevSequence INTEGER,
|
|
81
|
+
prevActorId TEXT,
|
|
53
82
|
timestamp INTEGER NOT NULL,
|
|
54
83
|
data BLOB NOT NULL,
|
|
55
84
|
FOREIGN KEY(feedPrivateId) REFERENCES feeds(feedPrivateId)
|
|
56
85
|
)`;
|
|
57
|
-
|
|
58
|
-
|
|
86
|
+
yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_position ON blocks(feedPrivateId, position)`;
|
|
87
|
+
yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_sequence_actorId ON blocks(feedPrivateId, sequence, actorId)`;
|
|
59
88
|
|
|
60
|
-
|
|
61
|
-
|
|
89
|
+
// Subscriptions Table
|
|
90
|
+
yield* sql`CREATE TABLE IF NOT EXISTS subscriptions (
|
|
62
91
|
subscriptionId TEXT PRIMARY KEY,
|
|
63
92
|
expiresAt INTEGER NOT NULL,
|
|
64
93
|
feedPrivateIds TEXT NOT NULL -- JSON array
|
|
65
94
|
)`;
|
|
66
95
|
|
|
67
|
-
|
|
68
|
-
|
|
96
|
+
// Cursor Tokens Table
|
|
97
|
+
yield* sql`CREATE TABLE IF NOT EXISTS cursor_tokens (
|
|
69
98
|
spaceId TEXT PRIMARY KEY,
|
|
70
99
|
token TEXT NOT NULL
|
|
71
100
|
)`;
|
|
72
|
-
});
|
|
73
101
|
|
|
74
|
-
|
|
102
|
+
// Sync State Table.
|
|
103
|
+
yield* sql`CREATE TABLE IF NOT EXISTS sync_state (
|
|
104
|
+
spaceId TEXT NOT NULL,
|
|
105
|
+
feedNamespace TEXT NOT NULL,
|
|
106
|
+
lastPulledPosition INTEGER NOT NULL DEFAULT -1,
|
|
107
|
+
PRIMARY KEY (spaceId, feedNamespace)
|
|
108
|
+
)`;
|
|
109
|
+
}).pipe(Effect.withSpan('FeedStore.migrate')),
|
|
110
|
+
);
|
|
75
111
|
|
|
76
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Ensures a feed row exists and returns its internal feed ID.
|
|
114
|
+
*/
|
|
115
|
+
#ensureFeed = Effect.fn('Feed.ensureFeed')(
|
|
77
116
|
(
|
|
78
117
|
spaceId: string,
|
|
79
118
|
feedId: string,
|
|
@@ -91,10 +130,13 @@ export class FeedStore {
|
|
|
91
130
|
INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
|
|
92
131
|
`;
|
|
93
132
|
return newRows[0].feedPrivateId;
|
|
94
|
-
}),
|
|
133
|
+
}).pipe(Effect.withSpan('FeedStore.ensureFeed')),
|
|
95
134
|
);
|
|
96
135
|
|
|
97
|
-
|
|
136
|
+
/**
|
|
137
|
+
* Ensures cursor token exists for a space and returns it.
|
|
138
|
+
*/
|
|
139
|
+
#ensureCursorToken = Effect.fn('Feed.ensureCursorToken')(
|
|
98
140
|
(spaceId: string): Effect.Effect<string, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
99
141
|
Effect.gen(this, function* () {
|
|
100
142
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -104,11 +146,12 @@ export class FeedStore {
|
|
|
104
146
|
const token = crypto.randomUUID().replace(/-/g, '').slice(0, 6);
|
|
105
147
|
yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
|
|
106
148
|
return token;
|
|
107
|
-
}),
|
|
149
|
+
}).pipe(Effect.withSpan('FeedStore.ensureCursorToken')),
|
|
108
150
|
);
|
|
109
151
|
|
|
110
|
-
|
|
111
|
-
|
|
152
|
+
/**
|
|
153
|
+
* Queries feed blocks by feed IDs or subscription with cursor/position pagination.
|
|
154
|
+
*/
|
|
112
155
|
query = Effect.fn('Feed.query')(
|
|
113
156
|
(request: QueryRequest): Effect.Effect<QueryResponse, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
114
157
|
Effect.gen(this, function* () {
|
|
@@ -121,6 +164,15 @@ export class FeedStore {
|
|
|
121
164
|
return yield* Effect.die(new Error('spaceId is required'));
|
|
122
165
|
}
|
|
123
166
|
|
|
167
|
+
if (
|
|
168
|
+
(request.position !== undefined ? 1 : 0) +
|
|
169
|
+
(request.cursor !== undefined ? 1 : 0) +
|
|
170
|
+
(request.unpositionedOnly === true ? 1 : 0) >
|
|
171
|
+
1
|
|
172
|
+
) {
|
|
173
|
+
return yield* Effect.die(new Error('Only one of position, cursor, or unpositionedOnly can be used'));
|
|
174
|
+
}
|
|
175
|
+
|
|
124
176
|
if (request.cursor) {
|
|
125
177
|
const { token, insertionId } = decodeCursor(request.cursor as FeedCursor);
|
|
126
178
|
if (!token || insertionId === undefined || isNaN(insertionId)) {
|
|
@@ -130,8 +182,8 @@ export class FeedStore {
|
|
|
130
182
|
cursorInsertionId = insertionId;
|
|
131
183
|
}
|
|
132
184
|
|
|
133
|
-
// Validate Token if cursor used
|
|
134
|
-
const validCursorToken = yield* this
|
|
185
|
+
// Validate Token if cursor used.
|
|
186
|
+
const validCursorToken = yield* this.#ensureCursorToken(request.spaceId);
|
|
135
187
|
if (request.cursor && cursorToken !== validCursorToken) {
|
|
136
188
|
return yield* Effect.die(new Error(`Cursor token mismatch`));
|
|
137
189
|
}
|
|
@@ -150,8 +202,8 @@ export class FeedStore {
|
|
|
150
202
|
|
|
151
203
|
const position = request.position ?? -1;
|
|
152
204
|
|
|
153
|
-
// Resolve
|
|
154
|
-
if ('subscriptionId' in request.query) {
|
|
205
|
+
// Resolve subscriptions or feed IDs.
|
|
206
|
+
if (request.query && 'subscriptionId' in request.query) {
|
|
155
207
|
const rows = yield* sql<{ feedPrivateIds: string; expiresAt: number }>`
|
|
156
208
|
SELECT feedPrivateIds, expiresAt FROM subscriptions WHERE subscriptionId = ${request.query.subscriptionId}
|
|
157
209
|
`;
|
|
@@ -167,50 +219,57 @@ export class FeedStore {
|
|
|
167
219
|
}
|
|
168
220
|
}
|
|
169
221
|
}
|
|
170
|
-
} else if ('feedIds' in request.query) {
|
|
171
|
-
feedIds = [...
|
|
222
|
+
} else if (request.query && 'feedIds' in request.query) {
|
|
223
|
+
feedIds = [...request.query.feedIds];
|
|
172
224
|
} else {
|
|
173
225
|
feedIds = undefined;
|
|
174
226
|
}
|
|
175
227
|
|
|
176
228
|
if (feedIds !== undefined && feedIds.length === 0) {
|
|
177
|
-
return {
|
|
229
|
+
return {
|
|
230
|
+
requestId: request.requestId,
|
|
231
|
+
blocks: [],
|
|
232
|
+
nextCursor: encodeCursor(validCursorToken, -1),
|
|
233
|
+
hasMore: false,
|
|
234
|
+
};
|
|
178
235
|
}
|
|
179
236
|
|
|
180
|
-
// Fetch
|
|
237
|
+
// Fetch blocks.
|
|
181
238
|
const query = sql<Block>`
|
|
182
|
-
SELECT blocks.*, feeds.feedId
|
|
239
|
+
SELECT blocks.*, feeds.feedId, feeds.feedNamespace
|
|
183
240
|
FROM blocks
|
|
184
241
|
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
185
242
|
WHERE 1=1
|
|
186
243
|
${feedIds !== undefined ? sql`AND feeds.feedId IN ${sql.in(feedIds)}` : sql``}
|
|
187
244
|
${request.spaceId ? sql`AND feeds.spaceId = ${request.spaceId}` : sql``}
|
|
188
|
-
${
|
|
189
|
-
'feedNamespace' in request.query && request.query.feedNamespace
|
|
190
|
-
? sql`AND feeds.feedNamespace = ${request.query.feedNamespace}`
|
|
191
|
-
: sql``
|
|
192
|
-
}
|
|
245
|
+
${sql`AND feeds.feedNamespace = ${request.feedNamespace}`}
|
|
193
246
|
`;
|
|
194
247
|
|
|
195
|
-
// Add filter based on cursor or position
|
|
248
|
+
// Add filter based on cursor or position.
|
|
196
249
|
const filter = request.cursor
|
|
197
250
|
? sql`AND blocks.insertionId > ${cursorInsertionId}`
|
|
198
|
-
:
|
|
251
|
+
: request.unpositionedOnly
|
|
252
|
+
? sql`AND blocks.position IS NULL`
|
|
253
|
+
: sql`AND (blocks.position > ${position} OR blocks.position IS NULL)`;
|
|
199
254
|
|
|
200
255
|
const orderBy = request.cursor
|
|
201
256
|
? sql`ORDER BY blocks.insertionId ASC`
|
|
202
257
|
: sql`ORDER BY blocks.position ASC NULLS LAST`;
|
|
203
258
|
|
|
259
|
+
const requestLimit = request.limit;
|
|
260
|
+
const queryLimit = requestLimit != null ? requestLimit + 1 : undefined;
|
|
204
261
|
const rows = yield* sql<Block>`
|
|
205
262
|
${query}
|
|
206
263
|
${filter}
|
|
207
264
|
${orderBy}
|
|
208
|
-
${
|
|
265
|
+
${queryLimit != null ? sql`LIMIT ${queryLimit}` : sql``}
|
|
209
266
|
`;
|
|
210
267
|
|
|
211
|
-
const
|
|
268
|
+
const hasMore = requestLimit != null && rows.length > requestLimit;
|
|
269
|
+
const slice = hasMore ? rows.slice(0, requestLimit) : rows;
|
|
270
|
+
const blocks = slice.map((row) => ({
|
|
212
271
|
...row,
|
|
213
|
-
// Have to buffer otherwise we get empty Uint8Array.
|
|
272
|
+
// Have to clone buffer otherwise we get empty Uint8Array.
|
|
214
273
|
data: new Uint8Array(row.data),
|
|
215
274
|
}));
|
|
216
275
|
|
|
@@ -222,10 +281,13 @@ export class FeedStore {
|
|
|
222
281
|
}
|
|
223
282
|
}
|
|
224
283
|
|
|
225
|
-
return { requestId: request.requestId, blocks, nextCursor } satisfies QueryResponse;
|
|
226
|
-
}),
|
|
284
|
+
return { requestId: request.requestId, blocks, nextCursor, hasMore } satisfies QueryResponse;
|
|
285
|
+
}).pipe(Effect.withSpan('FeedStore.query')),
|
|
227
286
|
);
|
|
228
287
|
|
|
288
|
+
/**
|
|
289
|
+
* Creates a subscription and stores the resolved internal feed IDs.
|
|
290
|
+
*/
|
|
229
291
|
subscribe = Effect.fn('Feed.subscribe')(
|
|
230
292
|
(request: SubscribeRequest): Effect.Effect<SubscribeResponse, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
231
293
|
Effect.gen(this, function* () {
|
|
@@ -241,7 +303,7 @@ export class FeedStore {
|
|
|
241
303
|
|
|
242
304
|
const feedPrivateIds = yield* Effect.forEach(
|
|
243
305
|
request.feedIds,
|
|
244
|
-
(feedId) => this
|
|
306
|
+
(feedId) => this.#ensureFeed(request.spaceId!, feedId),
|
|
245
307
|
{ concurrency: 'unbounded' },
|
|
246
308
|
);
|
|
247
309
|
|
|
@@ -255,119 +317,404 @@ export class FeedStore {
|
|
|
255
317
|
subscriptionId,
|
|
256
318
|
expiresAt,
|
|
257
319
|
};
|
|
258
|
-
}),
|
|
320
|
+
}).pipe(Effect.withSpan('FeedStore.subscribe')),
|
|
259
321
|
);
|
|
260
322
|
|
|
261
|
-
|
|
262
|
-
|
|
323
|
+
/**
|
|
324
|
+
* Get the last pulled position for the given space and namespace.
|
|
325
|
+
* Returns -1 if no sync state exists yet.
|
|
326
|
+
*/
|
|
327
|
+
getSyncState = (opts: {
|
|
328
|
+
spaceId: SpaceId;
|
|
329
|
+
feedNamespace: string;
|
|
330
|
+
}): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
331
|
+
Effect.gen(this, function* () {
|
|
332
|
+
const sql = yield* SqlClient.SqlClient;
|
|
333
|
+
const rows = yield* sql<{ lastPulledPosition: number }>`
|
|
334
|
+
SELECT lastPulledPosition FROM sync_state
|
|
335
|
+
WHERE spaceId = ${opts.spaceId} AND feedNamespace = ${opts.feedNamespace}
|
|
336
|
+
`;
|
|
337
|
+
return rows[0]?.lastPulledPosition ?? -1;
|
|
338
|
+
}).pipe(Effect.withSpan('FeedStore.getSyncState'));
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Update the last pulled position for the given space and namespace.
|
|
342
|
+
*/
|
|
343
|
+
setSyncState = (opts: {
|
|
344
|
+
spaceId: SpaceId;
|
|
345
|
+
feedNamespace: string;
|
|
346
|
+
lastPulledPosition: number;
|
|
347
|
+
}): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
|
|
348
|
+
Effect.gen(this, function* () {
|
|
349
|
+
const sql = yield* SqlClient.SqlClient;
|
|
350
|
+
yield* sql`
|
|
351
|
+
INSERT INTO sync_state (spaceId, feedNamespace, lastPulledPosition)
|
|
352
|
+
VALUES (${opts.spaceId}, ${opts.feedNamespace}, ${opts.lastPulledPosition})
|
|
353
|
+
ON CONFLICT (spaceId, feedNamespace) DO UPDATE SET lastPulledPosition = ${opts.lastPulledPosition}
|
|
354
|
+
`;
|
|
355
|
+
}).pipe(Effect.withSpan('FeedStore.setSyncState'));
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Validates that appended blocks do not introduce position or Lamport conflicts.
|
|
359
|
+
*/
|
|
360
|
+
#validateAppendBlocks = Effect.fn('FeedStore.validateAppendBlocks')(
|
|
361
|
+
(
|
|
362
|
+
blocks: readonly Block[],
|
|
363
|
+
feedPrivateIds: Map<string, number>,
|
|
364
|
+
): Effect.Effect<
|
|
365
|
+
void,
|
|
366
|
+
SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
|
|
367
|
+
SqlClient.SqlClient
|
|
368
|
+
> =>
|
|
263
369
|
Effect.gen(this, function* () {
|
|
264
370
|
const sql = yield* SqlClient.SqlClient;
|
|
265
|
-
const positions: number[] = [];
|
|
266
371
|
|
|
267
|
-
|
|
268
|
-
|
|
372
|
+
// Validate request-level uniqueness before writing.
|
|
373
|
+
const positionFeedIdByKey = new Map<string, string>();
|
|
374
|
+
const lamportFeedIdByKey = new Map<string, string>();
|
|
375
|
+
for (const block of blocks) {
|
|
376
|
+
const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
|
|
377
|
+
if (!this.#options.assignPositions && block.position != null) {
|
|
378
|
+
const positionKey = `${feedPrivateId}|${block.position}`;
|
|
379
|
+
if (positionFeedIdByKey.has(positionKey)) {
|
|
380
|
+
yield* Effect.fail(
|
|
381
|
+
new DuplicateBlockPositionError({
|
|
382
|
+
feedId: block.feedId!,
|
|
383
|
+
position: block.position,
|
|
384
|
+
}),
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
positionFeedIdByKey.set(positionKey, block.feedId!);
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
const lamportKey = `${feedPrivateId}|${block.actorId}|${block.sequence}`;
|
|
391
|
+
if (lamportFeedIdByKey.has(lamportKey)) {
|
|
392
|
+
yield* Effect.fail(
|
|
393
|
+
new DuplicateBlockLamportTimestampError({
|
|
394
|
+
feedId: block.feedId!,
|
|
395
|
+
actorId: block.actorId,
|
|
396
|
+
sequence: block.sequence,
|
|
397
|
+
}),
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
lamportFeedIdByKey.set(lamportKey, block.feedId!);
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Bulk-validate against persisted rows with exact tuple matching.
|
|
404
|
+
const lamportConditions = blocks.map((block) => {
|
|
405
|
+
const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
|
|
406
|
+
return sql`(feedPrivateId = ${feedPrivateId} AND actorId = ${block.actorId} AND sequence = ${block.sequence})`;
|
|
407
|
+
});
|
|
408
|
+
if (lamportConditions.length > 0) {
|
|
409
|
+
const existingLamports = yield* sql<{ feedPrivateId: number; actorId: string; sequence: number }>`
|
|
410
|
+
SELECT feedPrivateId, actorId, sequence
|
|
411
|
+
FROM blocks
|
|
412
|
+
WHERE ${sql.or(lamportConditions)}
|
|
413
|
+
LIMIT 1
|
|
414
|
+
`;
|
|
415
|
+
const lamportConflict = existingLamports[0];
|
|
416
|
+
if (lamportConflict) {
|
|
417
|
+
const key = `${lamportConflict.feedPrivateId}|${lamportConflict.actorId}|${lamportConflict.sequence}`;
|
|
418
|
+
yield* Effect.fail(
|
|
419
|
+
new DuplicateBlockLamportTimestampError({
|
|
420
|
+
feedId: lamportFeedIdByKey.get(key)!,
|
|
421
|
+
actorId: lamportConflict.actorId,
|
|
422
|
+
sequence: lamportConflict.sequence,
|
|
423
|
+
}),
|
|
424
|
+
);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
if (!this.#options.assignPositions) {
|
|
429
|
+
const positionedBlocks = blocks.filter((block) => block.position != null);
|
|
430
|
+
const positionConditions = positionedBlocks.map((block) => {
|
|
431
|
+
const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
|
|
432
|
+
return sql`(feedPrivateId = ${feedPrivateId} AND position = ${block.position})`;
|
|
433
|
+
});
|
|
434
|
+
if (positionConditions.length > 0) {
|
|
435
|
+
const existingPositions = yield* sql<{ feedPrivateId: number; position: number }>`
|
|
436
|
+
SELECT feedPrivateId, position
|
|
437
|
+
FROM blocks
|
|
438
|
+
WHERE position IS NOT NULL
|
|
439
|
+
AND (${sql.or(positionConditions)})
|
|
440
|
+
LIMIT 1
|
|
441
|
+
`;
|
|
442
|
+
const positionConflict = existingPositions[0];
|
|
443
|
+
if (positionConflict) {
|
|
444
|
+
const key = `${positionConflict.feedPrivateId}|${positionConflict.position}`;
|
|
445
|
+
yield* Effect.fail(
|
|
446
|
+
new DuplicateBlockPositionError({
|
|
447
|
+
feedId: positionFeedIdByKey.get(key)!,
|
|
448
|
+
position: positionConflict.position,
|
|
449
|
+
}),
|
|
450
|
+
);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
269
453
|
}
|
|
454
|
+
}),
|
|
455
|
+
);
|
|
270
456
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
457
|
+
/**
|
|
458
|
+
* Appends blocks for a space/namespace and optionally assigns global positions.
|
|
459
|
+
*/
|
|
460
|
+
append = (
|
|
461
|
+
request: AppendRequest,
|
|
462
|
+
): Effect.Effect<
|
|
463
|
+
AppendResponse,
|
|
464
|
+
SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
|
|
465
|
+
SqlClient.SqlClient | SqlTransaction.SqlTransaction
|
|
466
|
+
> =>
|
|
467
|
+
Effect.gen(this, function* () {
|
|
468
|
+
if (!request.spaceId) {
|
|
469
|
+
return yield* Effect.die(new Error('spaceId required for append'));
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
assertArgument(
|
|
473
|
+
isWellKnownNamespace(request.feedNamespace),
|
|
474
|
+
'request.feedNamespace',
|
|
475
|
+
'specified well-known namespace',
|
|
476
|
+
);
|
|
477
|
+
|
|
478
|
+
// Validate all blocks upfront.
|
|
479
|
+
for (const block of request.blocks) {
|
|
480
|
+
assertArgument(block.feedId, 'block.feedId', 'feedId is required');
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// Wrap in transaction to ensure atomicity when assigning positions.
|
|
484
|
+
const sqlTransaction = yield* SqlTransaction.SqlTransaction;
|
|
485
|
+
const positions = yield* sqlTransaction.withTransaction(
|
|
486
|
+
Effect.gen(this, function* () {
|
|
487
|
+
const sql = yield* SqlClient.SqlClient;
|
|
488
|
+
|
|
489
|
+
// 1. Collect unique feed IDs and batch #ensureFeed calls.
|
|
490
|
+
const feedKeys = new Map<string, { feedId: string }>();
|
|
491
|
+
for (const block of request.blocks) {
|
|
492
|
+
const key = block.feedId!;
|
|
493
|
+
if (!feedKeys.has(key)) {
|
|
494
|
+
feedKeys.set(key, { feedId: block.feedId! });
|
|
495
|
+
}
|
|
496
|
+
}
|
|
274
497
|
|
|
275
|
-
|
|
276
|
-
|
|
498
|
+
const feedPrivateIds = new Map<string, number>();
|
|
499
|
+
yield* Effect.forEach(
|
|
500
|
+
[...feedKeys.entries()],
|
|
501
|
+
([key, { feedId }]) =>
|
|
502
|
+
Effect.gen(this, function* () {
|
|
503
|
+
const id = yield* this.#ensureFeed(request.spaceId!, feedId, request.feedNamespace);
|
|
504
|
+
feedPrivateIds.set(key, id);
|
|
505
|
+
}),
|
|
506
|
+
{ concurrency: 'unbounded' },
|
|
507
|
+
);
|
|
508
|
+
|
|
509
|
+
yield* this.#validateAppendBlocks(request.blocks, feedPrivateIds);
|
|
510
|
+
|
|
511
|
+
// 2. Get max position per namespace ONCE (not per block).
|
|
512
|
+
const maxPositions = new Map<string, number>();
|
|
513
|
+
if (this.#options.assignPositions) {
|
|
277
514
|
const maxPosResult = yield* sql<{ maxPos: number | null }>`
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
positions.push(nextPos);
|
|
515
|
+
SELECT MAX(position) as maxPos
|
|
516
|
+
FROM blocks
|
|
517
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
518
|
+
WHERE feeds.spaceId = ${request.spaceId} AND feeds.feedNamespace = ${request.feedNamespace}
|
|
519
|
+
`;
|
|
520
|
+
maxPositions.set(request.feedNamespace, maxPosResult[0]?.maxPos ?? -1);
|
|
285
521
|
}
|
|
286
522
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
523
|
+
// 3. Insert all blocks and compute positions.
|
|
524
|
+
const positions: number[] = [];
|
|
525
|
+
for (const block of request.blocks) {
|
|
526
|
+
const key = block.feedId!;
|
|
527
|
+
const feedPrivateId = feedPrivateIds.get(key)!;
|
|
528
|
+
|
|
529
|
+
let positionToInsert: number | null = null;
|
|
530
|
+
if (this.#options.assignPositions) {
|
|
531
|
+
const currentMax = maxPositions.get(request.feedNamespace)!;
|
|
532
|
+
positionToInsert = currentMax + 1;
|
|
533
|
+
maxPositions.set(request.feedNamespace, positionToInsert); // Increment for next block in same namespace.
|
|
534
|
+
positions.push(positionToInsert);
|
|
535
|
+
} else if (block.position != null) {
|
|
536
|
+
positionToInsert = block.position;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
yield* sql`
|
|
540
|
+
INSERT INTO blocks (
|
|
541
|
+
feedPrivateId, position, sequence, actorId,
|
|
542
|
+
prevSequence, prevActorId, timestamp, data
|
|
543
|
+
) VALUES (
|
|
544
|
+
${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
|
|
545
|
+
${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
|
|
546
|
+
) ON CONFLICT DO NOTHING
|
|
295
547
|
`;
|
|
296
|
-
|
|
548
|
+
}
|
|
297
549
|
|
|
298
|
-
|
|
550
|
+
return positions;
|
|
551
|
+
}),
|
|
552
|
+
);
|
|
299
553
|
|
|
300
|
-
|
|
301
|
-
}),
|
|
302
|
-
);
|
|
554
|
+
this.onNewBlocks.emit();
|
|
303
555
|
|
|
556
|
+
return { requestId: request.requestId, positions };
|
|
557
|
+
}).pipe(Effect.withSpan('FeedStore.append'));
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* Creates local blocks with sequential predecessors and appends grouped batches.
|
|
561
|
+
*/
|
|
304
562
|
appendLocal = Effect.fn('Feed.appendLocal')(
|
|
305
563
|
(
|
|
306
564
|
messages: { spaceId: string; feedId: string; feedNamespace: string; data: Uint8Array }[],
|
|
307
|
-
): Effect.Effect<
|
|
565
|
+
): Effect.Effect<
|
|
566
|
+
Block[],
|
|
567
|
+
SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
|
|
568
|
+
SqlClient.SqlClient | SqlTransaction.SqlTransaction
|
|
569
|
+
> =>
|
|
308
570
|
Effect.gen(this, function* () {
|
|
309
571
|
const sql = yield* SqlClient.SqlClient;
|
|
310
|
-
const blocks: Block[] = [];
|
|
311
|
-
|
|
312
|
-
// Group by spaceId for efficient processing? Or just iterate.
|
|
313
|
-
// Assuming small batch.
|
|
314
572
|
|
|
573
|
+
// 1. Collect unique feeds and ensure they exist.
|
|
574
|
+
type FeedKey = string; // `${spaceId}|${feedNamespace}|${feedId}`
|
|
575
|
+
const feedKeys = new Map<FeedKey, { spaceId: string; feedId: string; feedNamespace: string }>();
|
|
315
576
|
for (const msg of messages) {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
577
|
+
const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
|
|
578
|
+
if (!feedKeys.has(key)) {
|
|
579
|
+
feedKeys.set(key, { spaceId: msg.spaceId, feedId: msg.feedId, feedNamespace: msg.feedNamespace });
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// Batch ensure feeds and get their private IDs.
|
|
584
|
+
const feedPrivateIds = new Map<FeedKey, number>();
|
|
585
|
+
yield* Effect.forEach(
|
|
586
|
+
[...feedKeys.entries()],
|
|
587
|
+
([key, { spaceId, feedId, feedNamespace }]) =>
|
|
588
|
+
Effect.gen(this, function* () {
|
|
589
|
+
const id = yield* this.#ensureFeed(spaceId, feedId, feedNamespace);
|
|
590
|
+
feedPrivateIds.set(key, id);
|
|
591
|
+
}),
|
|
592
|
+
{ concurrency: 'unbounded' },
|
|
593
|
+
);
|
|
594
|
+
|
|
595
|
+
// 2. Get last sequence for each unique feed (batch query).
|
|
596
|
+
const lastSeqs = new Map<FeedKey, { sequence: number; actorId: string } | null>();
|
|
597
|
+
for (const [key] of feedKeys) {
|
|
598
|
+
const feedPrivateId = feedPrivateIds.get(key)!;
|
|
335
599
|
const lastBlockResult = yield* sql<{ sequence: number; actorId: string }>`
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
600
|
+
SELECT sequence, actorId FROM blocks
|
|
601
|
+
WHERE feedPrivateId = ${feedPrivateId}
|
|
602
|
+
ORDER BY sequence DESC
|
|
603
|
+
LIMIT 1
|
|
604
|
+
`;
|
|
605
|
+
lastSeqs.set(key, lastBlockResult[0] ?? null);
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
// 3. Build all blocks with correct sequences.
|
|
609
|
+
// Track in-flight sequences per feed to handle multiple messages to same feed.
|
|
610
|
+
const currentSeqs = new Map<FeedKey, { sequence: number; actorId: string }>();
|
|
611
|
+
const blocks: Block[] = [];
|
|
612
|
+
const blocksBySpaceNamespace = new Map<string, { spaceId: string; feedNamespace: string; blocks: Block[] }>();
|
|
341
613
|
|
|
342
|
-
|
|
343
|
-
const
|
|
344
|
-
|
|
345
|
-
|
|
614
|
+
for (const msg of messages) {
|
|
615
|
+
const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
|
|
616
|
+
|
|
617
|
+
// Determine predecessor: either from in-flight blocks or from DB.
|
|
618
|
+
let sequence: number;
|
|
619
|
+
let prevSequence: number | null;
|
|
620
|
+
let prevActorId: string | null;
|
|
621
|
+
|
|
622
|
+
const inFlight = currentSeqs.get(key);
|
|
623
|
+
if (inFlight) {
|
|
624
|
+
// We've already added blocks for this feed - continue from last in-flight.
|
|
625
|
+
sequence = inFlight.sequence + 1;
|
|
626
|
+
prevSequence = inFlight.sequence;
|
|
627
|
+
prevActorId = inFlight.actorId;
|
|
628
|
+
} else {
|
|
629
|
+
// First block for this feed - use DB state.
|
|
630
|
+
const lastBlock = lastSeqs.get(key);
|
|
631
|
+
sequence = (lastBlock?.sequence ?? -1) + 1;
|
|
632
|
+
prevSequence = lastBlock?.sequence ?? null;
|
|
633
|
+
prevActorId = lastBlock?.actorId ?? null;
|
|
634
|
+
}
|
|
346
635
|
|
|
347
636
|
const block: Block = {
|
|
348
|
-
feedId:
|
|
349
|
-
actorId: this.
|
|
637
|
+
feedId: msg.feedId,
|
|
638
|
+
actorId: this.#options.localActorId,
|
|
350
639
|
sequence,
|
|
351
|
-
|
|
352
|
-
|
|
640
|
+
prevActorId,
|
|
641
|
+
prevSequence,
|
|
353
642
|
timestamp: Date.now(),
|
|
354
643
|
data: msg.data,
|
|
355
|
-
position: null, //
|
|
644
|
+
position: null, // Assigned by append.
|
|
356
645
|
};
|
|
646
|
+
|
|
357
647
|
blocks.push(block);
|
|
358
648
|
|
|
359
|
-
//
|
|
649
|
+
// Update in-flight tracking.
|
|
650
|
+
currentSeqs.set(key, { sequence, actorId: this.#options.localActorId });
|
|
651
|
+
|
|
652
|
+
// Group by (spaceId, feedNamespace).
|
|
653
|
+
const spaceNamespaceKey = `${msg.spaceId}|${msg.feedNamespace}`;
|
|
654
|
+
if (!blocksBySpaceNamespace.has(spaceNamespaceKey)) {
|
|
655
|
+
blocksBySpaceNamespace.set(spaceNamespaceKey, {
|
|
656
|
+
spaceId: msg.spaceId,
|
|
657
|
+
feedNamespace: msg.feedNamespace,
|
|
658
|
+
blocks: [],
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
blocksBySpaceNamespace.get(spaceNamespaceKey)!.blocks.push(block);
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// 4. Call append once per (spaceId, namespace) batch.
|
|
665
|
+
for (const { spaceId, feedNamespace, blocks } of blocksBySpaceNamespace.values()) {
|
|
360
666
|
yield* this.append({
|
|
361
667
|
requestId: 'local-append',
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
feedId: msg.feedId,
|
|
668
|
+
blocks,
|
|
669
|
+
spaceId,
|
|
670
|
+
feedNamespace,
|
|
366
671
|
});
|
|
367
672
|
}
|
|
673
|
+
|
|
368
674
|
return blocks;
|
|
369
|
-
}),
|
|
675
|
+
}).pipe(Effect.withSpan('FeedStore.appendLocal')),
|
|
370
676
|
);
|
|
677
|
+
|
|
678
|
+
/**
|
|
679
|
+
* Sets positions for existing blocks while preventing conflicting reassignments.
|
|
680
|
+
*/
|
|
681
|
+
setPosition = (request: {
|
|
682
|
+
spaceId: string;
|
|
683
|
+
blocks: (Pick<Block, 'feedId' | 'actorId' | 'sequence' | 'position'> & { feedNamespace: string })[];
|
|
684
|
+
}): Effect.Effect<void, SqlError.SqlError | PositionConflictError, SqlClient.SqlClient> =>
|
|
685
|
+
Effect.gen(this, function* () {
|
|
686
|
+
const sql = yield* SqlClient.SqlClient;
|
|
687
|
+
for (const block of request.blocks) {
|
|
688
|
+
const existing = yield* sql<{ position: number | null }>`
|
|
689
|
+
SELECT position FROM blocks
|
|
690
|
+
WHERE feedPrivateId = (
|
|
691
|
+
SELECT feedPrivateId FROM feeds
|
|
692
|
+
WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
|
|
693
|
+
)
|
|
694
|
+
AND actorId = ${block.actorId} AND sequence = ${block.sequence}
|
|
695
|
+
`;
|
|
696
|
+
const current = existing[0];
|
|
697
|
+
if (current?.position != null && current.position !== block.position) {
|
|
698
|
+
yield* Effect.fail(
|
|
699
|
+
new PositionConflictError({
|
|
700
|
+
feedId: block.feedId,
|
|
701
|
+
actorId: block.actorId,
|
|
702
|
+
sequence: block.sequence,
|
|
703
|
+
currentPosition: current.position,
|
|
704
|
+
requestedPosition: block.position,
|
|
705
|
+
}),
|
|
706
|
+
);
|
|
707
|
+
}
|
|
708
|
+
yield* sql`
|
|
709
|
+
UPDATE blocks SET position = ${block.position}
|
|
710
|
+
WHERE feedPrivateId = (
|
|
711
|
+
SELECT feedPrivateId FROM feeds
|
|
712
|
+
WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
|
|
713
|
+
)
|
|
714
|
+
AND actorId = ${block.actorId} AND sequence = ${block.sequence}
|
|
715
|
+
`;
|
|
716
|
+
}
|
|
717
|
+
}).pipe(Effect.withSpan('FeedStore.setPosition'));
|
|
371
718
|
}
|
|
372
719
|
|
|
373
720
|
const encodeCursor = (token: string, insertionId: number) => FeedCursor.make(`${token}|${insertionId}`);
|