@dxos/feed 0.0.0 → 0.8.4-main.1068cf700f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +7 -2
  2. package/dist/lib/neutral/chunk-UOP2Q6KY.mjs +744 -0
  3. package/dist/lib/neutral/chunk-UOP2Q6KY.mjs.map +7 -0
  4. package/dist/lib/neutral/index.mjs +17 -0
  5. package/dist/lib/neutral/index.mjs.map +7 -0
  6. package/dist/lib/neutral/meta.json +1 -0
  7. package/dist/lib/neutral/testing/index.mjs +173 -0
  8. package/dist/lib/neutral/testing/index.mjs.map +7 -0
  9. package/dist/types/src/errors.d.ts +106 -0
  10. package/dist/types/src/errors.d.ts.map +1 -0
  11. package/dist/types/src/feed-store.d.ts +86 -0
  12. package/dist/types/src/feed-store.d.ts.map +1 -0
  13. package/dist/types/src/feed-store.test.d.ts +2 -0
  14. package/dist/types/src/feed-store.test.d.ts.map +1 -0
  15. package/dist/types/src/index.d.ts +5 -0
  16. package/dist/types/src/index.d.ts.map +1 -0
  17. package/dist/types/src/sync-client.d.ts +53 -0
  18. package/dist/types/src/sync-client.d.ts.map +1 -0
  19. package/dist/types/src/sync-server.d.ts +27 -0
  20. package/dist/types/src/sync-server.d.ts.map +1 -0
  21. package/dist/types/src/sync.test.d.ts +2 -0
  22. package/dist/types/src/sync.test.d.ts.map +1 -0
  23. package/dist/types/src/testing/index.d.ts +2 -0
  24. package/dist/types/src/testing/index.d.ts.map +1 -0
  25. package/dist/types/src/testing/test-builder.d.ts +78 -0
  26. package/dist/types/src/testing/test-builder.d.ts.map +1 -0
  27. package/dist/types/tsconfig.tsbuildinfo +1 -0
  28. package/package.json +20 -13
  29. package/src/errors.ts +38 -0
  30. package/src/feed-store.test.ts +283 -35
  31. package/src/feed-store.ts +479 -132
  32. package/src/index.ts +3 -1
  33. package/src/sync-client.ts +188 -0
  34. package/src/sync-server.ts +92 -0
  35. package/src/sync.test.ts +151 -91
  36. package/src/testing/index.ts +1 -1
  37. package/src/testing/test-builder.ts +223 -0
  38. package/src/protocol.ts +0 -110
package/src/feed-store.ts CHANGED
@@ -7,73 +7,112 @@ import type * as SqlError from '@effect/sql/SqlError';
7
7
  import * as Effect from 'effect/Effect';
8
8
 
9
9
  import { Event } from '@dxos/async';
10
-
11
- import {
12
- type AppendRequest,
13
- type AppendResponse,
14
- type Block,
15
- FeedCursor,
16
- type QueryRequest,
17
- type QueryResponse,
18
- type SubscribeRequest,
19
- type SubscribeResponse,
20
- } from './protocol';
10
+ import { assertArgument } from '@dxos/invariant';
11
+ import { type SpaceId } from '@dxos/keys';
12
+ import { FeedProtocol } from '@dxos/protocols';
13
+ import { SqlTransaction } from '@dxos/sql-sqlite';
14
+
15
+ import { DuplicateBlockLamportTimestampError, DuplicateBlockPositionError, PositionConflictError } from './errors';
16
+
17
+ type AppendRequest = FeedProtocol.AppendRequest;
18
+ type AppendResponse = FeedProtocol.AppendResponse;
19
+ type Block = FeedProtocol.Block;
20
+ const FeedCursor = FeedProtocol.FeedCursor;
21
+ type FeedCursor = FeedProtocol.FeedCursor;
22
+ const isWellKnownNamespace = FeedProtocol.isWellKnownNamespace;
23
+ type QueryRequest = FeedProtocol.QueryRequest;
24
+ type QueryResponse = FeedProtocol.QueryResponse;
25
+ type SubscribeRequest = FeedProtocol.SubscribeRequest;
26
+ type SubscribeResponse = FeedProtocol.SubscribeResponse;
21
27
 
22
28
  export interface FeedStoreOptions {
29
+ /**
30
+ * The actor ID of the local user.
31
+ */
23
32
  localActorId: string;
33
+
34
+ /**
35
+ * Whether to assign positions to appended blocks.
36
+ * Only a single peer (usually the server) can assign positions.
37
+ */
24
38
  assignPositions: boolean;
25
39
  }
26
40
 
41
+ /**
42
+ * Persistent storage for feed metadata, blocks, subscriptions, and sync state.
43
+ *
44
+ */
27
45
  export class FeedStore {
28
- constructor(private readonly _options: FeedStoreOptions) {}
46
+ readonly #options: FeedStoreOptions;
47
+
48
+ constructor(options: FeedStoreOptions) {
49
+ this.#options = options;
50
+ }
29
51
 
52
+ /**
53
+ * Emits after successful block append operations.
54
+ */
30
55
  readonly onNewBlocks = new Event<void>();
31
56
 
32
- migrate = Effect.fn('FeedStore.migrate')(function* () {
33
- const sql = yield* SqlClient.SqlClient;
57
+ /**
58
+ * Creates required feed store tables and indexes if they do not exist.
59
+ */
60
+ migrate = Effect.fn('FeedStore.migrate')(() =>
61
+ Effect.gen(function* () {
62
+ const sql = yield* SqlClient.SqlClient;
34
63
 
35
- // Feeds Table
36
- yield* sql`CREATE TABLE IF NOT EXISTS feeds (
64
+ // Feeds Table
65
+ yield* sql`CREATE TABLE IF NOT EXISTS feeds (
37
66
  feedPrivateId INTEGER PRIMARY KEY AUTOINCREMENT,
38
67
  spaceId TEXT NOT NULL,
39
68
  feedId TEXT NOT NULL,
40
69
  feedNamespace TEXT
41
70
  )`;
42
- yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_feeds_spaceId_feedId ON feeds(spaceId, feedId)`;
71
+ yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_feeds_spaceId_feedId ON feeds(spaceId, feedId)`;
43
72
 
44
- // Blocks Table
45
- yield* sql`CREATE TABLE IF NOT EXISTS blocks (
73
+ // Blocks Table
74
+ yield* sql`CREATE TABLE IF NOT EXISTS blocks (
46
75
  insertionId INTEGER PRIMARY KEY AUTOINCREMENT,
47
76
  feedPrivateId INTEGER NOT NULL,
48
77
  position INTEGER,
49
78
  sequence INTEGER NOT NULL,
50
79
  actorId TEXT NOT NULL,
51
- predSequence INTEGER,
52
- predActorId TEXT,
80
+ prevSequence INTEGER,
81
+ prevActorId TEXT,
53
82
  timestamp INTEGER NOT NULL,
54
83
  data BLOB NOT NULL,
55
84
  FOREIGN KEY(feedPrivateId) REFERENCES feeds(feedPrivateId)
56
85
  )`;
57
- yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_position ON blocks(feedPrivateId, position)`;
58
- yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_sequence_actorId ON blocks(feedPrivateId, sequence, actorId)`;
86
+ yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_position ON blocks(feedPrivateId, position)`;
87
+ yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_sequence_actorId ON blocks(feedPrivateId, sequence, actorId)`;
59
88
 
60
- // Subscriptions Table
61
- yield* sql`CREATE TABLE IF NOT EXISTS subscriptions (
89
+ // Subscriptions Table
90
+ yield* sql`CREATE TABLE IF NOT EXISTS subscriptions (
62
91
  subscriptionId TEXT PRIMARY KEY,
63
92
  expiresAt INTEGER NOT NULL,
64
93
  feedPrivateIds TEXT NOT NULL -- JSON array
65
94
  )`;
66
95
 
67
- // Cursor Tokens Table
68
- yield* sql`CREATE TABLE IF NOT EXISTS cursor_tokens (
96
+ // Cursor Tokens Table
97
+ yield* sql`CREATE TABLE IF NOT EXISTS cursor_tokens (
69
98
  spaceId TEXT PRIMARY KEY,
70
99
  token TEXT NOT NULL
71
100
  )`;
72
- });
73
101
 
74
- // Internal Logic
102
+ // Sync State Table.
103
+ yield* sql`CREATE TABLE IF NOT EXISTS sync_state (
104
+ spaceId TEXT NOT NULL,
105
+ feedNamespace TEXT NOT NULL,
106
+ lastPulledPosition INTEGER NOT NULL DEFAULT -1,
107
+ PRIMARY KEY (spaceId, feedNamespace)
108
+ )`;
109
+ }).pipe(Effect.withSpan('FeedStore.migrate')),
110
+ );
75
111
 
76
- private _ensureFeed = Effect.fn('Feed.ensureFeed')(
112
+ /**
113
+ * Ensures a feed row exists and returns its internal feed ID.
114
+ */
115
+ #ensureFeed = Effect.fn('Feed.ensureFeed')(
77
116
  (
78
117
  spaceId: string,
79
118
  feedId: string,
@@ -91,10 +130,13 @@ export class FeedStore {
91
130
  INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
92
131
  `;
93
132
  return newRows[0].feedPrivateId;
94
- }),
133
+ }).pipe(Effect.withSpan('FeedStore.ensureFeed')),
95
134
  );
96
135
 
97
- private _ensureCursorToken = Effect.fn('Feed.ensureCursorToken')(
136
+ /**
137
+ * Ensures cursor token exists for a space and returns it.
138
+ */
139
+ #ensureCursorToken = Effect.fn('Feed.ensureCursorToken')(
98
140
  (spaceId: string): Effect.Effect<string, SqlError.SqlError, SqlClient.SqlClient> =>
99
141
  Effect.gen(this, function* () {
100
142
  const sql = yield* SqlClient.SqlClient;
@@ -104,11 +146,12 @@ export class FeedStore {
104
146
  const token = crypto.randomUUID().replace(/-/g, '').slice(0, 6);
105
147
  yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
106
148
  return token;
107
- }),
149
+ }).pipe(Effect.withSpan('FeedStore.ensureCursorToken')),
108
150
  );
109
151
 
110
- // RPCs
111
-
152
+ /**
153
+ * Queries feed blocks by feed IDs or subscription with cursor/position pagination.
154
+ */
112
155
  query = Effect.fn('Feed.query')(
113
156
  (request: QueryRequest): Effect.Effect<QueryResponse, SqlError.SqlError, SqlClient.SqlClient> =>
114
157
  Effect.gen(this, function* () {
@@ -121,6 +164,15 @@ export class FeedStore {
121
164
  return yield* Effect.die(new Error('spaceId is required'));
122
165
  }
123
166
 
167
+ if (
168
+ (request.position !== undefined ? 1 : 0) +
169
+ (request.cursor !== undefined ? 1 : 0) +
170
+ (request.unpositionedOnly === true ? 1 : 0) >
171
+ 1
172
+ ) {
173
+ return yield* Effect.die(new Error('Only one of position, cursor, or unpositionedOnly can be used'));
174
+ }
175
+
124
176
  if (request.cursor) {
125
177
  const { token, insertionId } = decodeCursor(request.cursor as FeedCursor);
126
178
  if (!token || insertionId === undefined || isNaN(insertionId)) {
@@ -130,8 +182,8 @@ export class FeedStore {
130
182
  cursorInsertionId = insertionId;
131
183
  }
132
184
 
133
- // Validate Token if cursor used
134
- const validCursorToken = yield* this._ensureCursorToken(request.spaceId);
185
+ // Validate Token if cursor used.
186
+ const validCursorToken = yield* this.#ensureCursorToken(request.spaceId);
135
187
  if (request.cursor && cursorToken !== validCursorToken) {
136
188
  return yield* Effect.die(new Error(`Cursor token mismatch`));
137
189
  }
@@ -150,8 +202,8 @@ export class FeedStore {
150
202
 
151
203
  const position = request.position ?? -1;
152
204
 
153
- // Resolve Subscriptions or FeedIds
154
- if ('subscriptionId' in request.query) {
205
+ // Resolve subscriptions or feed IDs.
206
+ if (request.query && 'subscriptionId' in request.query) {
155
207
  const rows = yield* sql<{ feedPrivateIds: string; expiresAt: number }>`
156
208
  SELECT feedPrivateIds, expiresAt FROM subscriptions WHERE subscriptionId = ${request.query.subscriptionId}
157
209
  `;
@@ -167,50 +219,57 @@ export class FeedStore {
167
219
  }
168
220
  }
169
221
  }
170
- } else if ('feedIds' in request.query) {
171
- feedIds = [...(request.query as any).feedIds];
222
+ } else if (request.query && 'feedIds' in request.query) {
223
+ feedIds = [...request.query.feedIds];
172
224
  } else {
173
225
  feedIds = undefined;
174
226
  }
175
227
 
176
228
  if (feedIds !== undefined && feedIds.length === 0) {
177
- return { requestId: request.requestId, blocks: [], nextCursor: encodeCursor(validCursorToken, -1) };
229
+ return {
230
+ requestId: request.requestId,
231
+ blocks: [],
232
+ nextCursor: encodeCursor(validCursorToken, -1),
233
+ hasMore: false,
234
+ };
178
235
  }
179
236
 
180
- // Fetch Blocks
237
+ // Fetch blocks.
181
238
  const query = sql<Block>`
182
- SELECT blocks.*, feeds.feedId
239
+ SELECT blocks.*, feeds.feedId, feeds.feedNamespace
183
240
  FROM blocks
184
241
  JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
185
242
  WHERE 1=1
186
243
  ${feedIds !== undefined ? sql`AND feeds.feedId IN ${sql.in(feedIds)}` : sql``}
187
244
  ${request.spaceId ? sql`AND feeds.spaceId = ${request.spaceId}` : sql``}
188
- ${
189
- 'feedNamespace' in request.query && request.query.feedNamespace
190
- ? sql`AND feeds.feedNamespace = ${request.query.feedNamespace}`
191
- : sql``
192
- }
245
+ ${sql`AND feeds.feedNamespace = ${request.feedNamespace}`}
193
246
  `;
194
247
 
195
- // Add filter based on cursor or position
248
+ // Add filter based on cursor or position.
196
249
  const filter = request.cursor
197
250
  ? sql`AND blocks.insertionId > ${cursorInsertionId}`
198
- : sql`AND (blocks.position > ${position} OR blocks.position IS NULL)`;
251
+ : request.unpositionedOnly
252
+ ? sql`AND blocks.position IS NULL`
253
+ : sql`AND (blocks.position > ${position} OR blocks.position IS NULL)`;
199
254
 
200
255
  const orderBy = request.cursor
201
256
  ? sql`ORDER BY blocks.insertionId ASC`
202
257
  : sql`ORDER BY blocks.position ASC NULLS LAST`;
203
258
 
259
+ const requestLimit = request.limit;
260
+ const queryLimit = requestLimit != null ? requestLimit + 1 : undefined;
204
261
  const rows = yield* sql<Block>`
205
262
  ${query}
206
263
  ${filter}
207
264
  ${orderBy}
208
- ${request.limit ? sql`LIMIT ${request.limit}` : sql``}
265
+ ${queryLimit != null ? sql`LIMIT ${queryLimit}` : sql``}
209
266
  `;
210
267
 
211
- const blocks = rows.map((row) => ({
268
+ const hasMore = requestLimit != null && rows.length > requestLimit;
269
+ const slice = hasMore ? rows.slice(0, requestLimit) : rows;
270
+ const blocks = slice.map((row) => ({
212
271
  ...row,
213
- // Have to buffer otherwise we get empty Uint8Array.
272
+ // Have to clone buffer otherwise we get empty Uint8Array.
214
273
  data: new Uint8Array(row.data),
215
274
  }));
216
275
 
@@ -222,10 +281,13 @@ export class FeedStore {
222
281
  }
223
282
  }
224
283
 
225
- return { requestId: request.requestId, blocks, nextCursor } satisfies QueryResponse;
226
- }),
284
+ return { requestId: request.requestId, blocks, nextCursor, hasMore } satisfies QueryResponse;
285
+ }).pipe(Effect.withSpan('FeedStore.query')),
227
286
  );
228
287
 
288
+ /**
289
+ * Creates a subscription and stores the resolved internal feed IDs.
290
+ */
229
291
  subscribe = Effect.fn('Feed.subscribe')(
230
292
  (request: SubscribeRequest): Effect.Effect<SubscribeResponse, SqlError.SqlError, SqlClient.SqlClient> =>
231
293
  Effect.gen(this, function* () {
@@ -241,7 +303,7 @@ export class FeedStore {
241
303
 
242
304
  const feedPrivateIds = yield* Effect.forEach(
243
305
  request.feedIds,
244
- (feedId) => this._ensureFeed(request.spaceId!, feedId),
306
+ (feedId) => this.#ensureFeed(request.spaceId!, feedId),
245
307
  { concurrency: 'unbounded' },
246
308
  );
247
309
 
@@ -255,119 +317,404 @@ export class FeedStore {
255
317
  subscriptionId,
256
318
  expiresAt,
257
319
  };
258
- }),
320
+ }).pipe(Effect.withSpan('FeedStore.subscribe')),
259
321
  );
260
322
 
261
- append = Effect.fn('Feed.append')(
262
- (request: AppendRequest): Effect.Effect<AppendResponse, SqlError.SqlError, SqlClient.SqlClient> =>
323
+ /**
324
+ * Get the last pulled position for the given space and namespace.
325
+ * Returns -1 if no sync state exists yet.
326
+ */
327
+ getSyncState = (opts: {
328
+ spaceId: SpaceId;
329
+ feedNamespace: string;
330
+ }): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
331
+ Effect.gen(this, function* () {
332
+ const sql = yield* SqlClient.SqlClient;
333
+ const rows = yield* sql<{ lastPulledPosition: number }>`
334
+ SELECT lastPulledPosition FROM sync_state
335
+ WHERE spaceId = ${opts.spaceId} AND feedNamespace = ${opts.feedNamespace}
336
+ `;
337
+ return rows[0]?.lastPulledPosition ?? -1;
338
+ }).pipe(Effect.withSpan('FeedStore.getSyncState'));
339
+
340
+ /**
341
+ * Update the last pulled position for the given space and namespace.
342
+ */
343
+ setSyncState = (opts: {
344
+ spaceId: SpaceId;
345
+ feedNamespace: string;
346
+ lastPulledPosition: number;
347
+ }): Effect.Effect<void, SqlError.SqlError, SqlClient.SqlClient> =>
348
+ Effect.gen(this, function* () {
349
+ const sql = yield* SqlClient.SqlClient;
350
+ yield* sql`
351
+ INSERT INTO sync_state (spaceId, feedNamespace, lastPulledPosition)
352
+ VALUES (${opts.spaceId}, ${opts.feedNamespace}, ${opts.lastPulledPosition})
353
+ ON CONFLICT (spaceId, feedNamespace) DO UPDATE SET lastPulledPosition = ${opts.lastPulledPosition}
354
+ `;
355
+ }).pipe(Effect.withSpan('FeedStore.setSyncState'));
356
+
357
+ /**
358
+ * Validates that appended blocks do not introduce position or Lamport conflicts.
359
+ */
360
+ #validateAppendBlocks = Effect.fn('FeedStore.validateAppendBlocks')(
361
+ (
362
+ blocks: readonly Block[],
363
+ feedPrivateIds: Map<string, number>,
364
+ ): Effect.Effect<
365
+ void,
366
+ SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
367
+ SqlClient.SqlClient
368
+ > =>
263
369
  Effect.gen(this, function* () {
264
370
  const sql = yield* SqlClient.SqlClient;
265
- const positions: number[] = [];
266
371
 
267
- if (!request.spaceId) {
268
- return yield* Effect.die(new Error('spaceId required for append'));
372
+ // Validate request-level uniqueness before writing.
373
+ const positionFeedIdByKey = new Map<string, string>();
374
+ const lamportFeedIdByKey = new Map<string, string>();
375
+ for (const block of blocks) {
376
+ const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
377
+ if (!this.#options.assignPositions && block.position != null) {
378
+ const positionKey = `${feedPrivateId}|${block.position}`;
379
+ if (positionFeedIdByKey.has(positionKey)) {
380
+ yield* Effect.fail(
381
+ new DuplicateBlockPositionError({
382
+ feedId: block.feedId!,
383
+ position: block.position,
384
+ }),
385
+ );
386
+ }
387
+ positionFeedIdByKey.set(positionKey, block.feedId!);
388
+ }
389
+
390
+ const lamportKey = `${feedPrivateId}|${block.actorId}|${block.sequence}`;
391
+ if (lamportFeedIdByKey.has(lamportKey)) {
392
+ yield* Effect.fail(
393
+ new DuplicateBlockLamportTimestampError({
394
+ feedId: block.feedId!,
395
+ actorId: block.actorId,
396
+ sequence: block.sequence,
397
+ }),
398
+ );
399
+ }
400
+ lamportFeedIdByKey.set(lamportKey, block.feedId!);
401
+ }
402
+
403
+ // Bulk-validate against persisted rows with exact tuple matching.
404
+ const lamportConditions = blocks.map((block) => {
405
+ const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
406
+ return sql`(feedPrivateId = ${feedPrivateId} AND actorId = ${block.actorId} AND sequence = ${block.sequence})`;
407
+ });
408
+ if (lamportConditions.length > 0) {
409
+ const existingLamports = yield* sql<{ feedPrivateId: number; actorId: string; sequence: number }>`
410
+ SELECT feedPrivateId, actorId, sequence
411
+ FROM blocks
412
+ WHERE ${sql.or(lamportConditions)}
413
+ LIMIT 1
414
+ `;
415
+ const lamportConflict = existingLamports[0];
416
+ if (lamportConflict) {
417
+ const key = `${lamportConflict.feedPrivateId}|${lamportConflict.actorId}|${lamportConflict.sequence}`;
418
+ yield* Effect.fail(
419
+ new DuplicateBlockLamportTimestampError({
420
+ feedId: lamportFeedIdByKey.get(key)!,
421
+ actorId: lamportConflict.actorId,
422
+ sequence: lamportConflict.sequence,
423
+ }),
424
+ );
425
+ }
426
+ }
427
+
428
+ if (!this.#options.assignPositions) {
429
+ const positionedBlocks = blocks.filter((block) => block.position != null);
430
+ const positionConditions = positionedBlocks.map((block) => {
431
+ const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
432
+ return sql`(feedPrivateId = ${feedPrivateId} AND position = ${block.position})`;
433
+ });
434
+ if (positionConditions.length > 0) {
435
+ const existingPositions = yield* sql<{ feedPrivateId: number; position: number }>`
436
+ SELECT feedPrivateId, position
437
+ FROM blocks
438
+ WHERE position IS NOT NULL
439
+ AND (${sql.or(positionConditions)})
440
+ LIMIT 1
441
+ `;
442
+ const positionConflict = existingPositions[0];
443
+ if (positionConflict) {
444
+ const key = `${positionConflict.feedPrivateId}|${positionConflict.position}`;
445
+ yield* Effect.fail(
446
+ new DuplicateBlockPositionError({
447
+ feedId: positionFeedIdByKey.get(key)!,
448
+ position: positionConflict.position,
449
+ }),
450
+ );
451
+ }
452
+ }
269
453
  }
454
+ }),
455
+ );
270
456
 
271
- for (const block of request.blocks) {
272
- const feedId = request.feedId ?? block.actorId;
273
- const feedPrivateId = yield* this._ensureFeed(request.spaceId, feedId, request.namespace);
457
+ /**
458
+ * Appends blocks for a space/namespace and optionally assigns global positions.
459
+ */
460
+ append = (
461
+ request: AppendRequest,
462
+ ): Effect.Effect<
463
+ AppendResponse,
464
+ SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
465
+ SqlClient.SqlClient | SqlTransaction.SqlTransaction
466
+ > =>
467
+ Effect.gen(this, function* () {
468
+ if (!request.spaceId) {
469
+ return yield* Effect.die(new Error('spaceId required for append'));
470
+ }
471
+
472
+ assertArgument(
473
+ isWellKnownNamespace(request.feedNamespace),
474
+ 'request.feedNamespace',
475
+ 'specified well-known namespace',
476
+ );
477
+
478
+ // Validate all blocks upfront.
479
+ for (const block of request.blocks) {
480
+ assertArgument(block.feedId, 'block.feedId', 'feedId is required');
481
+ }
482
+
483
+ // Wrap in transaction to ensure atomicity when assigning positions.
484
+ const sqlTransaction = yield* SqlTransaction.SqlTransaction;
485
+ const positions = yield* sqlTransaction.withTransaction(
486
+ Effect.gen(this, function* () {
487
+ const sql = yield* SqlClient.SqlClient;
488
+
489
+ // 1. Collect unique feed IDs and batch #ensureFeed calls.
490
+ const feedKeys = new Map<string, { feedId: string }>();
491
+ for (const block of request.blocks) {
492
+ const key = block.feedId!;
493
+ if (!feedKeys.has(key)) {
494
+ feedKeys.set(key, { feedId: block.feedId! });
495
+ }
496
+ }
274
497
 
275
- let nextPos = null;
276
- if (this._options.assignPositions) {
498
+ const feedPrivateIds = new Map<string, number>();
499
+ yield* Effect.forEach(
500
+ [...feedKeys.entries()],
501
+ ([key, { feedId }]) =>
502
+ Effect.gen(this, function* () {
503
+ const id = yield* this.#ensureFeed(request.spaceId!, feedId, request.feedNamespace);
504
+ feedPrivateIds.set(key, id);
505
+ }),
506
+ { concurrency: 'unbounded' },
507
+ );
508
+
509
+ yield* this.#validateAppendBlocks(request.blocks, feedPrivateIds);
510
+
511
+ // 2. Get max position per namespace ONCE (not per block).
512
+ const maxPositions = new Map<string, number>();
513
+ if (this.#options.assignPositions) {
277
514
  const maxPosResult = yield* sql<{ maxPos: number | null }>`
278
- SELECT MAX(position) as maxPos
279
- FROM blocks
280
- JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
281
- WHERE feeds.spaceId = ${request.spaceId}
282
- `;
283
- nextPos = (maxPosResult[0]?.maxPos ?? -1) + 1;
284
- positions.push(nextPos);
515
+ SELECT MAX(position) as maxPos
516
+ FROM blocks
517
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
518
+ WHERE feeds.spaceId = ${request.spaceId} AND feeds.feedNamespace = ${request.feedNamespace}
519
+ `;
520
+ maxPositions.set(request.feedNamespace, maxPosResult[0]?.maxPos ?? -1);
285
521
  }
286
522
 
287
- yield* sql`
288
- INSERT INTO blocks (
289
- feedPrivateId, position, sequence, actorId,
290
- predSequence, predActorId, timestamp, data
291
- ) VALUES (
292
- ${feedPrivateId}, ${nextPos}, ${block.sequence}, ${block.actorId},
293
- ${block.predSequence}, ${block.predActorId}, ${block.timestamp}, ${block.data}
294
- ) ON CONFLICT DO NOTHING
523
+ // 3. Insert all blocks and compute positions.
524
+ const positions: number[] = [];
525
+ for (const block of request.blocks) {
526
+ const key = block.feedId!;
527
+ const feedPrivateId = feedPrivateIds.get(key)!;
528
+
529
+ let positionToInsert: number | null = null;
530
+ if (this.#options.assignPositions) {
531
+ const currentMax = maxPositions.get(request.feedNamespace)!;
532
+ positionToInsert = currentMax + 1;
533
+ maxPositions.set(request.feedNamespace, positionToInsert); // Increment for next block in same namespace.
534
+ positions.push(positionToInsert);
535
+ } else if (block.position != null) {
536
+ positionToInsert = block.position;
537
+ }
538
+
539
+ yield* sql`
540
+ INSERT INTO blocks (
541
+ feedPrivateId, position, sequence, actorId,
542
+ prevSequence, prevActorId, timestamp, data
543
+ ) VALUES (
544
+ ${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
545
+ ${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
546
+ ) ON CONFLICT DO NOTHING
295
547
  `;
296
- }
548
+ }
297
549
 
298
- this.onNewBlocks.emit();
550
+ return positions;
551
+ }),
552
+ );
299
553
 
300
- return { requestId: request.requestId, positions };
301
- }),
302
- );
554
+ this.onNewBlocks.emit();
303
555
 
556
+ return { requestId: request.requestId, positions };
557
+ }).pipe(Effect.withSpan('FeedStore.append'));
558
+
559
+ /**
560
+ * Creates local blocks with sequential predecessors and appends grouped batches.
561
+ */
304
562
  appendLocal = Effect.fn('Feed.appendLocal')(
305
563
  (
306
564
  messages: { spaceId: string; feedId: string; feedNamespace: string; data: Uint8Array }[],
307
- ): Effect.Effect<Block[], SqlError.SqlError, SqlClient.SqlClient> =>
565
+ ): Effect.Effect<
566
+ Block[],
567
+ SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
568
+ SqlClient.SqlClient | SqlTransaction.SqlTransaction
569
+ > =>
308
570
  Effect.gen(this, function* () {
309
571
  const sql = yield* SqlClient.SqlClient;
310
- const blocks: Block[] = [];
311
-
312
- // Group by spaceId for efficient processing? Or just iterate.
313
- // Assuming small batch.
314
572
 
573
+ // 1. Collect unique feeds and ensure they exist.
574
+ type FeedKey = string; // `${spaceId}|${feedNamespace}|${feedId}`
575
+ const feedKeys = new Map<FeedKey, { spaceId: string; feedId: string; feedNamespace: string }>();
315
576
  for (const msg of messages) {
316
- // Get last block for this feed to determine sequence and predecessor.
317
- // Assumes we are the writer (localActorId).
318
- // If we are not localActorId, we shouldn't be creating blocks this way?
319
- // The user said: "uses local actorId". So feedId in message might be ignored?
320
- // Or message provides feedId?
321
- // User said: "client provides (only data + spaceId, feedId, feedNamespace)".
322
- // If client provides feedId, does it match localActorId?
323
- // If localActorId is fixed for the device, and we are appending to "my" feed, feedId should be localActorId.
324
- // If the client explicitly passes feedId, maybe they want to write to a specific feed (e.g. if they have multiple identities?).
325
- // But spec says "uses local actorId". I will use `this._options.localActorId`.
326
- // But `messages` arg has `feedId`.
327
- // I'll assume `msg.feedId` must match `localActorId` or we just use `localActorId` and ignore `msg.feedId`?
328
- // User: "client provides ... feedId".
329
- // I will use passed feedId but it should probably match localActorId if we are signing? (No signing here yet).
330
- // I'll use passed `feedId`.
331
-
332
- const feedPrivateId = yield* this._ensureFeed(msg.spaceId, msg.feedId, msg.feedNamespace);
333
-
334
- // Get last block to determine sequence.
577
+ const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
578
+ if (!feedKeys.has(key)) {
579
+ feedKeys.set(key, { spaceId: msg.spaceId, feedId: msg.feedId, feedNamespace: msg.feedNamespace });
580
+ }
581
+ }
582
+
583
+ // Batch ensure feeds and get their private IDs.
584
+ const feedPrivateIds = new Map<FeedKey, number>();
585
+ yield* Effect.forEach(
586
+ [...feedKeys.entries()],
587
+ ([key, { spaceId, feedId, feedNamespace }]) =>
588
+ Effect.gen(this, function* () {
589
+ const id = yield* this.#ensureFeed(spaceId, feedId, feedNamespace);
590
+ feedPrivateIds.set(key, id);
591
+ }),
592
+ { concurrency: 'unbounded' },
593
+ );
594
+
595
+ // 2. Get last sequence for each unique feed (batch query).
596
+ const lastSeqs = new Map<FeedKey, { sequence: number; actorId: string } | null>();
597
+ for (const [key] of feedKeys) {
598
+ const feedPrivateId = feedPrivateIds.get(key)!;
335
599
  const lastBlockResult = yield* sql<{ sequence: number; actorId: string }>`
336
- SELECT sequence, actorId FROM blocks
337
- WHERE feedPrivateId = ${feedPrivateId}
338
- ORDER BY sequence DESC
339
- LIMIT 1
340
- `; // This gets checks only THIS feed.
600
+ SELECT sequence, actorId FROM blocks
601
+ WHERE feedPrivateId = ${feedPrivateId}
602
+ ORDER BY sequence DESC
603
+ LIMIT 1
604
+ `;
605
+ lastSeqs.set(key, lastBlockResult[0] ?? null);
606
+ }
607
+
608
+ // 3. Build all blocks with correct sequences.
609
+ // Track in-flight sequences per feed to handle multiple messages to same feed.
610
+ const currentSeqs = new Map<FeedKey, { sequence: number; actorId: string }>();
611
+ const blocks: Block[] = [];
612
+ const blocksBySpaceNamespace = new Map<string, { spaceId: string; feedNamespace: string; blocks: Block[] }>();
341
613
 
342
- const lastBlock = lastBlockResult[0];
343
- const sequence = (lastBlock?.sequence ?? -1) + 1;
344
- const predSequence = lastBlock?.sequence ?? null;
345
- const predActorId = lastBlock?.actorId ?? null;
614
+ for (const msg of messages) {
615
+ const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
616
+
617
+ // Determine predecessor: either from in-flight blocks or from DB.
618
+ let sequence: number;
619
+ let prevSequence: number | null;
620
+ let prevActorId: string | null;
621
+
622
+ const inFlight = currentSeqs.get(key);
623
+ if (inFlight) {
624
+ // We've already added blocks for this feed - continue from last in-flight.
625
+ sequence = inFlight.sequence + 1;
626
+ prevSequence = inFlight.sequence;
627
+ prevActorId = inFlight.actorId;
628
+ } else {
629
+ // First block for this feed - use DB state.
630
+ const lastBlock = lastSeqs.get(key);
631
+ sequence = (lastBlock?.sequence ?? -1) + 1;
632
+ prevSequence = lastBlock?.sequence ?? null;
633
+ prevActorId = lastBlock?.actorId ?? null;
634
+ }
346
635
 
347
636
  const block: Block = {
348
- feedId: undefined,
349
- actorId: this._options.localActorId,
637
+ feedId: msg.feedId,
638
+ actorId: this.#options.localActorId,
350
639
  sequence,
351
- predActorId,
352
- predSequence,
640
+ prevActorId,
641
+ prevSequence,
353
642
  timestamp: Date.now(),
354
643
  data: msg.data,
355
- position: null, // assigned by append
644
+ position: null, // Assigned by append.
356
645
  };
646
+
357
647
  blocks.push(block);
358
648
 
359
- // Call append to persist (and assign position if allowed).
649
+ // Update in-flight tracking.
650
+ currentSeqs.set(key, { sequence, actorId: this.#options.localActorId });
651
+
652
+ // Group by (spaceId, feedNamespace).
653
+ const spaceNamespaceKey = `${msg.spaceId}|${msg.feedNamespace}`;
654
+ if (!blocksBySpaceNamespace.has(spaceNamespaceKey)) {
655
+ blocksBySpaceNamespace.set(spaceNamespaceKey, {
656
+ spaceId: msg.spaceId,
657
+ feedNamespace: msg.feedNamespace,
658
+ blocks: [],
659
+ });
660
+ }
661
+ blocksBySpaceNamespace.get(spaceNamespaceKey)!.blocks.push(block);
662
+ }
663
+
664
+ // 4. Call append once per (spaceId, namespace) batch.
665
+ for (const { spaceId, feedNamespace, blocks } of blocksBySpaceNamespace.values()) {
360
666
  yield* this.append({
361
667
  requestId: 'local-append',
362
- namespace: msg.feedNamespace,
363
- blocks: [block],
364
- spaceId: msg.spaceId,
365
- feedId: msg.feedId,
668
+ blocks,
669
+ spaceId,
670
+ feedNamespace,
366
671
  });
367
672
  }
673
+
368
674
  return blocks;
369
- }),
675
+ }).pipe(Effect.withSpan('FeedStore.appendLocal')),
370
676
  );
677
+
678
+ /**
679
+ * Sets positions for existing blocks while preventing conflicting reassignments.
680
+ */
681
+ setPosition = (request: {
682
+ spaceId: string;
683
+ blocks: (Pick<Block, 'feedId' | 'actorId' | 'sequence' | 'position'> & { feedNamespace: string })[];
684
+ }): Effect.Effect<void, SqlError.SqlError | PositionConflictError, SqlClient.SqlClient> =>
685
+ Effect.gen(this, function* () {
686
+ const sql = yield* SqlClient.SqlClient;
687
+ for (const block of request.blocks) {
688
+ const existing = yield* sql<{ position: number | null }>`
689
+ SELECT position FROM blocks
690
+ WHERE feedPrivateId = (
691
+ SELECT feedPrivateId FROM feeds
692
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
693
+ )
694
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
695
+ `;
696
+ const current = existing[0];
697
+ if (current?.position != null && current.position !== block.position) {
698
+ yield* Effect.fail(
699
+ new PositionConflictError({
700
+ feedId: block.feedId,
701
+ actorId: block.actorId,
702
+ sequence: block.sequence,
703
+ currentPosition: current.position,
704
+ requestedPosition: block.position,
705
+ }),
706
+ );
707
+ }
708
+ yield* sql`
709
+ UPDATE blocks SET position = ${block.position}
710
+ WHERE feedPrivateId = (
711
+ SELECT feedPrivateId FROM feeds
712
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
713
+ )
714
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
715
+ `;
716
+ }
717
+ }).pipe(Effect.withSpan('FeedStore.setPosition'));
371
718
  }
372
719
 
373
720
  const encodeCursor = (token: string, insertionId: number) => FeedCursor.make(`${token}|${insertionId}`);