@dxos/feed 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1009 @@
1
+ import * as SqlClient from "@effect/sql/SqlClient";
2
+ import * as EffectContext from "effect/Context";
3
+ import * as Effect from "effect/Effect";
4
+ import { Event } from "@dxos/async";
5
+ import { assertArgument } from "@dxos/invariant";
6
+ import { FeedProtocol } from "@dxos/protocols";
7
+ import { SqlTransaction } from "@dxos/sql-sqlite";
8
+ import { BaseError } from "@dxos/errors";
9
+ import { log } from "@dxos/log";
10
+ import * as Array from "effect/Array";
11
+ import * as Deferred from "effect/Deferred";
12
+ import { ContextDisposedError } from "@dxos/context";
13
+ //#region src/errors.ts
14
+ var SyncRpcTimeoutError = class extends BaseError.extend("SyncRpcTimeoutError") {
15
+ constructor(args) {
16
+ super({
17
+ message: `Feed sync ${args.rpcTag} timed out after ${args.timeoutMs}ms (spaceId=${args.spaceId} feedNamespace=${args.feedNamespace} requestId=${args.requestId}).`,
18
+ context: args
19
+ });
20
+ }
21
+ };
22
+ var PositionConflictError = class extends BaseError.extend("PositionConflictError") {
23
+ constructor(args) {
24
+ super({
25
+ message: `Block already has position ${args.currentPosition}, cannot set to ${args.requestedPosition} (feedId=${args.feedId} actorId=${args.actorId} sequence=${args.sequence}).`,
26
+ context: args
27
+ });
28
+ }
29
+ };
30
+ //#endregion
31
+ //#region src/feed-store.ts
32
+ var FeedCursor = FeedProtocol.FeedCursor;
33
+ var isWellKnownNamespace = FeedProtocol.isWellKnownNamespace;
34
+ /**
35
+ * Effect service tag for {@link FeedStore}.
36
+ */
37
+ var FeedStoreService = class extends EffectContext.Tag("@dxos/feed/FeedStore")() {};
38
+ /**
39
+ * Persistent storage for feed metadata, blocks, subscriptions, and sync state.
40
+ *
41
+ */
42
+ var FeedStore = class {
43
+ #options;
44
+ constructor(options) {
45
+ this.#options = options;
46
+ }
47
+ /**
48
+ * Emits after successful block append operations.
49
+ */
50
+ onNewBlocks = new Event();
51
+ /**
52
+ * Creates required feed store tables and indexes if they do not exist.
53
+ */
54
+ migrate = Effect.fn("FeedStore.migrate")(() => Effect.gen(function* () {
55
+ const sql = yield* SqlClient.SqlClient;
56
+ yield* sql`CREATE TABLE IF NOT EXISTS feeds (
57
+ feedPrivateId INTEGER PRIMARY KEY AUTOINCREMENT,
58
+ spaceId TEXT NOT NULL,
59
+ feedId TEXT NOT NULL,
60
+ feedNamespace TEXT
61
+ )`;
62
+ yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_feeds_spaceId_feedId ON feeds(spaceId, feedId)`;
63
+ yield* sql`CREATE TABLE IF NOT EXISTS blocks (
64
+ insertionId INTEGER PRIMARY KEY AUTOINCREMENT,
65
+ feedPrivateId INTEGER NOT NULL,
66
+ position INTEGER,
67
+ sequence INTEGER NOT NULL,
68
+ actorId TEXT NOT NULL,
69
+ prevSequence INTEGER,
70
+ prevActorId TEXT,
71
+ timestamp INTEGER NOT NULL,
72
+ data BLOB NOT NULL,
73
+ FOREIGN KEY(feedPrivateId) REFERENCES feeds(feedPrivateId)
74
+ )`;
75
+ yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_position ON blocks(feedPrivateId, position)`;
76
+ yield* sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_blocks_feedPrivateId_sequence_actorId ON blocks(feedPrivateId, sequence, actorId)`;
77
+ yield* sql`CREATE TABLE IF NOT EXISTS subscriptions (
78
+ subscriptionId TEXT PRIMARY KEY,
79
+ expiresAt INTEGER NOT NULL,
80
+ feedPrivateIds TEXT NOT NULL -- JSON array
81
+ )`;
82
+ yield* sql`CREATE TABLE IF NOT EXISTS cursor_tokens (
83
+ spaceId TEXT PRIMARY KEY,
84
+ token TEXT NOT NULL
85
+ )`;
86
+ yield* sql`CREATE TABLE IF NOT EXISTS sync_state (
87
+ spaceId TEXT NOT NULL,
88
+ feedNamespace TEXT NOT NULL,
89
+ lastPulledPosition INTEGER NOT NULL DEFAULT -1,
90
+ PRIMARY KEY (spaceId, feedNamespace)
91
+ )`;
92
+ }).pipe(Effect.withSpan("FeedStore.migrate")));
93
+ /**
94
+ * Ensures a feed row exists and returns its internal feed ID.
95
+ */
96
+ #ensureFeed = Effect.fn("Feed.ensureFeed")((spaceId, feedId, namespace) => Effect.gen(this, function* () {
97
+ const sql = yield* SqlClient.SqlClient;
98
+ const rows = yield* sql`
99
+ SELECT feedPrivateId FROM feeds WHERE spaceId = ${spaceId} AND feedId = ${feedId}
100
+ `;
101
+ if (rows.length > 0) return rows[0].feedPrivateId;
102
+ return (yield* sql`
103
+ INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
104
+ `)[0].feedPrivateId;
105
+ }).pipe(Effect.withSpan("FeedStore.ensureFeed")));
106
+ /**
107
+ * Ensures cursor token exists for a space and returns it.
108
+ */
109
+ #ensureCursorToken = Effect.fn("Feed.ensureCursorToken")((spaceId) => Effect.gen(this, function* () {
110
+ const sql = yield* SqlClient.SqlClient;
111
+ const rows = yield* sql`SELECT token FROM cursor_tokens WHERE spaceId = ${spaceId}`;
112
+ if (rows.length > 0) return rows[0].token;
113
+ const token = crypto.randomUUID().replace(/-/g, "").slice(0, 6);
114
+ yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
115
+ return token;
116
+ }).pipe(Effect.withSpan("FeedStore.ensureCursorToken")));
117
+ /**
118
+ * Queries feed blocks by feed IDs or subscription with cursor/position pagination.
119
+ */
120
+ query = Effect.fn("Feed.query")((request) => Effect.gen(this, function* () {
121
+ const sql = yield* SqlClient.SqlClient;
122
+ let feedIds = [];
123
+ let cursorInsertionId = -1;
124
+ let cursorToken;
125
+ if (!request.spaceId) return yield* Effect.die(/* @__PURE__ */ new Error("spaceId is required"));
126
+ if ((request.position !== void 0 ? 1 : 0) + (request.cursor !== void 0 ? 1 : 0) + (request.unpositionedOnly === true ? 1 : 0) > 1) return yield* Effect.die(/* @__PURE__ */ new Error("Only one of position, cursor, or unpositionedOnly can be used"));
127
+ if (request.cursor) {
128
+ const { token, insertionId } = decodeCursor(request.cursor);
129
+ if (!token || insertionId === void 0 || isNaN(insertionId)) return yield* Effect.die(/* @__PURE__ */ new Error(`Invalid cursor format`));
130
+ cursorToken = token;
131
+ cursorInsertionId = insertionId;
132
+ }
133
+ const validCursorToken = yield* this.#ensureCursorToken(request.spaceId);
134
+ if (request.cursor && cursorToken !== validCursorToken) return yield* Effect.die(/* @__PURE__ */ new Error(`Cursor token mismatch`));
135
+ const position = request.position ?? -1;
136
+ if (request.query && "subscriptionId" in request.query) {
137
+ const rows = yield* sql`
138
+ SELECT feedPrivateIds, expiresAt FROM subscriptions WHERE subscriptionId = ${request.query.subscriptionId}
139
+ `;
140
+ if (rows.length > 0) {
141
+ const { feedPrivateIds, expiresAt } = rows[0];
142
+ if (Date.now() <= expiresAt) {
143
+ const privateIds = JSON.parse(feedPrivateIds);
144
+ if (privateIds.length > 0) feedIds = (yield* sql`
145
+ SELECT feedId FROM feeds WHERE feedPrivateId IN ${sql.in(privateIds)}
146
+ `).map((r) => r.feedId);
147
+ }
148
+ }
149
+ } else if (request.query && "feedIds" in request.query) feedIds = [...request.query.feedIds];
150
+ else feedIds = void 0;
151
+ if (feedIds !== void 0 && feedIds.length === 0) return {
152
+ requestId: request.requestId,
153
+ blocks: [],
154
+ nextCursor: encodeCursor(validCursorToken, -1),
155
+ hasMore: false
156
+ };
157
+ const query = sql`
158
+ SELECT blocks.*, feeds.feedId, feeds.feedNamespace
159
+ FROM blocks
160
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
161
+ WHERE 1=1
162
+ ${feedIds !== void 0 ? sql`AND feeds.feedId IN ${sql.in(feedIds)}` : sql``}
163
+ ${request.spaceId ? sql`AND feeds.spaceId = ${request.spaceId}` : sql``}
164
+ ${sql`AND feeds.feedNamespace = ${request.feedNamespace}`}
165
+ `;
166
+ const filter = request.cursor ? sql`AND blocks.insertionId > ${cursorInsertionId}` : request.unpositionedOnly ? sql`AND blocks.position IS NULL` : sql`AND (blocks.position > ${position} OR blocks.position IS NULL)`;
167
+ const orderBy = request.cursor ? sql`ORDER BY blocks.insertionId ASC` : sql`ORDER BY blocks.position ASC NULLS LAST`;
168
+ const requestLimit = request.limit;
169
+ const queryLimit = requestLimit != null ? requestLimit + 1 : void 0;
170
+ const rows = yield* sql`
171
+ ${query}
172
+ ${filter}
173
+ ${orderBy}
174
+ ${queryLimit != null ? sql`LIMIT ${queryLimit}` : sql``}
175
+ `;
176
+ const hasMore = requestLimit != null && rows.length > requestLimit;
177
+ const blocks = (hasMore ? rows.slice(0, requestLimit) : rows).map((row) => ({
178
+ ...row,
179
+ data: new Uint8Array(row.data)
180
+ }));
181
+ let nextCursor = request.cursor ?? encodeCursor(validCursorToken, -1);
182
+ if (blocks.length > 0 && request.spaceId) {
183
+ const lastBlock = blocks[blocks.length - 1];
184
+ if (lastBlock.insertionId !== void 0) nextCursor = encodeCursor(validCursorToken, lastBlock.insertionId);
185
+ }
186
+ return {
187
+ requestId: request.requestId,
188
+ blocks,
189
+ nextCursor,
190
+ hasMore
191
+ };
192
+ }).pipe(Effect.withSpan("FeedStore.query")));
193
+ /**
194
+ * Creates a subscription and stores the resolved internal feed IDs.
195
+ */
196
+ subscribe = Effect.fn("Feed.subscribe")((request) => Effect.gen(this, function* () {
197
+ const sql = yield* SqlClient.SqlClient;
198
+ const ttl = 3600 * 1e3;
199
+ const subscriptionId = crypto.randomUUID();
200
+ const expiresAt = Date.now() + ttl;
201
+ if (!request.spaceId) return yield* Effect.die(/* @__PURE__ */ new Error("spaceId required for subscribe"));
202
+ const feedPrivateIds = yield* Effect.forEach(request.feedIds, (feedId) => this.#ensureFeed(request.spaceId, feedId), { concurrency: "unbounded" });
203
+ yield* sql`
204
+ INSERT INTO subscriptions (subscriptionId, expiresAt, feedPrivateIds)
205
+ VALUES (${subscriptionId}, ${expiresAt}, ${JSON.stringify(feedPrivateIds)})
206
+ `;
207
+ return {
208
+ requestId: request.requestId,
209
+ subscriptionId,
210
+ expiresAt
211
+ };
212
+ }).pipe(Effect.withSpan("FeedStore.subscribe")));
213
+ /**
214
+ * Get the last pulled position for the given space and namespace.
215
+ * Returns -1 if no sync state exists yet.
216
+ */
217
+ getSyncState = (opts) => Effect.gen(this, function* () {
218
+ return (yield* (yield* SqlClient.SqlClient)`
219
+ SELECT lastPulledPosition FROM sync_state
220
+ WHERE spaceId = ${opts.spaceId} AND feedNamespace = ${opts.feedNamespace}
221
+ `)[0]?.lastPulledPosition ?? -1;
222
+ }).pipe(Effect.withSpan("FeedStore.getSyncState"));
223
+ /**
224
+ * Update the last pulled position for the given space and namespace.
225
+ */
226
+ setSyncState = (opts) => Effect.gen(this, function* () {
227
+ yield* (yield* SqlClient.SqlClient)`
228
+ INSERT INTO sync_state (spaceId, feedNamespace, lastPulledPosition)
229
+ VALUES (${opts.spaceId}, ${opts.feedNamespace}, ${opts.lastPulledPosition})
230
+ ON CONFLICT (spaceId, feedNamespace) DO UPDATE SET lastPulledPosition = ${opts.lastPulledPosition}
231
+ `;
232
+ }).pipe(Effect.withSpan("FeedStore.setSyncState"));
233
+ /**
234
+ * Returns the number of blocks pending push (no global position yet) in a space/namespace.
235
+ */
236
+ countUnpositionedBlocks = (opts) => Effect.gen(this, function* () {
237
+ return (yield* (yield* SqlClient.SqlClient)`
238
+ SELECT COUNT(*) AS count
239
+ FROM blocks
240
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
241
+ WHERE feeds.spaceId = ${opts.spaceId}
242
+ AND feeds.feedNamespace = ${opts.feedNamespace}
243
+ AND blocks.position IS NULL
244
+ `)[0]?.count ?? 0;
245
+ }).pipe(Effect.withSpan("FeedStore.countUnpositionedBlocks"));
246
+ /**
247
+ * Returns the total number of blocks stored locally for a space/namespace.
248
+ */
249
+ countNamespaceBlocks = (opts) => Effect.gen(this, function* () {
250
+ return (yield* (yield* SqlClient.SqlClient)`
251
+ SELECT COUNT(*) AS count
252
+ FROM blocks
253
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
254
+ WHERE feeds.spaceId = ${opts.spaceId}
255
+ AND feeds.feedNamespace = ${opts.feedNamespace}
256
+ `)[0]?.count ?? 0;
257
+ }).pipe(Effect.withSpan("FeedStore.countNamespaceBlocks"));
258
+ /**
259
+ * Returns the number of stored blocks for a single feed in a space/namespace.
260
+ * Intended as a low-level primitive for callers (for example Cloudflare Worker code)
261
+ * that need to make retention decisions under constrained storage resources.
262
+ */
263
+ countBlocks = (opts) => Effect.gen(this, function* () {
264
+ return (yield* (yield* SqlClient.SqlClient)`
265
+ SELECT COUNT(*) AS count
266
+ FROM blocks
267
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
268
+ WHERE feeds.spaceId = ${opts.spaceId}
269
+ AND feeds.feedNamespace = ${opts.feedNamespace}
270
+ AND feeds.feedId = ${opts.feedId}
271
+ `)[0]?.count ?? 0;
272
+ }).pipe(Effect.withSpan("FeedStore.countBlocks"));
273
+ /**
274
+ * Deletes the oldest blocks for a single feed in a space/namespace.
275
+ * This API intentionally does not enforce any retention policy (such as max size);
276
+ * callers decide when and how much to prune, which is useful for constrained
277
+ * environments like Cloudflare Workers.
278
+ *
279
+ * @returns Number of deleted rows.
280
+ */
281
+ deleteOldestBlocks = (opts) => Effect.gen(this, function* () {
282
+ const sql = yield* SqlClient.SqlClient;
283
+ if (opts.count <= 0) return 0;
284
+ return (yield* sql`
285
+ DELETE FROM blocks
286
+ WHERE insertionId IN (
287
+ SELECT blocks.insertionId
288
+ FROM blocks
289
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
290
+ WHERE feeds.spaceId = ${opts.spaceId}
291
+ AND feeds.feedNamespace = ${opts.feedNamespace}
292
+ AND feeds.feedId = ${opts.feedId}
293
+ ORDER BY blocks.insertionId ASC
294
+ LIMIT ${opts.count}
295
+ )
296
+ RETURNING insertionId
297
+ `).length;
298
+ }).pipe(Effect.withSpan("FeedStore.deleteOldestBlocks"));
299
+ /**
300
+ * Appends blocks for a space/namespace and optionally assigns global positions.
301
+ */
302
+ append = (request) => Effect.gen(this, function* () {
303
+ if (!request.spaceId) return yield* Effect.die(/* @__PURE__ */ new Error("spaceId required for append"));
304
+ assertArgument(isWellKnownNamespace(request.feedNamespace), "request.feedNamespace", "specified well-known namespace");
305
+ for (const block of request.blocks) assertArgument(block.feedId, "block.feedId", "feedId is required");
306
+ const positions = yield* (yield* SqlTransaction.SqlTransaction).withTransaction(Effect.gen(this, function* () {
307
+ const sql = yield* SqlClient.SqlClient;
308
+ const feedKeys = /* @__PURE__ */ new Map();
309
+ for (const block of request.blocks) {
310
+ const key = block.feedId;
311
+ if (!feedKeys.has(key)) feedKeys.set(key, { feedId: block.feedId });
312
+ }
313
+ const feedPrivateIds = /* @__PURE__ */ new Map();
314
+ yield* Effect.forEach([...feedKeys.entries()], ([key, { feedId }]) => Effect.gen(this, function* () {
315
+ const id = yield* this.#ensureFeed(request.spaceId, feedId, request.feedNamespace);
316
+ feedPrivateIds.set(key, id);
317
+ }), { concurrency: "unbounded" });
318
+ const maxPositions = /* @__PURE__ */ new Map();
319
+ if (this.#options.assignPositions) {
320
+ const maxPosResult = yield* sql`
321
+ SELECT MAX(position) as maxPos
322
+ FROM blocks
323
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
324
+ WHERE feeds.spaceId = ${request.spaceId} AND feeds.feedNamespace = ${request.feedNamespace}
325
+ `;
326
+ maxPositions.set(request.feedNamespace, maxPosResult[0]?.maxPos ?? -1);
327
+ }
328
+ const positions = [];
329
+ for (const block of request.blocks) {
330
+ const key = block.feedId;
331
+ const feedPrivateId = feedPrivateIds.get(key);
332
+ let positionToInsert = null;
333
+ if (this.#options.assignPositions) positionToInsert = maxPositions.get(request.feedNamespace) + 1;
334
+ else if (block.position != null) positionToInsert = block.position;
335
+ const inserted = yield* sql`
336
+ INSERT INTO blocks (
337
+ feedPrivateId, position, sequence, actorId,
338
+ prevSequence, prevActorId, timestamp, data
339
+ ) VALUES (
340
+ ${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
341
+ ${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
342
+ )
343
+ ON CONFLICT(feedPrivateId, sequence, actorId) DO NOTHING
344
+ RETURNING position
345
+ `;
346
+ if (!this.#options.assignPositions) continue;
347
+ if (inserted.length > 0) {
348
+ positions.push(positionToInsert);
349
+ maxPositions.set(request.feedNamespace, positionToInsert);
350
+ continue;
351
+ }
352
+ const existingPosition = (yield* sql`
353
+ SELECT position FROM blocks
354
+ WHERE feedPrivateId = ${feedPrivateId}
355
+ AND actorId = ${block.actorId}
356
+ AND sequence = ${block.sequence}
357
+ `)[0]?.position;
358
+ if (existingPosition != null) {
359
+ positions.push(existingPosition);
360
+ continue;
361
+ }
362
+ yield* sql`
363
+ UPDATE blocks SET position = ${positionToInsert}
364
+ WHERE feedPrivateId = ${feedPrivateId}
365
+ AND actorId = ${block.actorId}
366
+ AND sequence = ${block.sequence}
367
+ `;
368
+ positions.push(positionToInsert);
369
+ maxPositions.set(request.feedNamespace, positionToInsert);
370
+ }
371
+ return positions;
372
+ }));
373
+ this.onNewBlocks.emit();
374
+ return {
375
+ requestId: request.requestId,
376
+ positions
377
+ };
378
+ }).pipe(Effect.withSpan("FeedStore.append"));
379
+ /**
380
+ * Creates local blocks with sequential predecessors and appends grouped batches.
381
+ *
382
+ * A block whose object id is later superseded by a newer same-id block (a live feed object's
383
+ * `Obj.update`, persisted as a whole-object re-append) is never reclaimed — the index collapses
384
+ * reads to the latest block by id, but old blocks stay on disk indefinitely.
385
+ * TODO(wittjosiah): Add compaction/retention driven by `Feed.RetentionOptions`.
386
+ */
387
+ appendLocal = Effect.fn("Feed.appendLocal")((messages) => Effect.gen(this, function* () {
388
+ const sql = yield* SqlClient.SqlClient;
389
+ const feedKeys = /* @__PURE__ */ new Map();
390
+ for (const msg of messages) {
391
+ const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
392
+ if (!feedKeys.has(key)) feedKeys.set(key, {
393
+ spaceId: msg.spaceId,
394
+ feedId: msg.feedId,
395
+ feedNamespace: msg.feedNamespace
396
+ });
397
+ }
398
+ const feedPrivateIds = /* @__PURE__ */ new Map();
399
+ yield* Effect.forEach([...feedKeys.entries()], ([key, { spaceId, feedId, feedNamespace }]) => Effect.gen(this, function* () {
400
+ const id = yield* this.#ensureFeed(spaceId, feedId, feedNamespace);
401
+ feedPrivateIds.set(key, id);
402
+ }), { concurrency: "unbounded" });
403
+ const lastSeqs = /* @__PURE__ */ new Map();
404
+ for (const [key] of feedKeys) {
405
+ const lastBlockResult = yield* sql`
406
+ SELECT sequence, actorId FROM blocks
407
+ WHERE feedPrivateId = ${feedPrivateIds.get(key)}
408
+ ORDER BY sequence DESC
409
+ LIMIT 1
410
+ `;
411
+ lastSeqs.set(key, lastBlockResult[0] ?? null);
412
+ }
413
+ const currentSeqs = /* @__PURE__ */ new Map();
414
+ const blocks = [];
415
+ const blocksBySpaceNamespace = /* @__PURE__ */ new Map();
416
+ for (const msg of messages) {
417
+ const key = `${msg.spaceId}|${msg.feedNamespace}|${msg.feedId}`;
418
+ let sequence;
419
+ let prevSequence;
420
+ let prevActorId;
421
+ const inFlight = currentSeqs.get(key);
422
+ if (inFlight) {
423
+ sequence = inFlight.sequence + 1;
424
+ prevSequence = inFlight.sequence;
425
+ prevActorId = inFlight.actorId;
426
+ } else {
427
+ const lastBlock = lastSeqs.get(key);
428
+ sequence = (lastBlock?.sequence ?? -1) + 1;
429
+ prevSequence = lastBlock?.sequence ?? null;
430
+ prevActorId = lastBlock?.actorId ?? null;
431
+ }
432
+ const block = {
433
+ feedId: msg.feedId,
434
+ actorId: this.#options.localActorId,
435
+ sequence,
436
+ prevActorId,
437
+ prevSequence,
438
+ timestamp: Date.now(),
439
+ data: msg.data,
440
+ position: null
441
+ };
442
+ blocks.push(block);
443
+ currentSeqs.set(key, {
444
+ sequence,
445
+ actorId: this.#options.localActorId
446
+ });
447
+ const spaceNamespaceKey = `${msg.spaceId}|${msg.feedNamespace}`;
448
+ if (!blocksBySpaceNamespace.has(spaceNamespaceKey)) blocksBySpaceNamespace.set(spaceNamespaceKey, {
449
+ spaceId: msg.spaceId,
450
+ feedNamespace: msg.feedNamespace,
451
+ blocks: []
452
+ });
453
+ blocksBySpaceNamespace.get(spaceNamespaceKey).blocks.push(block);
454
+ }
455
+ const positionByBlock = /* @__PURE__ */ new Map();
456
+ for (const { spaceId, feedNamespace, blocks: batchBlocks } of blocksBySpaceNamespace.values()) {
457
+ const { positions } = yield* this.append({
458
+ requestId: "local-append",
459
+ blocks: batchBlocks,
460
+ spaceId,
461
+ feedNamespace
462
+ });
463
+ for (let i = 0; i < batchBlocks.length; i++) positionByBlock.set(batchBlocks[i], positions[i] ?? null);
464
+ }
465
+ return blocks.map((block) => ({
466
+ ...block,
467
+ position: positionByBlock.get(block) ?? block.position
468
+ }));
469
+ }).pipe(Effect.withSpan("FeedStore.appendLocal")));
470
+ /**
471
+ * Sets positions for existing blocks while preventing conflicting reassignments.
472
+ */
473
+ setPosition = (request) => Effect.gen(this, function* () {
474
+ const sql = yield* SqlClient.SqlClient;
475
+ for (const block of request.blocks) {
476
+ if ((yield* sql`
477
+ UPDATE blocks SET position = ${block.position}
478
+ WHERE feedPrivateId = (
479
+ SELECT feedPrivateId FROM feeds
480
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
481
+ )
482
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
483
+ AND (position IS NULL OR position = ${block.position})
484
+ RETURNING position
485
+ `).length > 0) continue;
486
+ const current = (yield* sql`
487
+ SELECT position FROM blocks
488
+ WHERE feedPrivateId = (
489
+ SELECT feedPrivateId FROM feeds
490
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
491
+ )
492
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
493
+ `)[0];
494
+ if (current?.position != null && current.position !== block.position) yield* Effect.fail(new PositionConflictError({
495
+ feedId: block.feedId,
496
+ actorId: block.actorId,
497
+ sequence: block.sequence,
498
+ currentPosition: current.position,
499
+ requestedPosition: block.position
500
+ }));
501
+ }
502
+ }).pipe(Effect.withSpan("FeedStore.setPosition"));
503
+ /**
504
+ * Gets all feeds and their blocks for a space, organized by feed ID and namespace.
505
+ * Used for space archive export.
506
+ */
507
+ getAllFeedsForSpace = (opts) => Effect.gen(this, function* () {
508
+ const sql = yield* SqlClient.SqlClient;
509
+ const feeds = yield* sql`
510
+ SELECT DISTINCT feedId, feedNamespace FROM feeds WHERE spaceId = ${opts.spaceId}
511
+ `;
512
+ const result = [];
513
+ for (const feed of feeds) {
514
+ const blocks = yield* sql`
515
+ SELECT blocks.*, feeds.feedId, feeds.feedNamespace
516
+ FROM blocks
517
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
518
+ WHERE feeds.spaceId = ${opts.spaceId}
519
+ AND feeds.feedId = ${feed.feedId}
520
+ AND feeds.feedNamespace = ${feed.feedNamespace}
521
+ ORDER BY blocks.sequence ASC
522
+ `;
523
+ result.push({
524
+ feedId: feed.feedId,
525
+ feedNamespace: feed.feedNamespace,
526
+ blocks: blocks.map((row) => ({
527
+ ...row,
528
+ data: new Uint8Array(row.data)
529
+ }))
530
+ });
531
+ }
532
+ return result;
533
+ }).pipe(Effect.withSpan("FeedStore.getAllFeedsForSpace"));
534
+ };
535
+ var encodeCursor = (token, insertionId) => FeedCursor.make(`${token}|${insertionId}`);
536
+ var decodeCursor = (cursor) => {
537
+ const [token, insertionId] = cursor.split("|");
538
+ return {
539
+ token,
540
+ insertionId: Number(insertionId)
541
+ };
542
+ };
543
+ //#endregion
544
+ //#region src/sync-server.ts
545
+ var __dxlog_file$1 = "/__w/dxos/dxos/packages/core/echo/feed/src/sync-server.ts";
546
+ var inboundSummary = (message) => ({
547
+ tag: message._tag,
548
+ senderPeerId: message.senderPeerId,
549
+ recipientPeerId: message.recipientPeerId,
550
+ requestId: "requestId" in message ? message.requestId : void 0,
551
+ spaceId: "spaceId" in message ? message.spaceId : void 0,
552
+ feedNamespace: "feedNamespace" in message ? message.feedNamespace : void 0,
553
+ blockCount: "blocks" in message ? message.blocks.length : void 0,
554
+ position: "position" in message ? message.position : void 0,
555
+ limit: "limit" in message ? message.limit : void 0
556
+ });
557
+ /**
558
+ * Server-side sync: receives protocol messages, runs query/append against the store, sends responses.
559
+ * Reusable for many peers: sets senderPeerId = peerId and recipientPeerId = incoming message's senderPeerId on replies.
560
+ */
561
+ var SyncServer = class {
562
+ #peerId;
563
+ #feedStore;
564
+ #sendMessage;
565
+ constructor(options) {
566
+ this.#peerId = options.peerId;
567
+ this.#feedStore = options.feedStore;
568
+ this.#sendMessage = options.sendMessage;
569
+ }
570
+ /**
571
+ * Receive a message from a client. Handles QueryRequest and AppendRequest; sends response via sendMessage with correct peer ids.
572
+ */
573
+ handleMessage(ctx, message) {
574
+ const self = this;
575
+ log("feed sync server received message", {
576
+ peerId: self.#peerId,
577
+ ...inboundSummary(message)
578
+ }, {
579
+ "~LogMeta": "~LogMeta",
580
+ F: __dxlog_file$1,
581
+ L: 63,
582
+ S: this
583
+ });
584
+ const recipientPeerId = message.senderPeerId;
585
+ const withPeerIds = (payload) => ({
586
+ ...payload,
587
+ senderPeerId: self.#peerId,
588
+ recipientPeerId
589
+ });
590
+ switch (message._tag) {
591
+ case "QueryRequest": {
592
+ const req = message;
593
+ return Effect.gen(function* () {
594
+ const response = yield* self.#feedStore.query(req);
595
+ log("feed sync server query completed", {
596
+ peerId: self.#peerId,
597
+ recipientPeerId,
598
+ requestId: req.requestId,
599
+ spaceId: req.spaceId,
600
+ feedNamespace: req.feedNamespace,
601
+ returnedBlocks: response.blocks.length,
602
+ hasMore: response.hasMore
603
+ }, {
604
+ "~LogMeta": "~LogMeta",
605
+ F: __dxlog_file$1,
606
+ L: 79,
607
+ S: this
608
+ });
609
+ yield* self.#sendMessage(ctx, withPeerIds({
610
+ _tag: "QueryResponse",
611
+ ...response
612
+ }));
613
+ }).pipe(Effect.catchAll((err) => {
614
+ log("feed sync server query failed", {
615
+ peerId: self.#peerId,
616
+ recipientPeerId,
617
+ requestId: req.requestId,
618
+ spaceId: req.spaceId,
619
+ feedNamespace: req.feedNamespace,
620
+ cause: err instanceof Error ? err.message : String(err)
621
+ }, {
622
+ "~LogMeta": "~LogMeta",
623
+ F: __dxlog_file$1,
624
+ L: 91,
625
+ S: this
626
+ });
627
+ return self.#sendMessage(ctx, withPeerIds({
628
+ _tag: "Error",
629
+ message: err instanceof Error ? err.message : String(err)
630
+ }));
631
+ }));
632
+ }
633
+ case "AppendRequest": {
634
+ const req = message;
635
+ return Effect.gen(function* () {
636
+ const response = yield* self.#feedStore.append(req);
637
+ log("feed sync server append completed", {
638
+ peerId: self.#peerId,
639
+ recipientPeerId,
640
+ requestId: req.requestId,
641
+ spaceId: req.spaceId,
642
+ feedNamespace: req.feedNamespace,
643
+ blockCount: req.blocks.length,
644
+ assignedPositions: response.positions.length
645
+ }, {
646
+ "~LogMeta": "~LogMeta",
647
+ F: __dxlog_file$1,
648
+ L: 113,
649
+ S: this
650
+ });
651
+ yield* self.#sendMessage(ctx, withPeerIds({
652
+ _tag: "AppendResponse",
653
+ ...response
654
+ }));
655
+ }).pipe(Effect.catchAll((err) => {
656
+ log("feed sync server append failed", {
657
+ peerId: self.#peerId,
658
+ recipientPeerId,
659
+ requestId: req.requestId,
660
+ spaceId: req.spaceId,
661
+ feedNamespace: req.feedNamespace,
662
+ blockCount: req.blocks.length,
663
+ cause: err instanceof Error ? err.message : String(err)
664
+ }, {
665
+ "~LogMeta": "~LogMeta",
666
+ F: __dxlog_file$1,
667
+ L: 125,
668
+ S: this
669
+ });
670
+ return self.#sendMessage(ctx, withPeerIds({
671
+ _tag: "Error",
672
+ message: err instanceof Error ? err.message : String(err)
673
+ }));
674
+ }));
675
+ }
676
+ default:
677
+ log("feed sync server ignoring unsupported message", {
678
+ peerId: self.#peerId,
679
+ tag: message._tag
680
+ }, {
681
+ "~LogMeta": "~LogMeta",
682
+ F: __dxlog_file$1,
683
+ L: 145,
684
+ S: this
685
+ });
686
+ return Effect.void;
687
+ }
688
+ }
689
+ };
690
+ //#endregion
691
+ //#region src/sync-client.ts
692
+ var __dxlog_file = "/__w/dxos/dxos/packages/core/echo/feed/src/sync-client.ts";
693
+ /** Default timeout for feed sync RPCs awaiting an edge response. */
694
+ var DEFAULT_SYNC_RPC_TIMEOUT_MS = 3e4;
695
+ /**
696
+ * Client-side sync: pull/push by sending protocol messages and handling responses.
697
+ * Sets senderPeerId = peerId and recipientPeerId = serverPeerId on all requests.
698
+ * Uses a map of requestId -> Deferred to match responses to in-flight requests.
699
+ * handleMessage completes the Deferred for the requestId (if any) and removes the handler; no handler = noop.
700
+ */
701
+ var SyncClient = class {
702
+ #peerId;
703
+ #serverPeerId;
704
+ #feedStore;
705
+ #sendMessage;
706
+ #rpcTimeoutMs;
707
+ #handlers = /* @__PURE__ */ new Map();
708
+ constructor(options) {
709
+ this.#peerId = options.peerId;
710
+ this.#serverPeerId = options.serverPeerId;
711
+ this.#feedStore = options.feedStore;
712
+ this.#sendMessage = options.sendMessage;
713
+ this.#rpcTimeoutMs = options.rpcTimeoutMs ?? 3e4;
714
+ }
715
+ #withPeerIds(payload) {
716
+ if (payload._tag === "QueryRequest") return {
717
+ ...payload,
718
+ senderPeerId: this.#peerId,
719
+ recipientPeerId: this.#serverPeerId
720
+ };
721
+ return {
722
+ ...payload,
723
+ senderPeerId: this.#peerId,
724
+ recipientPeerId: this.#serverPeerId
725
+ };
726
+ }
727
+ /**
728
+ * Receive a message from the server. If a handler is registered for the message's requestId, complete it and remove; else noop.
729
+ */
730
+ handleMessage(message) {
731
+ const requestId = "requestId" in message && message.requestId != null ? String(message.requestId) : void 0;
732
+ if (requestId == null) {
733
+ log.trace("feed sync client response ignored (no request id)", {
734
+ tag: message._tag,
735
+ senderPeerId: message.senderPeerId
736
+ }, {
737
+ "~LogMeta": "~LogMeta",
738
+ F: __dxlog_file,
739
+ L: 90,
740
+ S: this
741
+ });
742
+ return Effect.void;
743
+ }
744
+ const deferred = this.#handlers.get(requestId);
745
+ if (deferred == null) {
746
+ log.trace("feed sync client response ignored (no pending rpc)", {
747
+ requestId,
748
+ tag: message._tag,
749
+ senderPeerId: message.senderPeerId
750
+ }, {
751
+ "~LogMeta": "~LogMeta",
752
+ F: __dxlog_file,
753
+ L: 98,
754
+ S: this
755
+ });
756
+ return Effect.void;
757
+ }
758
+ this.#handlers.delete(requestId);
759
+ log("feed sync client rpc completed", {
760
+ requestId,
761
+ tag: message._tag,
762
+ blockCount: "blocks" in message ? message.blocks.length : void 0,
763
+ positionCount: "positions" in message ? message.positions.length : void 0,
764
+ error: message._tag === "Error" ? message.message : void 0
765
+ }, {
766
+ "~LogMeta": "~LogMeta",
767
+ F: __dxlog_file,
768
+ L: 106,
769
+ S: this
770
+ });
771
+ if (message._tag === "Error") return Effect.andThen(Deferred.fail(deferred, new Error(message.message)), () => Effect.void);
772
+ return Effect.andThen(Deferred.succeed(deferred, message), () => Effect.void);
773
+ }
774
+ /** Removes the pending handler entry and unregisters the ctx onDispose hook. Idempotent. */
775
+ #disposeHandler(requestId, cleanupDispose) {
776
+ cleanupDispose();
777
+ this.#handlers.delete(requestId);
778
+ }
779
+ #awaitRpcResponse(requestId, deferred, cleanupDispose, meta) {
780
+ const self = this;
781
+ const timeoutMs = self.#rpcTimeoutMs;
782
+ return Effect.ensuring(Deferred.await(deferred).pipe(Effect.timeoutFail({
783
+ duration: timeoutMs,
784
+ onTimeout: () => new SyncRpcTimeoutError({
785
+ requestId,
786
+ spaceId: meta.spaceId,
787
+ feedNamespace: meta.feedNamespace,
788
+ rpcTag: meta.rpcTag,
789
+ timeoutMs
790
+ })
791
+ }), Effect.tapError((cause) => Effect.sync(() => {
792
+ if (cause instanceof SyncRpcTimeoutError) log("feed sync client rpc timed out", {
793
+ requestId,
794
+ spaceId: meta.spaceId,
795
+ feedNamespace: meta.feedNamespace,
796
+ rpcTag: meta.rpcTag,
797
+ timeoutMs
798
+ }, {
799
+ "~LogMeta": "~LogMeta",
800
+ F: __dxlog_file,
801
+ L: 149,
802
+ S: this
803
+ });
804
+ }))), Effect.sync(() => self.#disposeHandler(requestId, cleanupDispose)));
805
+ }
806
+ pull(ctx, opts) {
807
+ const self = this;
808
+ return Effect.gen(function* () {
809
+ const lastPulledPosition = yield* self.#feedStore.getSyncState({
810
+ spaceId: opts.spaceId,
811
+ feedNamespace: opts.feedNamespace
812
+ });
813
+ const requestId = crypto.randomUUID();
814
+ const deferred = yield* Deferred.make();
815
+ self.#handlers.set(requestId, deferred);
816
+ const cleanupDispose = ctx.disposed ? () => {} : ctx.onDispose(() => {
817
+ Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
818
+ });
819
+ if (ctx.disposed) yield* Deferred.fail(deferred, new ContextDisposedError());
820
+ const request = {
821
+ _tag: "QueryRequest",
822
+ requestId,
823
+ spaceId: opts.spaceId,
824
+ feedNamespace: opts.feedNamespace,
825
+ position: lastPulledPosition,
826
+ limit: opts.limit
827
+ };
828
+ log("feed sync client pull rpc sending", {
829
+ requestId,
830
+ spaceId: opts.spaceId,
831
+ feedNamespace: opts.feedNamespace,
832
+ afterPosition: lastPulledPosition,
833
+ limit: opts.limit
834
+ }, {
835
+ "~LogMeta": "~LogMeta",
836
+ F: __dxlog_file,
837
+ L: 197,
838
+ S: this
839
+ });
840
+ yield* self.#sendMessage(ctx, self.#withPeerIds(request)).pipe(Effect.tapErrorCause(() => Effect.sync(() => self.#disposeHandler(requestId, cleanupDispose))));
841
+ const message = yield* self.#awaitRpcResponse(requestId, deferred, cleanupDispose, {
842
+ spaceId: opts.spaceId,
843
+ feedNamespace: opts.feedNamespace,
844
+ rpcTag: "QueryRequest"
845
+ });
846
+ const response = yield* self.#expectResponse(requestId, message, "QueryResponse");
847
+ if (response.blocks.length === 0) {
848
+ log.trace("feed sync client pull done (empty batch)", {
849
+ requestId,
850
+ spaceId: opts.spaceId,
851
+ feedNamespace: opts.feedNamespace
852
+ }, {
853
+ "~LogMeta": "~LogMeta",
854
+ F: __dxlog_file,
855
+ L: 214,
856
+ S: this
857
+ });
858
+ return { done: true };
859
+ }
860
+ yield* self.#feedStore.append({
861
+ spaceId: opts.spaceId,
862
+ feedNamespace: opts.feedNamespace,
863
+ blocks: response.blocks
864
+ });
865
+ const maxPulledPosition = response.blocks.reduce((max, block) => block.position != null && block.position > max ? block.position : max, lastPulledPosition);
866
+ yield* self.#feedStore.setSyncState({
867
+ spaceId: opts.spaceId,
868
+ feedNamespace: opts.feedNamespace,
869
+ lastPulledPosition: maxPulledPosition
870
+ });
871
+ log("feed sync client pull applied batch", {
872
+ requestId,
873
+ spaceId: opts.spaceId,
874
+ feedNamespace: opts.feedNamespace,
875
+ batchSize: response.blocks.length,
876
+ hasMore: response.hasMore,
877
+ maxPulledPosition
878
+ }, {
879
+ "~LogMeta": "~LogMeta",
880
+ F: __dxlog_file,
881
+ L: 238,
882
+ S: this
883
+ });
884
+ return { done: false };
885
+ });
886
+ }
887
+ /**
888
+ * Probes remote for blocks after the last pulled position without mutating local storage.
889
+ * Returns the number of blocks in the first batch (0 when caught up with remote).
890
+ */
891
+ peekPull(ctx, opts) {
892
+ const self = this;
893
+ return Effect.gen(function* () {
894
+ const lastPulledPosition = yield* self.#feedStore.getSyncState({
895
+ spaceId: opts.spaceId,
896
+ feedNamespace: opts.feedNamespace
897
+ });
898
+ const requestId = crypto.randomUUID();
899
+ const deferred = yield* Deferred.make();
900
+ self.#handlers.set(requestId, deferred);
901
+ const cleanupDispose = ctx.disposed ? () => {} : ctx.onDispose(() => {
902
+ Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
903
+ });
904
+ if (ctx.disposed) yield* Deferred.fail(deferred, new ContextDisposedError());
905
+ const request = {
906
+ _tag: "QueryRequest",
907
+ requestId,
908
+ spaceId: opts.spaceId,
909
+ feedNamespace: opts.feedNamespace,
910
+ position: lastPulledPosition,
911
+ limit: opts.limit
912
+ };
913
+ yield* self.#sendMessage(ctx, self.#withPeerIds(request)).pipe(Effect.tapErrorCause(() => Effect.sync(() => self.#disposeHandler(requestId, cleanupDispose))));
914
+ const message = yield* self.#awaitRpcResponse(requestId, deferred, cleanupDispose, {
915
+ spaceId: opts.spaceId,
916
+ feedNamespace: opts.feedNamespace,
917
+ rpcTag: "QueryRequest"
918
+ });
919
+ return { blocksToPull: (yield* self.#expectResponse(requestId, message, "QueryResponse")).blocks.length };
920
+ });
921
+ }
922
+ push(ctx, opts) {
923
+ const self = this;
924
+ return Effect.gen(function* () {
925
+ const unpositioned = yield* self.#feedStore.query({
926
+ spaceId: opts.spaceId,
927
+ feedNamespace: opts.feedNamespace,
928
+ unpositionedOnly: true,
929
+ limit: opts.limit
930
+ });
931
+ if (unpositioned.blocks.length === 0) {
932
+ log.trace("feed sync client push skipped (nothing to send)", {
933
+ spaceId: opts.spaceId,
934
+ feedNamespace: opts.feedNamespace
935
+ }, {
936
+ "~LogMeta": "~LogMeta",
937
+ F: __dxlog_file,
938
+ L: 317,
939
+ S: this
940
+ });
941
+ return { done: true };
942
+ }
943
+ const requestId = crypto.randomUUID();
944
+ const deferred = yield* Deferred.make();
945
+ self.#handlers.set(requestId, deferred);
946
+ const cleanupDispose = ctx.disposed ? () => {} : ctx.onDispose(() => {
947
+ Effect.runFork(Deferred.fail(deferred, new ContextDisposedError()));
948
+ });
949
+ if (ctx.disposed) yield* Deferred.fail(deferred, new ContextDisposedError());
950
+ const request = {
951
+ _tag: "AppendRequest",
952
+ requestId,
953
+ spaceId: opts.spaceId,
954
+ feedNamespace: opts.feedNamespace,
955
+ blocks: unpositioned.blocks
956
+ };
957
+ log("feed sync client push rpc sending", {
958
+ requestId,
959
+ spaceId: opts.spaceId,
960
+ feedNamespace: opts.feedNamespace,
961
+ blockCount: unpositioned.blocks.length
962
+ }, {
963
+ "~LogMeta": "~LogMeta",
964
+ F: __dxlog_file,
965
+ L: 341,
966
+ S: this
967
+ });
968
+ yield* self.#sendMessage(ctx, self.#withPeerIds(request)).pipe(Effect.tapErrorCause(() => Effect.sync(() => self.#disposeHandler(requestId, cleanupDispose))));
969
+ const message = yield* self.#awaitRpcResponse(requestId, deferred, cleanupDispose, {
970
+ spaceId: opts.spaceId,
971
+ feedNamespace: opts.feedNamespace,
972
+ rpcTag: "AppendRequest"
973
+ });
974
+ const response = yield* self.#expectResponse(requestId, message, "AppendResponse");
975
+ yield* self.#feedStore.setPosition({
976
+ spaceId: opts.spaceId,
977
+ blocks: Array.zipWith(response.positions, unpositioned.blocks, (position, block) => ({
978
+ feedId: block.feedId,
979
+ feedNamespace: opts.feedNamespace,
980
+ actorId: block.actorId,
981
+ sequence: block.sequence,
982
+ position
983
+ }))
984
+ });
985
+ log("feed sync client push positions applied", {
986
+ requestId,
987
+ spaceId: opts.spaceId,
988
+ feedNamespace: opts.feedNamespace,
989
+ positionCount: response.positions.length
990
+ }, {
991
+ "~LogMeta": "~LogMeta",
992
+ F: __dxlog_file,
993
+ L: 366,
994
+ S: this
995
+ });
996
+ return { done: false };
997
+ });
998
+ }
999
+ #expectResponse(requestId, message, expectedTag) {
1000
+ if (message._tag === "Error") return Effect.fail(new Error(message.message));
1001
+ const requestIdMsg = "requestId" in message ? String(message.requestId) : void 0;
1002
+ if (message._tag !== expectedTag || requestIdMsg !== requestId) return Effect.fail(/* @__PURE__ */ new Error(`Unexpected message: expected ${expectedTag} with requestId ${requestId}`));
1003
+ return Effect.succeed(message);
1004
+ }
1005
+ };
1006
+ //#endregion
1007
+ export { FeedStoreService as a, FeedStore as i, SyncClient as n, PositionConflictError as o, SyncServer as r, SyncRpcTimeoutError as s, DEFAULT_SYNC_RPC_TIMEOUT_MS as t };
1008
+
1009
+ //# sourceMappingURL=chunk-sync-client.mjs.map