@dxos/feed 0.8.4-main.ef1bc66f44 → 0.8.4-main.effb148878
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +0 -1
- package/dist/lib/neutral/{chunk-Q5PE2Q5P.mjs → chunk-UC5ETQIJ.mjs} +410 -123
- package/dist/lib/neutral/chunk-UC5ETQIJ.mjs.map +7 -0
- package/dist/lib/neutral/index.mjs +5 -5
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/lib/neutral/testing/index.mjs +8 -7
- package/dist/lib/neutral/testing/index.mjs.map +3 -3
- package/dist/types/src/errors.d.ts +43 -47
- package/dist/types/src/errors.d.ts.map +1 -1
- package/dist/types/src/feed-store.d.ts +53 -4
- package/dist/types/src/feed-store.d.ts.map +1 -1
- package/dist/types/src/sync-client.d.ts +22 -3
- package/dist/types/src/sync-client.d.ts.map +1 -1
- package/dist/types/src/sync-client.test.d.ts +2 -0
- package/dist/types/src/sync-client.test.d.ts.map +1 -0
- package/dist/types/src/sync-server.d.ts +3 -2
- package/dist/types/src/sync-server.d.ts.map +1 -1
- package/dist/types/src/testing/test-builder.d.ts +5 -5
- package/dist/types/src/testing/test-builder.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +16 -21
- package/src/errors.ts +10 -14
- package/src/feed-store.test.ts +264 -96
- package/src/feed-store.ts +227 -129
- package/src/sync-client.test.ts +63 -0
- package/src/sync-client.ts +215 -17
- package/src/sync-server.ts +72 -12
- package/src/testing/test-builder.ts +11 -11
- package/dist/lib/neutral/chunk-Q5PE2Q5P.mjs.map +0 -7
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
// src/errors.ts
|
|
2
2
|
import { BaseError } from "@dxos/errors";
|
|
3
|
-
var
|
|
3
|
+
var SyncRpcTimeoutError = class extends BaseError.extend("SyncRpcTimeoutError") {
|
|
4
4
|
constructor(args) {
|
|
5
|
-
super(
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
constructor(args) {
|
|
10
|
-
super(`Non-unique block Lamport timestamp detected (feedId=${args.feedId} actorId=${args.actorId} sequence=${args.sequence}).`);
|
|
5
|
+
super({
|
|
6
|
+
message: `Feed sync ${args.rpcTag} timed out after ${args.timeoutMs}ms (spaceId=${args.spaceId} feedNamespace=${args.feedNamespace} requestId=${args.requestId}).`,
|
|
7
|
+
context: args
|
|
8
|
+
});
|
|
11
9
|
}
|
|
12
10
|
};
|
|
13
11
|
var PositionConflictError = class extends BaseError.extend("PositionConflictError") {
|
|
14
12
|
constructor(args) {
|
|
15
|
-
super(
|
|
13
|
+
super({
|
|
14
|
+
message: `Block already has position ${args.currentPosition}, cannot set to ${args.requestedPosition} (feedId=${args.feedId} actorId=${args.actorId} sequence=${args.sequence}).`,
|
|
15
|
+
context: args
|
|
16
|
+
});
|
|
16
17
|
}
|
|
17
18
|
};
|
|
18
19
|
|
|
@@ -84,7 +85,9 @@ var FeedStore = class {
|
|
|
84
85
|
const rows = yield* sql`
|
|
85
86
|
SELECT feedPrivateId FROM feeds WHERE spaceId = ${spaceId} AND feedId = ${feedId}
|
|
86
87
|
`;
|
|
87
|
-
if (rows.length > 0)
|
|
88
|
+
if (rows.length > 0) {
|
|
89
|
+
return rows[0].feedPrivateId;
|
|
90
|
+
}
|
|
88
91
|
const newRows = yield* sql`
|
|
89
92
|
INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
|
|
90
93
|
`;
|
|
@@ -96,7 +99,9 @@ var FeedStore = class {
|
|
|
96
99
|
#ensureCursorToken = Effect.fn("Feed.ensureCursorToken")((spaceId) => Effect.gen(this, function* () {
|
|
97
100
|
const sql = yield* SqlClient.SqlClient;
|
|
98
101
|
const rows = yield* sql`SELECT token FROM cursor_tokens WHERE spaceId = ${spaceId}`;
|
|
99
|
-
if (rows.length > 0)
|
|
102
|
+
if (rows.length > 0) {
|
|
103
|
+
return rows[0].token;
|
|
104
|
+
}
|
|
100
105
|
const token = crypto.randomUUID().replace(/-/g, "").slice(0, 6);
|
|
101
106
|
yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
|
|
102
107
|
return token;
|
|
@@ -247,80 +252,80 @@ var FeedStore = class {
|
|
|
247
252
|
`;
|
|
248
253
|
}).pipe(Effect.withSpan("FeedStore.setSyncState"));
|
|
249
254
|
/**
|
|
250
|
-
*
|
|
255
|
+
* Returns the number of blocks pending push (no global position yet) in a space/namespace.
|
|
251
256
|
*/
|
|
252
|
-
|
|
257
|
+
countUnpositionedBlocks = (opts) => Effect.gen(this, function* () {
|
|
253
258
|
const sql = yield* SqlClient.SqlClient;
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
SELECT feedPrivateId, position
|
|
308
|
-
FROM blocks
|
|
309
|
-
WHERE position IS NOT NULL
|
|
310
|
-
AND (${sql.or(positionConditions)})
|
|
311
|
-
LIMIT 1
|
|
312
|
-
`;
|
|
313
|
-
const positionConflict = existingPositions[0];
|
|
314
|
-
if (positionConflict) {
|
|
315
|
-
const key = `${positionConflict.feedPrivateId}|${positionConflict.position}`;
|
|
316
|
-
yield* Effect.fail(new DuplicateBlockPositionError({
|
|
317
|
-
feedId: positionFeedIdByKey.get(key),
|
|
318
|
-
position: positionConflict.position
|
|
319
|
-
}));
|
|
320
|
-
}
|
|
321
|
-
}
|
|
259
|
+
const rows = yield* sql`
|
|
260
|
+
SELECT COUNT(*) AS count
|
|
261
|
+
FROM blocks
|
|
262
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
263
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
264
|
+
AND feeds.feedNamespace = ${opts.feedNamespace}
|
|
265
|
+
AND blocks.position IS NULL
|
|
266
|
+
`;
|
|
267
|
+
return rows[0]?.count ?? 0;
|
|
268
|
+
}).pipe(Effect.withSpan("FeedStore.countUnpositionedBlocks"));
|
|
269
|
+
/**
|
|
270
|
+
* Returns the total number of blocks stored locally for a space/namespace.
|
|
271
|
+
*/
|
|
272
|
+
countNamespaceBlocks = (opts) => Effect.gen(this, function* () {
|
|
273
|
+
const sql = yield* SqlClient.SqlClient;
|
|
274
|
+
const rows = yield* sql`
|
|
275
|
+
SELECT COUNT(*) AS count
|
|
276
|
+
FROM blocks
|
|
277
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
278
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
279
|
+
AND feeds.feedNamespace = ${opts.feedNamespace}
|
|
280
|
+
`;
|
|
281
|
+
return rows[0]?.count ?? 0;
|
|
282
|
+
}).pipe(Effect.withSpan("FeedStore.countNamespaceBlocks"));
|
|
283
|
+
/**
|
|
284
|
+
* Returns the number of stored blocks for a single feed in a space/namespace.
|
|
285
|
+
* Intended as a low-level primitive for callers (for example Cloudflare Worker code)
|
|
286
|
+
* that need to make retention decisions under constrained storage resources.
|
|
287
|
+
*/
|
|
288
|
+
countBlocks = (opts) => Effect.gen(this, function* () {
|
|
289
|
+
const sql = yield* SqlClient.SqlClient;
|
|
290
|
+
const rows = yield* sql`
|
|
291
|
+
SELECT COUNT(*) AS count
|
|
292
|
+
FROM blocks
|
|
293
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
294
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
295
|
+
AND feeds.feedNamespace = ${opts.feedNamespace}
|
|
296
|
+
AND feeds.feedId = ${opts.feedId}
|
|
297
|
+
`;
|
|
298
|
+
return rows[0]?.count ?? 0;
|
|
299
|
+
}).pipe(Effect.withSpan("FeedStore.countBlocks"));
|
|
300
|
+
/**
|
|
301
|
+
* Deletes the oldest blocks for a single feed in a space/namespace.
|
|
302
|
+
* This API intentionally does not enforce any retention policy (such as max size);
|
|
303
|
+
* callers decide when and how much to prune, which is useful for constrained
|
|
304
|
+
* environments like Cloudflare Workers.
|
|
305
|
+
*
|
|
306
|
+
* @returns Number of deleted rows.
|
|
307
|
+
*/
|
|
308
|
+
deleteOldestBlocks = (opts) => Effect.gen(this, function* () {
|
|
309
|
+
const sql = yield* SqlClient.SqlClient;
|
|
310
|
+
if (opts.count <= 0) {
|
|
311
|
+
return 0;
|
|
322
312
|
}
|
|
323
|
-
|
|
313
|
+
const deletedRows = yield* sql`
|
|
314
|
+
DELETE FROM blocks
|
|
315
|
+
WHERE insertionId IN (
|
|
316
|
+
SELECT blocks.insertionId
|
|
317
|
+
FROM blocks
|
|
318
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
319
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
320
|
+
AND feeds.feedNamespace = ${opts.feedNamespace}
|
|
321
|
+
AND feeds.feedId = ${opts.feedId}
|
|
322
|
+
ORDER BY blocks.insertionId ASC
|
|
323
|
+
LIMIT ${opts.count}
|
|
324
|
+
)
|
|
325
|
+
RETURNING insertionId
|
|
326
|
+
`;
|
|
327
|
+
return deletedRows.length;
|
|
328
|
+
}).pipe(Effect.withSpan("FeedStore.deleteOldestBlocks"));
|
|
324
329
|
/**
|
|
325
330
|
* Appends blocks for a space/namespace and optionally assigns global positions.
|
|
326
331
|
*/
|
|
@@ -353,7 +358,6 @@ var FeedStore = class {
|
|
|
353
358
|
}), {
|
|
354
359
|
concurrency: "unbounded"
|
|
355
360
|
});
|
|
356
|
-
yield* this.#validateAppendBlocks(request.blocks, feedPrivateIds);
|
|
357
361
|
const maxPositions = /* @__PURE__ */ new Map();
|
|
358
362
|
if (this.#options.assignPositions) {
|
|
359
363
|
const maxPosResult = yield* sql`
|
|
@@ -370,22 +374,48 @@ var FeedStore = class {
|
|
|
370
374
|
const feedPrivateId = feedPrivateIds.get(key);
|
|
371
375
|
let positionToInsert = null;
|
|
372
376
|
if (this.#options.assignPositions) {
|
|
373
|
-
|
|
374
|
-
positionToInsert = currentMax + 1;
|
|
375
|
-
maxPositions.set(request.feedNamespace, positionToInsert);
|
|
376
|
-
positions2.push(positionToInsert);
|
|
377
|
+
positionToInsert = maxPositions.get(request.feedNamespace) + 1;
|
|
377
378
|
} else if (block.position != null) {
|
|
378
379
|
positionToInsert = block.position;
|
|
379
380
|
}
|
|
380
|
-
yield* sql`
|
|
381
|
+
const inserted = yield* sql`
|
|
381
382
|
INSERT INTO blocks (
|
|
382
|
-
feedPrivateId, position, sequence, actorId,
|
|
383
|
+
feedPrivateId, position, sequence, actorId,
|
|
383
384
|
prevSequence, prevActorId, timestamp, data
|
|
384
385
|
) VALUES (
|
|
385
386
|
${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
|
|
386
387
|
${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
|
|
387
|
-
)
|
|
388
|
+
)
|
|
389
|
+
ON CONFLICT(feedPrivateId, sequence, actorId) DO NOTHING
|
|
390
|
+
RETURNING position
|
|
388
391
|
`;
|
|
392
|
+
if (!this.#options.assignPositions) {
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
if (inserted.length > 0) {
|
|
396
|
+
positions2.push(positionToInsert);
|
|
397
|
+
maxPositions.set(request.feedNamespace, positionToInsert);
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
const existing = yield* sql`
|
|
401
|
+
SELECT position FROM blocks
|
|
402
|
+
WHERE feedPrivateId = ${feedPrivateId}
|
|
403
|
+
AND actorId = ${block.actorId}
|
|
404
|
+
AND sequence = ${block.sequence}
|
|
405
|
+
`;
|
|
406
|
+
const existingPosition = existing[0]?.position;
|
|
407
|
+
if (existingPosition != null) {
|
|
408
|
+
positions2.push(existingPosition);
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
yield* sql`
|
|
412
|
+
UPDATE blocks SET position = ${positionToInsert}
|
|
413
|
+
WHERE feedPrivateId = ${feedPrivateId}
|
|
414
|
+
AND actorId = ${block.actorId}
|
|
415
|
+
AND sequence = ${block.sequence}
|
|
416
|
+
`;
|
|
417
|
+
positions2.push(positionToInsert);
|
|
418
|
+
maxPositions.set(request.feedNamespace, positionToInsert);
|
|
389
419
|
}
|
|
390
420
|
return positions2;
|
|
391
421
|
}));
|
|
@@ -475,15 +505,22 @@ var FeedStore = class {
|
|
|
475
505
|
}
|
|
476
506
|
blocksBySpaceNamespace.get(spaceNamespaceKey).blocks.push(block);
|
|
477
507
|
}
|
|
478
|
-
|
|
479
|
-
|
|
508
|
+
const positionByBlock = /* @__PURE__ */ new Map();
|
|
509
|
+
for (const { spaceId, feedNamespace, blocks: batchBlocks } of blocksBySpaceNamespace.values()) {
|
|
510
|
+
const { positions } = yield* this.append({
|
|
480
511
|
requestId: "local-append",
|
|
481
|
-
blocks:
|
|
512
|
+
blocks: batchBlocks,
|
|
482
513
|
spaceId,
|
|
483
514
|
feedNamespace
|
|
484
515
|
});
|
|
516
|
+
for (let i = 0; i < batchBlocks.length; i++) {
|
|
517
|
+
positionByBlock.set(batchBlocks[i], positions[i] ?? null);
|
|
518
|
+
}
|
|
485
519
|
}
|
|
486
|
-
return blocks
|
|
520
|
+
return blocks.map((block) => ({
|
|
521
|
+
...block,
|
|
522
|
+
position: positionByBlock.get(block) ?? block.position
|
|
523
|
+
}));
|
|
487
524
|
}).pipe(Effect.withSpan("FeedStore.appendLocal")));
|
|
488
525
|
/**
|
|
489
526
|
* Sets positions for existing blocks while preventing conflicting reassignments.
|
|
@@ -491,6 +528,19 @@ var FeedStore = class {
|
|
|
491
528
|
setPosition = (request) => Effect.gen(this, function* () {
|
|
492
529
|
const sql = yield* SqlClient.SqlClient;
|
|
493
530
|
for (const block of request.blocks) {
|
|
531
|
+
const updated = yield* sql`
|
|
532
|
+
UPDATE blocks SET position = ${block.position}
|
|
533
|
+
WHERE feedPrivateId = (
|
|
534
|
+
SELECT feedPrivateId FROM feeds
|
|
535
|
+
WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
|
|
536
|
+
)
|
|
537
|
+
AND actorId = ${block.actorId} AND sequence = ${block.sequence}
|
|
538
|
+
AND (position IS NULL OR position = ${block.position})
|
|
539
|
+
RETURNING position
|
|
540
|
+
`;
|
|
541
|
+
if (updated.length > 0) {
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
494
544
|
const existing = yield* sql`
|
|
495
545
|
SELECT position FROM blocks
|
|
496
546
|
WHERE feedPrivateId = (
|
|
@@ -509,16 +559,39 @@ var FeedStore = class {
|
|
|
509
559
|
requestedPosition: block.position
|
|
510
560
|
}));
|
|
511
561
|
}
|
|
512
|
-
yield* sql`
|
|
513
|
-
UPDATE blocks SET position = ${block.position}
|
|
514
|
-
WHERE feedPrivateId = (
|
|
515
|
-
SELECT feedPrivateId FROM feeds
|
|
516
|
-
WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
|
|
517
|
-
)
|
|
518
|
-
AND actorId = ${block.actorId} AND sequence = ${block.sequence}
|
|
519
|
-
`;
|
|
520
562
|
}
|
|
521
563
|
}).pipe(Effect.withSpan("FeedStore.setPosition"));
|
|
564
|
+
/**
|
|
565
|
+
* Gets all feeds and their blocks for a space, organized by feed ID and namespace.
|
|
566
|
+
* Used for space archive export.
|
|
567
|
+
*/
|
|
568
|
+
getAllFeedsForSpace = (opts) => Effect.gen(this, function* () {
|
|
569
|
+
const sql = yield* SqlClient.SqlClient;
|
|
570
|
+
const feeds = yield* sql`
|
|
571
|
+
SELECT DISTINCT feedId, feedNamespace FROM feeds WHERE spaceId = ${opts.spaceId}
|
|
572
|
+
`;
|
|
573
|
+
const result = [];
|
|
574
|
+
for (const feed of feeds) {
|
|
575
|
+
const blocks = yield* sql`
|
|
576
|
+
SELECT blocks.*, feeds.feedId, feeds.feedNamespace
|
|
577
|
+
FROM blocks
|
|
578
|
+
JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
|
|
579
|
+
WHERE feeds.spaceId = ${opts.spaceId}
|
|
580
|
+
AND feeds.feedId = ${feed.feedId}
|
|
581
|
+
AND feeds.feedNamespace = ${feed.feedNamespace}
|
|
582
|
+
ORDER BY blocks.sequence ASC
|
|
583
|
+
`;
|
|
584
|
+
result.push({
|
|
585
|
+
feedId: feed.feedId,
|
|
586
|
+
feedNamespace: feed.feedNamespace,
|
|
587
|
+
blocks: blocks.map((row) => ({
|
|
588
|
+
...row,
|
|
589
|
+
data: new Uint8Array(row.data)
|
|
590
|
+
}))
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
return result;
|
|
594
|
+
}).pipe(Effect.withSpan("FeedStore.getAllFeedsForSpace"));
|
|
522
595
|
};
|
|
523
596
|
var encodeCursor = (token, insertionId) => FeedCursor.make(`${token}|${insertionId}`);
|
|
524
597
|
var decodeCursor = (cursor) => {
|
|
@@ -531,6 +604,19 @@ var decodeCursor = (cursor) => {
|
|
|
531
604
|
|
|
532
605
|
// src/sync-server.ts
|
|
533
606
|
import * as Effect2 from "effect/Effect";
|
|
607
|
+
import { log } from "@dxos/log";
|
|
608
|
+
var __dxlog_file = "/__w/dxos/dxos/packages/core/echo/feed/src/sync-server.ts";
|
|
609
|
+
var inboundSummary = (message) => ({
|
|
610
|
+
tag: message._tag,
|
|
611
|
+
senderPeerId: message.senderPeerId,
|
|
612
|
+
recipientPeerId: message.recipientPeerId,
|
|
613
|
+
requestId: "requestId" in message ? message.requestId : void 0,
|
|
614
|
+
spaceId: "spaceId" in message ? message.spaceId : void 0,
|
|
615
|
+
feedNamespace: "feedNamespace" in message ? message.feedNamespace : void 0,
|
|
616
|
+
blockCount: "blocks" in message ? message.blocks.length : void 0,
|
|
617
|
+
position: "position" in message ? message.position : void 0,
|
|
618
|
+
limit: "limit" in message ? message.limit : void 0
|
|
619
|
+
});
|
|
534
620
|
var SyncServer = class {
|
|
535
621
|
#peerId;
|
|
536
622
|
#feedStore;
|
|
@@ -543,8 +629,12 @@ var SyncServer = class {
|
|
|
543
629
|
/**
|
|
544
630
|
* Receive a message from a client. Handles QueryRequest and AppendRequest; sends response via sendMessage with correct peer ids.
|
|
545
631
|
*/
|
|
546
|
-
handleMessage(message) {
|
|
632
|
+
handleMessage(ctx, message) {
|
|
547
633
|
const self = this;
|
|
634
|
+
log("feed sync server received message", {
|
|
635
|
+
peerId: self.#peerId,
|
|
636
|
+
...inboundSummary(message)
|
|
637
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 33, S: this });
|
|
548
638
|
const recipientPeerId = message.senderPeerId;
|
|
549
639
|
const withPeerIds = (payload) => ({
|
|
550
640
|
...payload,
|
|
@@ -556,29 +646,72 @@ var SyncServer = class {
|
|
|
556
646
|
const req = message;
|
|
557
647
|
return Effect2.gen(function* () {
|
|
558
648
|
const response = yield* self.#feedStore.query(req);
|
|
559
|
-
|
|
649
|
+
log("feed sync server query completed", {
|
|
650
|
+
peerId: self.#peerId,
|
|
651
|
+
recipientPeerId,
|
|
652
|
+
requestId: req.requestId,
|
|
653
|
+
spaceId: req.spaceId,
|
|
654
|
+
feedNamespace: req.feedNamespace,
|
|
655
|
+
returnedBlocks: response.blocks.length,
|
|
656
|
+
hasMore: response.hasMore
|
|
657
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 49, S: this });
|
|
658
|
+
yield* self.#sendMessage(ctx, withPeerIds({
|
|
560
659
|
_tag: "QueryResponse",
|
|
561
660
|
...response
|
|
562
661
|
}));
|
|
563
|
-
}).pipe(Effect2.catchAll((err) =>
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
662
|
+
}).pipe(Effect2.catchAll((err) => {
|
|
663
|
+
log("feed sync server query failed", {
|
|
664
|
+
peerId: self.#peerId,
|
|
665
|
+
recipientPeerId,
|
|
666
|
+
requestId: req.requestId,
|
|
667
|
+
spaceId: req.spaceId,
|
|
668
|
+
feedNamespace: req.feedNamespace,
|
|
669
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
670
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 63, S: this });
|
|
671
|
+
return self.#sendMessage(ctx, withPeerIds({
|
|
672
|
+
_tag: "Error",
|
|
673
|
+
message: err instanceof Error ? err.message : String(err)
|
|
674
|
+
}));
|
|
675
|
+
}));
|
|
567
676
|
}
|
|
568
677
|
case "AppendRequest": {
|
|
569
678
|
const req = message;
|
|
570
679
|
return Effect2.gen(function* () {
|
|
571
680
|
const response = yield* self.#feedStore.append(req);
|
|
572
|
-
|
|
681
|
+
log("feed sync server append completed", {
|
|
682
|
+
peerId: self.#peerId,
|
|
683
|
+
recipientPeerId,
|
|
684
|
+
requestId: req.requestId,
|
|
685
|
+
spaceId: req.spaceId,
|
|
686
|
+
feedNamespace: req.feedNamespace,
|
|
687
|
+
blockCount: req.blocks.length,
|
|
688
|
+
assignedPositions: response.positions.length
|
|
689
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 82, S: this });
|
|
690
|
+
yield* self.#sendMessage(ctx, withPeerIds({
|
|
573
691
|
_tag: "AppendResponse",
|
|
574
692
|
...response
|
|
575
693
|
}));
|
|
576
|
-
}).pipe(Effect2.catchAll((err) =>
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
694
|
+
}).pipe(Effect2.catchAll((err) => {
|
|
695
|
+
log("feed sync server append failed", {
|
|
696
|
+
peerId: self.#peerId,
|
|
697
|
+
recipientPeerId,
|
|
698
|
+
requestId: req.requestId,
|
|
699
|
+
spaceId: req.spaceId,
|
|
700
|
+
feedNamespace: req.feedNamespace,
|
|
701
|
+
blockCount: req.blocks.length,
|
|
702
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
703
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 96, S: this });
|
|
704
|
+
return self.#sendMessage(ctx, withPeerIds({
|
|
705
|
+
_tag: "Error",
|
|
706
|
+
message: err instanceof Error ? err.message : String(err)
|
|
707
|
+
}));
|
|
708
|
+
}));
|
|
580
709
|
}
|
|
581
710
|
default:
|
|
711
|
+
log("feed sync server ignoring unsupported message", {
|
|
712
|
+
peerId: self.#peerId,
|
|
713
|
+
tag: message._tag
|
|
714
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 112, S: this });
|
|
582
715
|
return Effect2.void;
|
|
583
716
|
}
|
|
584
717
|
}
|
|
@@ -588,17 +721,23 @@ var SyncServer = class {
|
|
|
588
721
|
import * as Array from "effect/Array";
|
|
589
722
|
import * as Deferred from "effect/Deferred";
|
|
590
723
|
import * as Effect3 from "effect/Effect";
|
|
724
|
+
import { ContextDisposedError } from "@dxos/context";
|
|
725
|
+
import { log as log2 } from "@dxos/log";
|
|
726
|
+
var __dxlog_file2 = "/__w/dxos/dxos/packages/core/echo/feed/src/sync-client.ts";
|
|
727
|
+
var DEFAULT_SYNC_RPC_TIMEOUT_MS = 3e4;
|
|
591
728
|
var SyncClient = class {
|
|
592
729
|
#peerId;
|
|
593
730
|
#serverPeerId;
|
|
594
731
|
#feedStore;
|
|
595
732
|
#sendMessage;
|
|
733
|
+
#rpcTimeoutMs;
|
|
596
734
|
#handlers = /* @__PURE__ */ new Map();
|
|
597
735
|
constructor(options) {
|
|
598
736
|
this.#peerId = options.peerId;
|
|
599
737
|
this.#serverPeerId = options.serverPeerId;
|
|
600
738
|
this.#feedStore = options.feedStore;
|
|
601
739
|
this.#sendMessage = options.sendMessage;
|
|
740
|
+
this.#rpcTimeoutMs = options.rpcTimeoutMs ?? DEFAULT_SYNC_RPC_TIMEOUT_MS;
|
|
602
741
|
}
|
|
603
742
|
#withPeerIds(payload) {
|
|
604
743
|
if (payload._tag === "QueryRequest") {
|
|
@@ -619,16 +758,65 @@ var SyncClient = class {
|
|
|
619
758
|
*/
|
|
620
759
|
handleMessage(message) {
|
|
621
760
|
const requestId = "requestId" in message && message.requestId != null ? String(message.requestId) : void 0;
|
|
622
|
-
if (requestId == null)
|
|
761
|
+
if (requestId == null) {
|
|
762
|
+
log2.trace("feed sync client response ignored (no request id)", {
|
|
763
|
+
tag: message._tag,
|
|
764
|
+
senderPeerId: message.senderPeerId
|
|
765
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 49, S: this });
|
|
766
|
+
return Effect3.void;
|
|
767
|
+
}
|
|
623
768
|
const deferred = this.#handlers.get(requestId);
|
|
624
|
-
if (deferred == null)
|
|
769
|
+
if (deferred == null) {
|
|
770
|
+
log2.trace("feed sync client response ignored (no pending rpc)", {
|
|
771
|
+
requestId,
|
|
772
|
+
tag: message._tag,
|
|
773
|
+
senderPeerId: message.senderPeerId
|
|
774
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 57, S: this });
|
|
775
|
+
return Effect3.void;
|
|
776
|
+
}
|
|
625
777
|
this.#handlers.delete(requestId);
|
|
778
|
+
log2("feed sync client rpc completed", {
|
|
779
|
+
requestId,
|
|
780
|
+
tag: message._tag,
|
|
781
|
+
blockCount: "blocks" in message ? message.blocks.length : void 0,
|
|
782
|
+
positionCount: "positions" in message ? message.positions.length : void 0,
|
|
783
|
+
error: message._tag === "Error" ? message.message : void 0
|
|
784
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 65, S: this });
|
|
626
785
|
if (message._tag === "Error") {
|
|
627
786
|
return Effect3.andThen(Deferred.fail(deferred, new Error(message.message)), () => Effect3.void);
|
|
628
787
|
}
|
|
629
788
|
return Effect3.andThen(Deferred.succeed(deferred, message), () => Effect3.void);
|
|
630
789
|
}
|
|
631
|
-
|
|
790
|
+
/** Removes the pending handler entry and unregisters the ctx onDispose hook. Idempotent. */
|
|
791
|
+
#disposeHandler(requestId, cleanupDispose) {
|
|
792
|
+
cleanupDispose();
|
|
793
|
+
this.#handlers.delete(requestId);
|
|
794
|
+
}
|
|
795
|
+
#awaitRpcResponse(requestId, deferred, cleanupDispose, meta) {
|
|
796
|
+
const self = this;
|
|
797
|
+
const timeoutMs = self.#rpcTimeoutMs;
|
|
798
|
+
return Effect3.ensuring(Deferred.await(deferred).pipe(Effect3.timeoutFail({
|
|
799
|
+
duration: timeoutMs,
|
|
800
|
+
onTimeout: () => new SyncRpcTimeoutError({
|
|
801
|
+
requestId,
|
|
802
|
+
spaceId: meta.spaceId,
|
|
803
|
+
feedNamespace: meta.feedNamespace,
|
|
804
|
+
rpcTag: meta.rpcTag,
|
|
805
|
+
timeoutMs
|
|
806
|
+
})
|
|
807
|
+
}), Effect3.tapError((cause) => Effect3.sync(() => {
|
|
808
|
+
if (cause instanceof SyncRpcTimeoutError) {
|
|
809
|
+
log2("feed sync client rpc timed out", {
|
|
810
|
+
requestId,
|
|
811
|
+
spaceId: meta.spaceId,
|
|
812
|
+
feedNamespace: meta.feedNamespace,
|
|
813
|
+
rpcTag: meta.rpcTag,
|
|
814
|
+
timeoutMs
|
|
815
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 95, S: this });
|
|
816
|
+
}
|
|
817
|
+
}))), Effect3.sync(() => self.#disposeHandler(requestId, cleanupDispose)));
|
|
818
|
+
}
|
|
819
|
+
pull(ctx, opts) {
|
|
632
820
|
const self = this;
|
|
633
821
|
return Effect3.gen(function* () {
|
|
634
822
|
const lastPulledPosition = yield* self.#feedStore.getSyncState({
|
|
@@ -638,6 +826,13 @@ var SyncClient = class {
|
|
|
638
826
|
const requestId = crypto.randomUUID();
|
|
639
827
|
const deferred = yield* Deferred.make();
|
|
640
828
|
self.#handlers.set(requestId, deferred);
|
|
829
|
+
const cleanupDispose = ctx.disposed ? () => {
|
|
830
|
+
} : ctx.onDispose(() => {
|
|
831
|
+
Effect3.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
832
|
+
});
|
|
833
|
+
if (ctx.disposed) {
|
|
834
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
835
|
+
}
|
|
641
836
|
const request = {
|
|
642
837
|
_tag: "QueryRequest",
|
|
643
838
|
requestId,
|
|
@@ -646,10 +841,26 @@ var SyncClient = class {
|
|
|
646
841
|
position: lastPulledPosition,
|
|
647
842
|
limit: opts.limit
|
|
648
843
|
};
|
|
649
|
-
|
|
650
|
-
|
|
844
|
+
log2("feed sync client pull rpc sending", {
|
|
845
|
+
requestId,
|
|
846
|
+
spaceId: opts.spaceId,
|
|
847
|
+
feedNamespace: opts.feedNamespace,
|
|
848
|
+
afterPosition: lastPulledPosition,
|
|
849
|
+
limit: opts.limit
|
|
850
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 129, S: this });
|
|
851
|
+
yield* self.#sendMessage(ctx, self.#withPeerIds(request)).pipe(Effect3.tapErrorCause(() => Effect3.sync(() => self.#disposeHandler(requestId, cleanupDispose))));
|
|
852
|
+
const message = yield* self.#awaitRpcResponse(requestId, deferred, cleanupDispose, {
|
|
853
|
+
spaceId: opts.spaceId,
|
|
854
|
+
feedNamespace: opts.feedNamespace,
|
|
855
|
+
rpcTag: "QueryRequest"
|
|
856
|
+
});
|
|
651
857
|
const response = yield* self.#expectResponse(requestId, message, "QueryResponse");
|
|
652
858
|
if (response.blocks.length === 0) {
|
|
859
|
+
log2.trace("feed sync client pull done (empty batch)", {
|
|
860
|
+
requestId,
|
|
861
|
+
spaceId: opts.spaceId,
|
|
862
|
+
feedNamespace: opts.feedNamespace
|
|
863
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 144, S: this });
|
|
653
864
|
return {
|
|
654
865
|
done: true
|
|
655
866
|
};
|
|
@@ -665,12 +876,61 @@ var SyncClient = class {
|
|
|
665
876
|
feedNamespace: opts.feedNamespace,
|
|
666
877
|
lastPulledPosition: maxPulledPosition
|
|
667
878
|
});
|
|
879
|
+
log2("feed sync client pull applied batch", {
|
|
880
|
+
requestId,
|
|
881
|
+
spaceId: opts.spaceId,
|
|
882
|
+
feedNamespace: opts.feedNamespace,
|
|
883
|
+
batchSize: response.blocks.length,
|
|
884
|
+
hasMore: response.hasMore,
|
|
885
|
+
maxPulledPosition
|
|
886
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 165, S: this });
|
|
668
887
|
return {
|
|
669
888
|
done: false
|
|
670
889
|
};
|
|
671
890
|
});
|
|
672
891
|
}
|
|
673
|
-
|
|
892
|
+
/**
|
|
893
|
+
* Probes remote for blocks after the last pulled position without mutating local storage.
|
|
894
|
+
* Returns the number of blocks in the first batch (0 when caught up with remote).
|
|
895
|
+
*/
|
|
896
|
+
peekPull(ctx, opts) {
|
|
897
|
+
const self = this;
|
|
898
|
+
return Effect3.gen(function* () {
|
|
899
|
+
const lastPulledPosition = yield* self.#feedStore.getSyncState({
|
|
900
|
+
spaceId: opts.spaceId,
|
|
901
|
+
feedNamespace: opts.feedNamespace
|
|
902
|
+
});
|
|
903
|
+
const requestId = crypto.randomUUID();
|
|
904
|
+
const deferred = yield* Deferred.make();
|
|
905
|
+
self.#handlers.set(requestId, deferred);
|
|
906
|
+
const cleanupDispose = ctx.disposed ? () => {
|
|
907
|
+
} : ctx.onDispose(() => {
|
|
908
|
+
Effect3.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
909
|
+
});
|
|
910
|
+
if (ctx.disposed) {
|
|
911
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
912
|
+
}
|
|
913
|
+
const request = {
|
|
914
|
+
_tag: "QueryRequest",
|
|
915
|
+
requestId,
|
|
916
|
+
spaceId: opts.spaceId,
|
|
917
|
+
feedNamespace: opts.feedNamespace,
|
|
918
|
+
position: lastPulledPosition,
|
|
919
|
+
limit: opts.limit
|
|
920
|
+
};
|
|
921
|
+
yield* self.#sendMessage(ctx, self.#withPeerIds(request)).pipe(Effect3.tapErrorCause(() => Effect3.sync(() => self.#disposeHandler(requestId, cleanupDispose))));
|
|
922
|
+
const message = yield* self.#awaitRpcResponse(requestId, deferred, cleanupDispose, {
|
|
923
|
+
spaceId: opts.spaceId,
|
|
924
|
+
feedNamespace: opts.feedNamespace,
|
|
925
|
+
rpcTag: "QueryRequest"
|
|
926
|
+
});
|
|
927
|
+
const response = yield* self.#expectResponse(requestId, message, "QueryResponse");
|
|
928
|
+
return {
|
|
929
|
+
blocksToPull: response.blocks.length
|
|
930
|
+
};
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
push(ctx, opts) {
|
|
674
934
|
const self = this;
|
|
675
935
|
return Effect3.gen(function* () {
|
|
676
936
|
const unpositioned = yield* self.#feedStore.query({
|
|
@@ -680,6 +940,10 @@ var SyncClient = class {
|
|
|
680
940
|
limit: opts.limit
|
|
681
941
|
});
|
|
682
942
|
if (unpositioned.blocks.length === 0) {
|
|
943
|
+
log2.trace("feed sync client push skipped (nothing to send)", {
|
|
944
|
+
spaceId: opts.spaceId,
|
|
945
|
+
feedNamespace: opts.feedNamespace
|
|
946
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 227, S: this });
|
|
683
947
|
return {
|
|
684
948
|
done: true
|
|
685
949
|
};
|
|
@@ -687,6 +951,13 @@ var SyncClient = class {
|
|
|
687
951
|
const requestId = crypto.randomUUID();
|
|
688
952
|
const deferred = yield* Deferred.make();
|
|
689
953
|
self.#handlers.set(requestId, deferred);
|
|
954
|
+
const cleanupDispose = ctx.disposed ? () => {
|
|
955
|
+
} : ctx.onDispose(() => {
|
|
956
|
+
Effect3.runFork(Deferred.fail(deferred, new ContextDisposedError()));
|
|
957
|
+
});
|
|
958
|
+
if (ctx.disposed) {
|
|
959
|
+
yield* Deferred.fail(deferred, new ContextDisposedError());
|
|
960
|
+
}
|
|
690
961
|
const request = {
|
|
691
962
|
_tag: "AppendRequest",
|
|
692
963
|
requestId,
|
|
@@ -694,8 +965,18 @@ var SyncClient = class {
|
|
|
694
965
|
feedNamespace: opts.feedNamespace,
|
|
695
966
|
blocks: unpositioned.blocks
|
|
696
967
|
};
|
|
697
|
-
|
|
698
|
-
|
|
968
|
+
log2("feed sync client push rpc sending", {
|
|
969
|
+
requestId,
|
|
970
|
+
spaceId: opts.spaceId,
|
|
971
|
+
feedNamespace: opts.feedNamespace,
|
|
972
|
+
blockCount: unpositioned.blocks.length
|
|
973
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 251, S: this });
|
|
974
|
+
yield* self.#sendMessage(ctx, self.#withPeerIds(request)).pipe(Effect3.tapErrorCause(() => Effect3.sync(() => self.#disposeHandler(requestId, cleanupDispose))));
|
|
975
|
+
const message = yield* self.#awaitRpcResponse(requestId, deferred, cleanupDispose, {
|
|
976
|
+
spaceId: opts.spaceId,
|
|
977
|
+
feedNamespace: opts.feedNamespace,
|
|
978
|
+
rpcTag: "AppendRequest"
|
|
979
|
+
});
|
|
699
980
|
const response = yield* self.#expectResponse(requestId, message, "AppendResponse");
|
|
700
981
|
yield* self.#feedStore.setPosition({
|
|
701
982
|
spaceId: opts.spaceId,
|
|
@@ -707,6 +988,12 @@ var SyncClient = class {
|
|
|
707
988
|
position
|
|
708
989
|
}))
|
|
709
990
|
});
|
|
991
|
+
log2("feed sync client push positions applied", {
|
|
992
|
+
requestId,
|
|
993
|
+
spaceId: opts.spaceId,
|
|
994
|
+
feedNamespace: opts.feedNamespace,
|
|
995
|
+
positionCount: response.positions.length
|
|
996
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 274, S: this });
|
|
710
997
|
return {
|
|
711
998
|
done: false
|
|
712
999
|
};
|
|
@@ -725,11 +1012,11 @@ var SyncClient = class {
|
|
|
725
1012
|
};
|
|
726
1013
|
|
|
727
1014
|
export {
|
|
728
|
-
|
|
729
|
-
DuplicateBlockLamportTimestampError,
|
|
1015
|
+
SyncRpcTimeoutError,
|
|
730
1016
|
PositionConflictError,
|
|
731
1017
|
FeedStore,
|
|
732
1018
|
SyncServer,
|
|
1019
|
+
DEFAULT_SYNC_RPC_TIMEOUT_MS,
|
|
733
1020
|
SyncClient
|
|
734
1021
|
};
|
|
735
|
-
//# sourceMappingURL=chunk-
|
|
1022
|
+
//# sourceMappingURL=chunk-UC5ETQIJ.mjs.map
|