@dxos/feed 0.8.4-main.fcc0d83b33 → 0.8.4-staging.60fe92afc8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/feed",
3
- "version": "0.8.4-main.fcc0d83b33",
3
+ "version": "0.8.4-staging.60fe92afc8",
4
4
  "description": "Append-only replicatable log",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -8,7 +8,7 @@
8
8
  "type": "git",
9
9
  "url": "https://github.com/dxos/dxos"
10
10
  },
11
- "license": "MIT",
11
+ "license": "FSL-1.1-Apache-2.0",
12
12
  "author": "info@dxos.org",
13
13
  "sideEffects": true,
14
14
  "type": "module",
@@ -30,19 +30,19 @@
30
30
  "src"
31
31
  ],
32
32
  "dependencies": {
33
- "@effect/sql": "0.49.0",
34
- "@effect/sql-sqlite-node": "0.50.1",
35
- "effect": "3.20.0",
36
- "@dxos/async": "0.8.4-main.fcc0d83b33",
37
- "@dxos/effect": "0.8.4-main.fcc0d83b33",
38
- "@dxos/context": "0.8.4-main.fcc0d83b33",
39
- "@dxos/errors": "0.8.4-main.fcc0d83b33",
40
- "@dxos/invariant": "0.8.4-main.fcc0d83b33",
41
- "@dxos/keys": "0.8.4-main.fcc0d83b33",
42
- "@dxos/log": "0.8.4-main.fcc0d83b33",
43
- "@dxos/protocols": "0.8.4-main.fcc0d83b33",
44
- "@dxos/sql-sqlite": "0.8.4-main.fcc0d83b33",
45
- "@dxos/util": "0.8.4-main.fcc0d83b33"
33
+ "@effect/sql": "0.51.1",
34
+ "@effect/sql-sqlite-node": "0.52.0",
35
+ "effect": "3.21.3",
36
+ "@dxos/async": "0.8.4-staging.60fe92afc8",
37
+ "@dxos/context": "0.8.4-staging.60fe92afc8",
38
+ "@dxos/effect": "0.8.4-staging.60fe92afc8",
39
+ "@dxos/invariant": "0.8.4-staging.60fe92afc8",
40
+ "@dxos/log": "0.8.4-staging.60fe92afc8",
41
+ "@dxos/keys": "0.8.4-staging.60fe92afc8",
42
+ "@dxos/sql-sqlite": "0.8.4-staging.60fe92afc8",
43
+ "@dxos/util": "0.8.4-staging.60fe92afc8",
44
+ "@dxos/errors": "0.8.4-staging.60fe92afc8",
45
+ "@dxos/protocols": "0.8.4-staging.60fe92afc8"
46
46
  },
47
47
  "devDependencies": {
48
48
  "@effect/vitest": "0.29.0"
package/src/errors.ts CHANGED
@@ -4,6 +4,15 @@
4
4
 
5
5
  import { BaseError } from '@dxos/errors';
6
6
 
7
+ export class SyncRpcTimeoutError extends BaseError.extend('SyncRpcTimeoutError') {
8
+ constructor(args: { requestId: string; spaceId: string; feedNamespace: string; rpcTag: string; timeoutMs: number }) {
9
+ super({
10
+ message: `Feed sync ${args.rpcTag} timed out after ${args.timeoutMs}ms (spaceId=${args.spaceId} feedNamespace=${args.feedNamespace} requestId=${args.requestId}).`,
11
+ context: args,
12
+ });
13
+ }
14
+ }
15
+
7
16
  export class PositionConflictError extends BaseError.extend('PositionConflictError') {
8
17
  constructor(args: {
9
18
  feedId?: string;
@@ -7,7 +7,7 @@ import { describe, expect, it } from '@effect/vitest';
7
7
  import * as Effect from 'effect/Effect';
8
8
  import * as Layer from 'effect/Layer';
9
9
 
10
- import { ObjectId, SpaceId } from '@dxos/keys';
10
+ import { EntityId, SpaceId } from '@dxos/keys';
11
11
  import { log } from '@dxos/log';
12
12
  import { FeedProtocol } from '@dxos/protocols';
13
13
  import { SqlTransaction } from '@dxos/sql-sqlite';
@@ -33,7 +33,7 @@ describe('Feed V2', () => {
33
33
  it.effect('should append and query blocks via RPC', () =>
34
34
  Effect.gen(function* () {
35
35
  const spaceId = SpaceId.random();
36
- const feedId = ObjectId.random();
36
+ const feedId = EntityId.random();
37
37
 
38
38
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
39
39
  yield* feed.migrate();
@@ -78,7 +78,7 @@ describe('Feed V2', () => {
78
78
  it.effect('should persist feed namespace', () =>
79
79
  Effect.gen(function* () {
80
80
  const spaceId = SpaceId.random();
81
- const feedId = ObjectId.random();
81
+ const feedId = EntityId.random();
82
82
  const feedNamespace = WellKnownNamespaces.data;
83
83
 
84
84
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
@@ -111,7 +111,7 @@ describe('Feed V2', () => {
111
111
  it.effect('should use subscriptions', () =>
112
112
  Effect.gen(function* () {
113
113
  const spaceId = SpaceId.random();
114
- const feedId = ObjectId.random();
114
+ const feedId = EntityId.random();
115
115
 
116
116
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
117
117
  yield* feed.migrate();
@@ -156,7 +156,7 @@ describe('Feed V2', () => {
156
156
  it.effect('should allow position query with unpositionedOnly false', () =>
157
157
  Effect.gen(function* () {
158
158
  const spaceId = SpaceId.random();
159
- const feedId = ObjectId.random();
159
+ const feedId = EntityId.random();
160
160
 
161
161
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
162
162
  yield* feed.migrate();
@@ -346,7 +346,7 @@ describe('Feed V2', () => {
346
346
  it.effect('append local', () =>
347
347
  Effect.gen(function* () {
348
348
  const spaceId = SpaceId.random();
349
- const feedId = ObjectId.random();
349
+ const feedId = EntityId.random();
350
350
 
351
351
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
352
352
  yield* feed.migrate();
@@ -413,7 +413,7 @@ describe('Feed V2', () => {
413
413
  const nsTrace = WellKnownNamespaces.trace;
414
414
 
415
415
  // First append in each (spaceId, feedNamespace) pair starts at 0.
416
- const feedAData = ObjectId.random();
416
+ const feedAData = EntityId.random();
417
417
  const firstAData = yield* feed.appendLocal([
418
418
  {
419
419
  spaceId: spaceA,
@@ -422,7 +422,7 @@ describe('Feed V2', () => {
422
422
  data: new Uint8Array([1]),
423
423
  },
424
424
  ]);
425
- const feedATrace = ObjectId.random();
425
+ const feedATrace = EntityId.random();
426
426
  const firstATrace = yield* feed.appendLocal([
427
427
  {
428
428
  spaceId: spaceA,
@@ -431,7 +431,7 @@ describe('Feed V2', () => {
431
431
  data: new Uint8Array([2]),
432
432
  },
433
433
  ]);
434
- const feedBData = ObjectId.random();
434
+ const feedBData = EntityId.random();
435
435
  const firstBData = yield* feed.appendLocal([
436
436
  {
437
437
  spaceId: spaceB,
@@ -440,7 +440,7 @@ describe('Feed V2', () => {
440
440
  data: new Uint8Array([3]),
441
441
  },
442
442
  ]);
443
- const feedBTrace = ObjectId.random();
443
+ const feedBTrace = EntityId.random();
444
444
  const firstBHalo = yield* feed.appendLocal([
445
445
  {
446
446
  spaceId: spaceB,
@@ -451,7 +451,7 @@ describe('Feed V2', () => {
451
451
  ]);
452
452
 
453
453
  // A second append in one pair should only advance that specific pair.
454
- const feedAData2 = ObjectId.random();
454
+ const feedAData2 = EntityId.random();
455
455
  const secondAData = yield* feed.appendLocal([
456
456
  {
457
457
  spaceId: spaceA,
@@ -473,7 +473,7 @@ describe('Feed V2', () => {
473
473
  it.effect('tailing a feed', () =>
474
474
  Effect.gen(function* () {
475
475
  const spaceId = SpaceId.random();
476
- const feedId = ObjectId.random();
476
+ const feedId = EntityId.random();
477
477
 
478
478
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
479
479
  yield* feed.migrate();
@@ -649,10 +649,69 @@ describe('Feed V2', () => {
649
649
  }).pipe(Effect.provide(TestLayer)),
650
650
  );
651
651
 
652
+ it.effect('append returns existing positions for duplicate blocks and never wastes positions', () =>
653
+ Effect.gen(function* () {
654
+ const spaceId = SpaceId.random();
655
+ const feedId = EntityId.random();
656
+ const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
657
+ yield* feed.migrate();
658
+
659
+ const makeBlock = (sequence: number): Block =>
660
+ Block.make({
661
+ feedId,
662
+ actorId: ALICE,
663
+ sequence,
664
+ prevActorId: null,
665
+ prevSequence: null,
666
+ position: null,
667
+ timestamp: Date.now(),
668
+ data: new Uint8Array([sequence]),
669
+ });
670
+
671
+ const firstBlocks = [makeBlock(0), makeBlock(1), makeBlock(2)];
672
+ const firstAppend = yield* feed.append({
673
+ requestId: 'req-first',
674
+ spaceId,
675
+ feedNamespace: WellKnownNamespaces.data,
676
+ blocks: firstBlocks,
677
+ });
678
+ expect(firstAppend.positions).toEqual([0, 1, 2]);
679
+
680
+ // Re-appending the same blocks must return their existing positions and not advance the
681
+ // namespace position counter; otherwise the client would try to UPDATE blocks to wasted
682
+ // positions and hit the (feedPrivateId, position) UNIQUE constraint.
683
+ const secondAppend = yield* feed.append({
684
+ requestId: 'req-second',
685
+ spaceId,
686
+ feedNamespace: WellKnownNamespaces.data,
687
+ blocks: firstBlocks,
688
+ });
689
+ expect(secondAppend.positions).toEqual([0, 1, 2]);
690
+
691
+ // A subsequent append of new blocks should resume from position 3, not from a wasted slot.
692
+ const newAppend = yield* feed.append({
693
+ requestId: 'req-new',
694
+ spaceId,
695
+ feedNamespace: WellKnownNamespaces.data,
696
+ blocks: [makeBlock(3)],
697
+ });
698
+ expect(newAppend.positions).toEqual([3]);
699
+
700
+ const all = yield* feed.query({
701
+ requestId: 'req-query',
702
+ spaceId,
703
+ feedNamespace: WellKnownNamespaces.data,
704
+ query: { feedIds: [feedId] },
705
+ position: -1,
706
+ });
707
+ expect(all.blocks.map((block) => block.position)).toEqual([0, 1, 2, 3]);
708
+ }).pipe(Effect.provide(TestLayer)),
709
+ );
710
+
652
711
  it.effect('append ignores duplicate Lamport timestamp and preserves original data', () =>
653
712
  Effect.gen(function* () {
654
713
  const spaceId = SpaceId.random();
655
- const feedId = ObjectId.random();
714
+ const feedId = EntityId.random();
656
715
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: false });
657
716
  yield* feed.migrate();
658
717
 
package/src/feed-store.ts CHANGED
@@ -358,6 +358,45 @@ export class FeedStore {
358
358
  `;
359
359
  }).pipe(Effect.withSpan('FeedStore.setSyncState'));
360
360
 
361
+ /**
362
+ * Returns the number of blocks pending push (no global position yet) in a space/namespace.
363
+ */
364
+ countUnpositionedBlocks = (opts: {
365
+ spaceId: SpaceId;
366
+ feedNamespace: string;
367
+ }): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
368
+ Effect.gen(this, function* () {
369
+ const sql = yield* SqlClient.SqlClient;
370
+ const rows = yield* sql<{ count: number }>`
371
+ SELECT COUNT(*) AS count
372
+ FROM blocks
373
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
374
+ WHERE feeds.spaceId = ${opts.spaceId}
375
+ AND feeds.feedNamespace = ${opts.feedNamespace}
376
+ AND blocks.position IS NULL
377
+ `;
378
+ return rows[0]?.count ?? 0;
379
+ }).pipe(Effect.withSpan('FeedStore.countUnpositionedBlocks'));
380
+
381
+ /**
382
+ * Returns the total number of blocks stored locally for a space/namespace.
383
+ */
384
+ countNamespaceBlocks = (opts: {
385
+ spaceId: SpaceId;
386
+ feedNamespace: string;
387
+ }): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
388
+ Effect.gen(this, function* () {
389
+ const sql = yield* SqlClient.SqlClient;
390
+ const rows = yield* sql<{ count: number }>`
391
+ SELECT COUNT(*) AS count
392
+ FROM blocks
393
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
394
+ WHERE feeds.spaceId = ${opts.spaceId}
395
+ AND feeds.feedNamespace = ${opts.feedNamespace}
396
+ `;
397
+ return rows[0]?.count ?? 0;
398
+ }).pipe(Effect.withSpan('FeedStore.countNamespaceBlocks'));
399
+
361
400
  /**
362
401
  * Returns the number of stored blocks for a single feed in a space/namespace.
363
402
  * Intended as a low-level primitive for callers (for example Cloudflare Worker code)
@@ -480,6 +519,12 @@ export class FeedStore {
480
519
  }
481
520
 
482
521
  // 3. Insert all blocks and compute positions.
522
+ //
523
+ // Critical: when a block already exists (conflict on (feedPrivateId, sequence, actorId))
524
+ // we must (a) return the EXISTING position to the caller and (b) NOT advance the
525
+ // namespace position counter. Otherwise the response contains wasted slots that
526
+ // sync clients will try to UPDATE local rows to, tripping the
527
+ // (feedPrivateId, position) UNIQUE constraint and stalling sync indefinitely.
483
528
  const positions: number[] = [];
484
529
  for (const block of request.blocks) {
485
530
  const key = block.feedId!;
@@ -487,23 +532,57 @@ export class FeedStore {
487
532
 
488
533
  let positionToInsert: number | null = null;
489
534
  if (this.#options.assignPositions) {
490
- const currentMax = maxPositions.get(request.feedNamespace)!;
491
- positionToInsert = currentMax + 1;
492
- maxPositions.set(request.feedNamespace, positionToInsert); // Increment for next block in same namespace.
493
- positions.push(positionToInsert);
535
+ positionToInsert = maxPositions.get(request.feedNamespace)! + 1;
494
536
  } else if (block.position != null) {
495
537
  positionToInsert = block.position;
496
538
  }
497
539
 
498
- yield* sql`
540
+ const inserted = yield* sql<{ position: number | null }>`
499
541
  INSERT INTO blocks (
500
- feedPrivateId, position, sequence, actorId,
542
+ feedPrivateId, position, sequence, actorId,
501
543
  prevSequence, prevActorId, timestamp, data
502
544
  ) VALUES (
503
545
  ${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
504
546
  ${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
505
- ) ON CONFLICT DO NOTHING
547
+ )
548
+ ON CONFLICT(feedPrivateId, sequence, actorId) DO NOTHING
549
+ RETURNING position
506
550
  `;
551
+
552
+ if (!this.#options.assignPositions) {
553
+ continue;
554
+ }
555
+
556
+ if (inserted.length > 0) {
557
+ // New row written at the freshly allocated position.
558
+ positions.push(positionToInsert!);
559
+ maxPositions.set(request.feedNamespace, positionToInsert!);
560
+ continue;
561
+ }
562
+
563
+ // Duplicate Lamport tuple: return the stored position and leave maxPositions alone.
564
+ const existing = yield* sql<{ position: number | null }>`
565
+ SELECT position FROM blocks
566
+ WHERE feedPrivateId = ${feedPrivateId}
567
+ AND actorId = ${block.actorId}
568
+ AND sequence = ${block.sequence}
569
+ `;
570
+ const existingPosition = existing[0]?.position;
571
+ if (existingPosition != null) {
572
+ positions.push(existingPosition);
573
+ continue;
574
+ }
575
+
576
+ // Defensive: existing row carries no position (e.g. inserted via an earlier
577
+ // assignPositions=false path). Back-fill it so the caller can mark it positioned.
578
+ yield* sql`
579
+ UPDATE blocks SET position = ${positionToInsert}
580
+ WHERE feedPrivateId = ${feedPrivateId}
581
+ AND actorId = ${block.actorId}
582
+ AND sequence = ${block.sequence}
583
+ `;
584
+ positions.push(positionToInsert!);
585
+ maxPositions.set(request.feedNamespace, positionToInsert!);
507
586
  }
508
587
 
509
588
  return positions;
@@ -647,6 +726,23 @@ export class FeedStore {
647
726
  Effect.gen(this, function* () {
648
727
  const sql = yield* SqlClient.SqlClient;
649
728
  for (const block of request.blocks) {
729
+ // Fold the conflict check into the UPDATE itself: only write when the row is
730
+ // unset or already at the requested position. RETURNING tells us whether the
731
+ // write took effect, so we avoid the per-block SELECT round-trip on the
732
+ // common path. Conflicts (rare) take the slow path below.
733
+ const updated = yield* sql<{ position: number | null }>`
734
+ UPDATE blocks SET position = ${block.position}
735
+ WHERE feedPrivateId = (
736
+ SELECT feedPrivateId FROM feeds
737
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
738
+ )
739
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
740
+ AND (position IS NULL OR position = ${block.position})
741
+ RETURNING position
742
+ `;
743
+ if (updated.length > 0) {
744
+ continue;
745
+ }
650
746
  const existing = yield* sql<{ position: number | null }>`
651
747
  SELECT position FROM blocks
652
748
  WHERE feedPrivateId = (
@@ -667,14 +763,6 @@ export class FeedStore {
667
763
  }),
668
764
  );
669
765
  }
670
- yield* sql`
671
- UPDATE blocks SET position = ${block.position}
672
- WHERE feedPrivateId = (
673
- SELECT feedPrivateId FROM feeds
674
- WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
675
- )
676
- AND actorId = ${block.actorId} AND sequence = ${block.sequence}
677
- `;
678
766
  }
679
767
  }).pipe(Effect.withSpan('FeedStore.setPosition'));
680
768
 
@@ -0,0 +1,63 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
6
+ import * as Cause from 'effect/Cause';
7
+ import * as Effect from 'effect/Effect';
8
+ import * as Exit from 'effect/Exit';
9
+ import * as Layer from 'effect/Layer';
10
+ import * as ManagedRuntime from 'effect/ManagedRuntime';
11
+ import * as Option from 'effect/Option';
12
+ import { describe, expect, onTestFinished, test } from 'vitest';
13
+
14
+ import { Context } from '@dxos/context';
15
+ import { SpaceId } from '@dxos/keys';
16
+ import { FeedProtocol } from '@dxos/protocols';
17
+ import { SqlTransaction } from '@dxos/sql-sqlite';
18
+
19
+ import { SyncRpcTimeoutError } from './errors';
20
+ import { FeedStore } from './feed-store';
21
+ import { SyncClient } from './sync-client';
22
+
23
+ const WellKnownNamespaces = FeedProtocol.WellKnownNamespaces;
24
+
25
+ const TestLayer = SqlTransaction.layer.pipe(
26
+ Layer.provideMerge(
27
+ SqliteClient.layer({
28
+ filename: ':memory:',
29
+ }),
30
+ ),
31
+ );
32
+
33
+ describe('SyncClient', () => {
34
+ test('times out when edge does not respond', async () => {
35
+ const runtime = ManagedRuntime.make(TestLayer);
36
+ const spaceId = SpaceId.random();
37
+ const feedStore = new FeedStore({ localActorId: 'alice', assignPositions: false });
38
+ await runtime.runPromise(feedStore.migrate());
39
+
40
+ const syncClient = new SyncClient({
41
+ peerId: 'client-peer',
42
+ feedStore,
43
+ rpcTimeoutMs: 50,
44
+ sendMessage: () => Effect.void,
45
+ });
46
+
47
+ const ctx = new Context();
48
+ onTestFinished(() => void ctx.dispose());
49
+
50
+ const exit = await runtime.runPromiseExit(
51
+ syncClient.peekPull(ctx, {
52
+ spaceId,
53
+ feedNamespace: WellKnownNamespaces.data,
54
+ }),
55
+ );
56
+
57
+ expect(Exit.isFailure(exit)).toBe(true);
58
+ const error = Exit.isFailure(exit) ? Cause.failureOption(exit.cause) : Option.none();
59
+ expect(Option.isSome(error) && error.value instanceof SyncRpcTimeoutError).toBe(true);
60
+
61
+ await runtime.dispose();
62
+ });
63
+ });