@dxos/feed 0.8.4-staging.ac66bdf99f → 0.9.1-main.c7dcc2e112

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/feed",
3
- "version": "0.8.4-staging.ac66bdf99f",
3
+ "version": "0.9.1-main.c7dcc2e112",
4
4
  "description": "Append-only replicatable log",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -8,7 +8,7 @@
8
8
  "type": "git",
9
9
  "url": "https://github.com/dxos/dxos"
10
10
  },
11
- "license": "MIT",
11
+ "license": "FSL-1.1-Apache-2.0",
12
12
  "author": "info@dxos.org",
13
13
  "sideEffects": true,
14
14
  "type": "module",
@@ -25,34 +25,27 @@
25
25
  }
26
26
  },
27
27
  "types": "dist/types/src/index.d.ts",
28
- "typesVersions": {
29
- "*": {
30
- "testing": [
31
- "dist/types/src/testing/index.d.ts"
32
- ]
33
- }
34
- },
35
28
  "files": [
36
29
  "dist",
37
30
  "src"
38
31
  ],
39
32
  "dependencies": {
40
- "@effect/sql": "0.49.0",
41
- "@effect/sql-sqlite-node": "0.50.1",
42
- "effect": "3.20.0",
43
- "@dxos/async": "0.8.4-staging.ac66bdf99f",
44
- "@dxos/context": "0.8.4-staging.ac66bdf99f",
45
- "@dxos/effect": "0.8.4-staging.ac66bdf99f",
46
- "@dxos/invariant": "0.8.4-staging.ac66bdf99f",
47
- "@dxos/log": "0.8.4-staging.ac66bdf99f",
48
- "@dxos/sql-sqlite": "0.8.4-staging.ac66bdf99f",
49
- "@dxos/errors": "0.8.4-staging.ac66bdf99f",
50
- "@dxos/util": "0.8.4-staging.ac66bdf99f",
51
- "@dxos/protocols": "0.8.4-staging.ac66bdf99f",
52
- "@dxos/keys": "0.8.4-staging.ac66bdf99f"
33
+ "@effect/sql": "0.51.1",
34
+ "@effect/sql-sqlite-node": "0.52.0",
35
+ "effect": "3.21.3",
36
+ "@dxos/async": "0.9.1-main.c7dcc2e112",
37
+ "@dxos/context": "0.9.1-main.c7dcc2e112",
38
+ "@dxos/errors": "0.9.1-main.c7dcc2e112",
39
+ "@dxos/invariant": "0.9.1-main.c7dcc2e112",
40
+ "@dxos/keys": "0.9.1-main.c7dcc2e112",
41
+ "@dxos/effect": "0.9.1-main.c7dcc2e112",
42
+ "@dxos/protocols": "0.9.1-main.c7dcc2e112",
43
+ "@dxos/log": "0.9.1-main.c7dcc2e112",
44
+ "@dxos/util": "0.9.1-main.c7dcc2e112",
45
+ "@dxos/sql-sqlite": "0.9.1-main.c7dcc2e112"
53
46
  },
54
47
  "devDependencies": {
55
- "@effect/vitest": "0.27.0"
48
+ "@effect/vitest": "0.29.0"
56
49
  },
57
50
  "publishConfig": {
58
51
  "access": "public"
package/src/errors.ts CHANGED
@@ -4,6 +4,15 @@
4
4
 
5
5
  import { BaseError } from '@dxos/errors';
6
6
 
7
+ export class SyncRpcTimeoutError extends BaseError.extend('SyncRpcTimeoutError') {
8
+ constructor(args: { requestId: string; spaceId: string; feedNamespace: string; rpcTag: string; timeoutMs: number }) {
9
+ super({
10
+ message: `Feed sync ${args.rpcTag} timed out after ${args.timeoutMs}ms (spaceId=${args.spaceId} feedNamespace=${args.feedNamespace} requestId=${args.requestId}).`,
11
+ context: args,
12
+ });
13
+ }
14
+ }
15
+
7
16
  export class PositionConflictError extends BaseError.extend('PositionConflictError') {
8
17
  constructor(args: {
9
18
  feedId?: string;
@@ -7,7 +7,7 @@ import { describe, expect, it } from '@effect/vitest';
7
7
  import * as Effect from 'effect/Effect';
8
8
  import * as Layer from 'effect/Layer';
9
9
 
10
- import { ObjectId, SpaceId } from '@dxos/keys';
10
+ import { EntityId, SpaceId } from '@dxos/keys';
11
11
  import { log } from '@dxos/log';
12
12
  import { FeedProtocol } from '@dxos/protocols';
13
13
  import { SqlTransaction } from '@dxos/sql-sqlite';
@@ -33,7 +33,7 @@ describe('Feed V2', () => {
33
33
  it.effect('should append and query blocks via RPC', () =>
34
34
  Effect.gen(function* () {
35
35
  const spaceId = SpaceId.random();
36
- const feedId = ObjectId.random();
36
+ const feedId = EntityId.random();
37
37
 
38
38
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
39
39
  yield* feed.migrate();
@@ -78,7 +78,7 @@ describe('Feed V2', () => {
78
78
  it.effect('should persist feed namespace', () =>
79
79
  Effect.gen(function* () {
80
80
  const spaceId = SpaceId.random();
81
- const feedId = ObjectId.random();
81
+ const feedId = EntityId.random();
82
82
  const feedNamespace = WellKnownNamespaces.data;
83
83
 
84
84
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
@@ -111,7 +111,7 @@ describe('Feed V2', () => {
111
111
  it.effect('should use subscriptions', () =>
112
112
  Effect.gen(function* () {
113
113
  const spaceId = SpaceId.random();
114
- const feedId = ObjectId.random();
114
+ const feedId = EntityId.random();
115
115
 
116
116
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
117
117
  yield* feed.migrate();
@@ -156,7 +156,7 @@ describe('Feed V2', () => {
156
156
  it.effect('should allow position query with unpositionedOnly false', () =>
157
157
  Effect.gen(function* () {
158
158
  const spaceId = SpaceId.random();
159
- const feedId = ObjectId.random();
159
+ const feedId = EntityId.random();
160
160
 
161
161
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
162
162
  yield* feed.migrate();
@@ -312,7 +312,9 @@ describe('Feed V2', () => {
312
312
  // Ideally we use Cause.isDie(cause) -> error
313
313
  // But for quick check:
314
314
  let error = cause.value || cause.defect || cause;
315
- if (cause._tag === 'Die') error = cause.value || cause.defect;
315
+ if (cause._tag === 'Die') {
316
+ error = cause.value || cause.defect;
317
+ }
316
318
 
317
319
  expect(error).toBeDefined();
318
320
  expect(error.message).toBe('Cursor token mismatch');
@@ -344,7 +346,7 @@ describe('Feed V2', () => {
344
346
  it.effect('append local', () =>
345
347
  Effect.gen(function* () {
346
348
  const spaceId = SpaceId.random();
347
- const feedId = ObjectId.random();
349
+ const feedId = EntityId.random();
348
350
 
349
351
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
350
352
  yield* feed.migrate();
@@ -411,7 +413,7 @@ describe('Feed V2', () => {
411
413
  const nsTrace = WellKnownNamespaces.trace;
412
414
 
413
415
  // First append in each (spaceId, feedNamespace) pair starts at 0.
414
- const feedAData = ObjectId.random();
416
+ const feedAData = EntityId.random();
415
417
  const firstAData = yield* feed.appendLocal([
416
418
  {
417
419
  spaceId: spaceA,
@@ -420,7 +422,7 @@ describe('Feed V2', () => {
420
422
  data: new Uint8Array([1]),
421
423
  },
422
424
  ]);
423
- const feedATrace = ObjectId.random();
425
+ const feedATrace = EntityId.random();
424
426
  const firstATrace = yield* feed.appendLocal([
425
427
  {
426
428
  spaceId: spaceA,
@@ -429,7 +431,7 @@ describe('Feed V2', () => {
429
431
  data: new Uint8Array([2]),
430
432
  },
431
433
  ]);
432
- const feedBData = ObjectId.random();
434
+ const feedBData = EntityId.random();
433
435
  const firstBData = yield* feed.appendLocal([
434
436
  {
435
437
  spaceId: spaceB,
@@ -438,7 +440,7 @@ describe('Feed V2', () => {
438
440
  data: new Uint8Array([3]),
439
441
  },
440
442
  ]);
441
- const feedBTrace = ObjectId.random();
443
+ const feedBTrace = EntityId.random();
442
444
  const firstBHalo = yield* feed.appendLocal([
443
445
  {
444
446
  spaceId: spaceB,
@@ -449,7 +451,7 @@ describe('Feed V2', () => {
449
451
  ]);
450
452
 
451
453
  // A second append in one pair should only advance that specific pair.
452
- const feedAData2 = ObjectId.random();
454
+ const feedAData2 = EntityId.random();
453
455
  const secondAData = yield* feed.appendLocal([
454
456
  {
455
457
  spaceId: spaceA,
@@ -471,7 +473,7 @@ describe('Feed V2', () => {
471
473
  it.effect('tailing a feed', () =>
472
474
  Effect.gen(function* () {
473
475
  const spaceId = SpaceId.random();
474
- const feedId = ObjectId.random();
476
+ const feedId = EntityId.random();
475
477
 
476
478
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
477
479
  yield* feed.migrate();
@@ -647,10 +649,69 @@ describe('Feed V2', () => {
647
649
  }).pipe(Effect.provide(TestLayer)),
648
650
  );
649
651
 
652
+ it.effect('append returns existing positions for duplicate blocks and never wastes positions', () =>
653
+ Effect.gen(function* () {
654
+ const spaceId = SpaceId.random();
655
+ const feedId = EntityId.random();
656
+ const feed = new FeedStore({ localActorId: ALICE, assignPositions: true });
657
+ yield* feed.migrate();
658
+
659
+ const makeBlock = (sequence: number): Block =>
660
+ Block.make({
661
+ feedId,
662
+ actorId: ALICE,
663
+ sequence,
664
+ prevActorId: null,
665
+ prevSequence: null,
666
+ position: null,
667
+ timestamp: Date.now(),
668
+ data: new Uint8Array([sequence]),
669
+ });
670
+
671
+ const firstBlocks = [makeBlock(0), makeBlock(1), makeBlock(2)];
672
+ const firstAppend = yield* feed.append({
673
+ requestId: 'req-first',
674
+ spaceId,
675
+ feedNamespace: WellKnownNamespaces.data,
676
+ blocks: firstBlocks,
677
+ });
678
+ expect(firstAppend.positions).toEqual([0, 1, 2]);
679
+
680
+ // Re-appending the same blocks must return their existing positions and not advance the
681
+ // namespace position counter; otherwise the client would try to UPDATE blocks to wasted
682
+ // positions and hit the (feedPrivateId, position) UNIQUE constraint.
683
+ const secondAppend = yield* feed.append({
684
+ requestId: 'req-second',
685
+ spaceId,
686
+ feedNamespace: WellKnownNamespaces.data,
687
+ blocks: firstBlocks,
688
+ });
689
+ expect(secondAppend.positions).toEqual([0, 1, 2]);
690
+
691
+ // A subsequent append of new blocks should resume from position 3, not from a wasted slot.
692
+ const newAppend = yield* feed.append({
693
+ requestId: 'req-new',
694
+ spaceId,
695
+ feedNamespace: WellKnownNamespaces.data,
696
+ blocks: [makeBlock(3)],
697
+ });
698
+ expect(newAppend.positions).toEqual([3]);
699
+
700
+ const all = yield* feed.query({
701
+ requestId: 'req-query',
702
+ spaceId,
703
+ feedNamespace: WellKnownNamespaces.data,
704
+ query: { feedIds: [feedId] },
705
+ position: -1,
706
+ });
707
+ expect(all.blocks.map((block) => block.position)).toEqual([0, 1, 2, 3]);
708
+ }).pipe(Effect.provide(TestLayer)),
709
+ );
710
+
650
711
  it.effect('append ignores duplicate Lamport timestamp and preserves original data', () =>
651
712
  Effect.gen(function* () {
652
713
  const spaceId = SpaceId.random();
653
- const feedId = ObjectId.random();
714
+ const feedId = EntityId.random();
654
715
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: false });
655
716
  yield* feed.migrate();
656
717
 
package/src/feed-store.ts CHANGED
@@ -124,7 +124,9 @@ export class FeedStore {
124
124
  const rows = yield* sql<{ feedPrivateId: number }>`
125
125
  SELECT feedPrivateId FROM feeds WHERE spaceId = ${spaceId} AND feedId = ${feedId}
126
126
  `;
127
- if (rows.length > 0) return rows[0].feedPrivateId;
127
+ if (rows.length > 0) {
128
+ return rows[0].feedPrivateId;
129
+ }
128
130
 
129
131
  const newRows = yield* sql<{ feedPrivateId: number }>`
130
132
  INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
@@ -141,7 +143,9 @@ export class FeedStore {
141
143
  Effect.gen(this, function* () {
142
144
  const sql = yield* SqlClient.SqlClient;
143
145
  const rows = yield* sql<{ token: string }>`SELECT token FROM cursor_tokens WHERE spaceId = ${spaceId}`;
144
- if (rows.length > 0) return rows[0].token;
146
+ if (rows.length > 0) {
147
+ return rows[0].token;
148
+ }
145
149
 
146
150
  const token = crypto.randomUUID().replace(/-/g, '').slice(0, 6);
147
151
  yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
@@ -354,6 +358,45 @@ export class FeedStore {
354
358
  `;
355
359
  }).pipe(Effect.withSpan('FeedStore.setSyncState'));
356
360
 
361
+ /**
362
+ * Returns the number of blocks pending push (no global position yet) in a space/namespace.
363
+ */
364
+ countUnpositionedBlocks = (opts: {
365
+ spaceId: SpaceId;
366
+ feedNamespace: string;
367
+ }): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
368
+ Effect.gen(this, function* () {
369
+ const sql = yield* SqlClient.SqlClient;
370
+ const rows = yield* sql<{ count: number }>`
371
+ SELECT COUNT(*) AS count
372
+ FROM blocks
373
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
374
+ WHERE feeds.spaceId = ${opts.spaceId}
375
+ AND feeds.feedNamespace = ${opts.feedNamespace}
376
+ AND blocks.position IS NULL
377
+ `;
378
+ return rows[0]?.count ?? 0;
379
+ }).pipe(Effect.withSpan('FeedStore.countUnpositionedBlocks'));
380
+
381
+ /**
382
+ * Returns the total number of blocks stored locally for a space/namespace.
383
+ */
384
+ countNamespaceBlocks = (opts: {
385
+ spaceId: SpaceId;
386
+ feedNamespace: string;
387
+ }): Effect.Effect<number, SqlError.SqlError, SqlClient.SqlClient> =>
388
+ Effect.gen(this, function* () {
389
+ const sql = yield* SqlClient.SqlClient;
390
+ const rows = yield* sql<{ count: number }>`
391
+ SELECT COUNT(*) AS count
392
+ FROM blocks
393
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
394
+ WHERE feeds.spaceId = ${opts.spaceId}
395
+ AND feeds.feedNamespace = ${opts.feedNamespace}
396
+ `;
397
+ return rows[0]?.count ?? 0;
398
+ }).pipe(Effect.withSpan('FeedStore.countNamespaceBlocks'));
399
+
357
400
  /**
358
401
  * Returns the number of stored blocks for a single feed in a space/namespace.
359
402
  * Intended as a low-level primitive for callers (for example Cloudflare Worker code)
@@ -476,6 +519,12 @@ export class FeedStore {
476
519
  }
477
520
 
478
521
  // 3. Insert all blocks and compute positions.
522
+ //
523
+ // Critical: when a block already exists (conflict on (feedPrivateId, sequence, actorId))
524
+ // we must (a) return the EXISTING position to the caller and (b) NOT advance the
525
+ // namespace position counter. Otherwise the response contains wasted slots that
526
+ // sync clients will try to UPDATE local rows to, tripping the
527
+ // (feedPrivateId, position) UNIQUE constraint and stalling sync indefinitely.
479
528
  const positions: number[] = [];
480
529
  for (const block of request.blocks) {
481
530
  const key = block.feedId!;
@@ -483,23 +532,57 @@ export class FeedStore {
483
532
 
484
533
  let positionToInsert: number | null = null;
485
534
  if (this.#options.assignPositions) {
486
- const currentMax = maxPositions.get(request.feedNamespace)!;
487
- positionToInsert = currentMax + 1;
488
- maxPositions.set(request.feedNamespace, positionToInsert); // Increment for next block in same namespace.
489
- positions.push(positionToInsert);
535
+ positionToInsert = maxPositions.get(request.feedNamespace)! + 1;
490
536
  } else if (block.position != null) {
491
537
  positionToInsert = block.position;
492
538
  }
493
539
 
494
- yield* sql`
540
+ const inserted = yield* sql<{ position: number | null }>`
495
541
  INSERT INTO blocks (
496
- feedPrivateId, position, sequence, actorId,
542
+ feedPrivateId, position, sequence, actorId,
497
543
  prevSequence, prevActorId, timestamp, data
498
544
  ) VALUES (
499
545
  ${feedPrivateId}, ${positionToInsert}, ${block.sequence}, ${block.actorId},
500
546
  ${block.prevSequence}, ${block.prevActorId}, ${block.timestamp}, ${block.data}
501
- ) ON CONFLICT DO NOTHING
547
+ )
548
+ ON CONFLICT(feedPrivateId, sequence, actorId) DO NOTHING
549
+ RETURNING position
502
550
  `;
551
+
552
+ if (!this.#options.assignPositions) {
553
+ continue;
554
+ }
555
+
556
+ if (inserted.length > 0) {
557
+ // New row written at the freshly allocated position.
558
+ positions.push(positionToInsert!);
559
+ maxPositions.set(request.feedNamespace, positionToInsert!);
560
+ continue;
561
+ }
562
+
563
+ // Duplicate Lamport tuple: return the stored position and leave maxPositions alone.
564
+ const existing = yield* sql<{ position: number | null }>`
565
+ SELECT position FROM blocks
566
+ WHERE feedPrivateId = ${feedPrivateId}
567
+ AND actorId = ${block.actorId}
568
+ AND sequence = ${block.sequence}
569
+ `;
570
+ const existingPosition = existing[0]?.position;
571
+ if (existingPosition != null) {
572
+ positions.push(existingPosition);
573
+ continue;
574
+ }
575
+
576
+ // Defensive: existing row carries no position (e.g. inserted via an earlier
577
+ // assignPositions=false path). Back-fill it so the caller can mark it positioned.
578
+ yield* sql`
579
+ UPDATE blocks SET position = ${positionToInsert}
580
+ WHERE feedPrivateId = ${feedPrivateId}
581
+ AND actorId = ${block.actorId}
582
+ AND sequence = ${block.sequence}
583
+ `;
584
+ positions.push(positionToInsert!);
585
+ maxPositions.set(request.feedNamespace, positionToInsert!);
503
586
  }
504
587
 
505
588
  return positions;
@@ -643,6 +726,23 @@ export class FeedStore {
643
726
  Effect.gen(this, function* () {
644
727
  const sql = yield* SqlClient.SqlClient;
645
728
  for (const block of request.blocks) {
729
+ // Fold the conflict check into the UPDATE itself: only write when the row is
730
+ // unset or already at the requested position. RETURNING tells us whether the
731
+ // write took effect, so we avoid the per-block SELECT round-trip on the
732
+ // common path. Conflicts (rare) take the slow path below.
733
+ const updated = yield* sql<{ position: number | null }>`
734
+ UPDATE blocks SET position = ${block.position}
735
+ WHERE feedPrivateId = (
736
+ SELECT feedPrivateId FROM feeds
737
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
738
+ )
739
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
740
+ AND (position IS NULL OR position = ${block.position})
741
+ RETURNING position
742
+ `;
743
+ if (updated.length > 0) {
744
+ continue;
745
+ }
646
746
  const existing = yield* sql<{ position: number | null }>`
647
747
  SELECT position FROM blocks
648
748
  WHERE feedPrivateId = (
@@ -663,14 +763,6 @@ export class FeedStore {
663
763
  }),
664
764
  );
665
765
  }
666
- yield* sql`
667
- UPDATE blocks SET position = ${block.position}
668
- WHERE feedPrivateId = (
669
- SELECT feedPrivateId FROM feeds
670
- WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
671
- )
672
- AND actorId = ${block.actorId} AND sequence = ${block.sequence}
673
- `;
674
766
  }
675
767
  }).pipe(Effect.withSpan('FeedStore.setPosition'));
676
768
 
@@ -0,0 +1,63 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import * as SqliteClient from '@effect/sql-sqlite-node/SqliteClient';
6
+ import * as Cause from 'effect/Cause';
7
+ import * as Effect from 'effect/Effect';
8
+ import * as Exit from 'effect/Exit';
9
+ import * as Layer from 'effect/Layer';
10
+ import * as ManagedRuntime from 'effect/ManagedRuntime';
11
+ import * as Option from 'effect/Option';
12
+ import { describe, expect, onTestFinished, test } from 'vitest';
13
+
14
+ import { Context } from '@dxos/context';
15
+ import { SpaceId } from '@dxos/keys';
16
+ import { FeedProtocol } from '@dxos/protocols';
17
+ import { SqlTransaction } from '@dxos/sql-sqlite';
18
+
19
+ import { SyncRpcTimeoutError } from './errors';
20
+ import { FeedStore } from './feed-store';
21
+ import { SyncClient } from './sync-client';
22
+
23
+ const WellKnownNamespaces = FeedProtocol.WellKnownNamespaces;
24
+
25
+ const TestLayer = SqlTransaction.layer.pipe(
26
+ Layer.provideMerge(
27
+ SqliteClient.layer({
28
+ filename: ':memory:',
29
+ }),
30
+ ),
31
+ );
32
+
33
+ describe('SyncClient', () => {
34
+ test('times out when edge does not respond', async () => {
35
+ const runtime = ManagedRuntime.make(TestLayer);
36
+ const spaceId = SpaceId.random();
37
+ const feedStore = new FeedStore({ localActorId: 'alice', assignPositions: false });
38
+ await runtime.runPromise(feedStore.migrate());
39
+
40
+ const syncClient = new SyncClient({
41
+ peerId: 'client-peer',
42
+ feedStore,
43
+ rpcTimeoutMs: 50,
44
+ sendMessage: () => Effect.void,
45
+ });
46
+
47
+ const ctx = new Context();
48
+ onTestFinished(() => void ctx.dispose());
49
+
50
+ const exit = await runtime.runPromiseExit(
51
+ syncClient.peekPull(ctx, {
52
+ spaceId,
53
+ feedNamespace: WellKnownNamespaces.data,
54
+ }),
55
+ );
56
+
57
+ expect(Exit.isFailure(exit)).toBe(true);
58
+ const error = Exit.isFailure(exit) ? Cause.failureOption(exit.cause) : Option.none();
59
+ expect(Option.isSome(error) && error.value instanceof SyncRpcTimeoutError).toBe(true);
60
+
61
+ await runtime.dispose();
62
+ });
63
+ });