@dxos/feed 0.8.4-main.2244d791bb → 0.8.4-main.3fbcb4aa9b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dxos/feed",
3
- "version": "0.8.4-main.2244d791bb",
3
+ "version": "0.8.4-main.3fbcb4aa9b",
4
4
  "description": "Append-only replicatable log",
5
5
  "homepage": "https://dxos.org",
6
6
  "bugs": "https://github.com/dxos/dxos/issues",
@@ -25,13 +25,6 @@
25
25
  }
26
26
  },
27
27
  "types": "dist/types/src/index.d.ts",
28
- "typesVersions": {
29
- "*": {
30
- "testing": [
31
- "dist/types/src/testing/index.d.ts"
32
- ]
33
- }
34
- },
35
28
  "files": [
36
29
  "dist",
37
30
  "src"
@@ -39,18 +32,20 @@
39
32
  "dependencies": {
40
33
  "@effect/sql": "0.49.0",
41
34
  "@effect/sql-sqlite-node": "0.50.1",
42
- "@effect/vitest": "0.27.0",
43
- "effect": "3.19.16",
44
- "@dxos/async": "0.8.4-main.2244d791bb",
45
- "@dxos/effect": "0.8.4-main.2244d791bb",
46
- "@dxos/context": "0.8.4-main.2244d791bb",
47
- "@dxos/errors": "0.8.4-main.2244d791bb",
48
- "@dxos/keys": "0.8.4-main.2244d791bb",
49
- "@dxos/log": "0.8.4-main.2244d791bb",
50
- "@dxos/sql-sqlite": "0.8.4-main.2244d791bb",
51
- "@dxos/invariant": "0.8.4-main.2244d791bb",
52
- "@dxos/protocols": "0.8.4-main.2244d791bb",
53
- "@dxos/util": "0.8.4-main.2244d791bb"
35
+ "effect": "3.20.0",
36
+ "@dxos/context": "0.8.4-main.3fbcb4aa9b",
37
+ "@dxos/async": "0.8.4-main.3fbcb4aa9b",
38
+ "@dxos/errors": "0.8.4-main.3fbcb4aa9b",
39
+ "@dxos/invariant": "0.8.4-main.3fbcb4aa9b",
40
+ "@dxos/keys": "0.8.4-main.3fbcb4aa9b",
41
+ "@dxos/log": "0.8.4-main.3fbcb4aa9b",
42
+ "@dxos/sql-sqlite": "0.8.4-main.3fbcb4aa9b",
43
+ "@dxos/protocols": "0.8.4-main.3fbcb4aa9b",
44
+ "@dxos/util": "0.8.4-main.3fbcb4aa9b",
45
+ "@dxos/effect": "0.8.4-main.3fbcb4aa9b"
46
+ },
47
+ "devDependencies": {
48
+ "@effect/vitest": "0.29.0"
54
49
  },
55
50
  "publishConfig": {
56
51
  "access": "public"
package/src/errors.ts CHANGED
@@ -4,24 +4,6 @@
4
4
 
5
5
  import { BaseError } from '@dxos/errors';
6
6
 
7
- export class DuplicateBlockPositionError extends BaseError.extend('DuplicateBlockPositionError') {
8
- constructor(args: { feedId: string; position: number }) {
9
- super({
10
- message: `Non-unique block position detected (feedId=${args.feedId} position=${args.position}).`,
11
- context: args,
12
- });
13
- }
14
- }
15
-
16
- export class DuplicateBlockLamportTimestampError extends BaseError.extend('DuplicateBlockLamportTimestampError') {
17
- constructor(args: { feedId: string; actorId: string; sequence: number }) {
18
- super({
19
- message: `Non-unique block Lamport timestamp detected (feedId=${args.feedId} actorId=${args.actorId} sequence=${args.sequence}).`,
20
- context: args,
21
- });
22
- }
23
- }
24
-
25
7
  export class PositionConflictError extends BaseError.extend('PositionConflictError') {
26
8
  constructor(args: {
27
9
  feedId?: string;
@@ -12,7 +12,6 @@ import { log } from '@dxos/log';
12
12
  import { FeedProtocol } from '@dxos/protocols';
13
13
  import { SqlTransaction } from '@dxos/sql-sqlite';
14
14
 
15
- import { DuplicateBlockLamportTimestampError, DuplicateBlockPositionError } from './errors';
16
15
  import { FeedStore } from './feed-store';
17
16
 
18
17
  const Block = FeedProtocol.Block;
@@ -313,7 +312,9 @@ describe('Feed V2', () => {
313
312
  // Ideally we use Cause.isDie(cause) -> error
314
313
  // But for quick check:
315
314
  let error = cause.value || cause.defect || cause;
316
- if (cause._tag === 'Die') error = cause.value || cause.defect;
315
+ if (cause._tag === 'Die') {
316
+ error = cause.value || cause.defect;
317
+ }
317
318
 
318
319
  expect(error).toBeDefined();
319
320
  expect(error.message).toBe('Cursor token mismatch');
@@ -648,87 +649,66 @@ describe('Feed V2', () => {
648
649
  }).pipe(Effect.provide(TestLayer)),
649
650
  );
650
651
 
651
- it.effect('append fails on non-unique position in request', () =>
652
+ it.effect('append ignores duplicate Lamport timestamp and preserves original data', () =>
652
653
  Effect.gen(function* () {
653
654
  const spaceId = SpaceId.random();
654
655
  const feedId = ObjectId.random();
655
656
  const feed = new FeedStore({ localActorId: ALICE, assignPositions: false });
656
657
  yield* feed.migrate();
657
658
 
658
- const error = yield* feed
659
- .append({
660
- requestId: 'req-dup-position',
661
- spaceId,
662
- feedNamespace: WellKnownNamespaces.data,
663
- blocks: [
664
- Block.make({
665
- feedId,
666
- actorId: 'actor-1',
667
- sequence: 1,
668
- prevActorId: null,
669
- prevSequence: null,
670
- position: 10,
671
- timestamp: Date.now(),
672
- data: new Uint8Array([1]),
673
- }),
674
- Block.make({
675
- feedId,
676
- actorId: 'actor-2',
677
- sequence: 1,
678
- prevActorId: null,
679
- prevSequence: null,
680
- position: 10,
681
- timestamp: Date.now(),
682
- data: new Uint8Array([2]),
683
- }),
684
- ],
685
- })
686
- .pipe(Effect.flip);
659
+ const actorId = 'actor-1';
660
+ const sequence = 5;
661
+ const timestamp = Date.now();
687
662
 
688
- expect(error).toBeInstanceOf(DuplicateBlockPositionError);
689
- expect(error.message).toContain('Non-unique block position detected');
690
- }).pipe(Effect.provide(TestLayer)),
691
- );
663
+ yield* feed.append({
664
+ requestId: 'req-first',
665
+ spaceId,
666
+ feedNamespace: WellKnownNamespaces.data,
667
+ blocks: [
668
+ Block.make({
669
+ feedId,
670
+ actorId,
671
+ sequence,
672
+ prevActorId: null,
673
+ prevSequence: null,
674
+ position: null,
675
+ timestamp,
676
+ data: new Uint8Array([1]),
677
+ }),
678
+ ],
679
+ });
692
680
 
693
- it.effect('append fails on non-unique Lamport timestamp in request', () =>
694
- Effect.gen(function* () {
695
- const spaceId = SpaceId.random();
696
- const feedId = ObjectId.random();
697
- const feed = new FeedStore({ localActorId: ALICE, assignPositions: false });
698
- yield* feed.migrate();
681
+ // Same Lamport tuple with different data and position should be ignored.
682
+ yield* feed.append({
683
+ requestId: 'req-duplicate',
684
+ spaceId,
685
+ feedNamespace: WellKnownNamespaces.data,
686
+ blocks: [
687
+ Block.make({
688
+ feedId,
689
+ actorId,
690
+ sequence,
691
+ prevActorId: null,
692
+ prevSequence: null,
693
+ position: null,
694
+ timestamp: timestamp + 1,
695
+ data: new Uint8Array([2]),
696
+ }),
697
+ ],
698
+ });
699
699
 
700
- const error = yield* feed
701
- .append({
702
- requestId: 'req-dup-lamport',
703
- spaceId,
704
- feedNamespace: WellKnownNamespaces.data,
705
- blocks: [
706
- Block.make({
707
- feedId,
708
- actorId: 'actor-1',
709
- sequence: 5,
710
- prevActorId: null,
711
- prevSequence: null,
712
- position: 1,
713
- timestamp: Date.now(),
714
- data: new Uint8Array([1]),
715
- }),
716
- Block.make({
717
- feedId,
718
- actorId: 'actor-1',
719
- sequence: 5,
720
- prevActorId: null,
721
- prevSequence: null,
722
- position: 2,
723
- timestamp: Date.now(),
724
- data: new Uint8Array([2]),
725
- }),
726
- ],
727
- })
728
- .pipe(Effect.flip);
700
+ const queryRes = yield* feed.query({
701
+ requestId: 'req-query',
702
+ spaceId,
703
+ feedNamespace: WellKnownNamespaces.data,
704
+ query: { feedIds: [feedId] },
705
+ position: -1,
706
+ });
729
707
 
730
- expect(error).toBeInstanceOf(DuplicateBlockLamportTimestampError);
731
- expect(error.message).toContain('Non-unique block Lamport timestamp detected');
708
+ expect(queryRes.blocks.length).toBe(1);
709
+ expect(queryRes.blocks[0].actorId).toBe(actorId);
710
+ expect(queryRes.blocks[0].sequence).toBe(sequence);
711
+ expect(queryRes.blocks[0].data).toEqual(new Uint8Array([1]));
732
712
  }).pipe(Effect.provide(TestLayer)),
733
713
  );
734
714
  });
package/src/feed-store.ts CHANGED
@@ -12,7 +12,7 @@ import { type SpaceId } from '@dxos/keys';
12
12
  import { FeedProtocol } from '@dxos/protocols';
13
13
  import { SqlTransaction } from '@dxos/sql-sqlite';
14
14
 
15
- import { DuplicateBlockLamportTimestampError, DuplicateBlockPositionError, PositionConflictError } from './errors';
15
+ import { PositionConflictError } from './errors';
16
16
 
17
17
  type AppendRequest = FeedProtocol.AppendRequest;
18
18
  type AppendResponse = FeedProtocol.AppendResponse;
@@ -124,7 +124,9 @@ export class FeedStore {
124
124
  const rows = yield* sql<{ feedPrivateId: number }>`
125
125
  SELECT feedPrivateId FROM feeds WHERE spaceId = ${spaceId} AND feedId = ${feedId}
126
126
  `;
127
- if (rows.length > 0) return rows[0].feedPrivateId;
127
+ if (rows.length > 0) {
128
+ return rows[0].feedPrivateId;
129
+ }
128
130
 
129
131
  const newRows = yield* sql<{ feedPrivateId: number }>`
130
132
  INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
@@ -141,7 +143,9 @@ export class FeedStore {
141
143
  Effect.gen(this, function* () {
142
144
  const sql = yield* SqlClient.SqlClient;
143
145
  const rows = yield* sql<{ token: string }>`SELECT token FROM cursor_tokens WHERE spaceId = ${spaceId}`;
144
- if (rows.length > 0) return rows[0].token;
146
+ if (rows.length > 0) {
147
+ return rows[0].token;
148
+ }
145
149
 
146
150
  const token = crypto.randomUUID().replace(/-/g, '').slice(0, 6);
147
151
  yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
@@ -192,7 +196,7 @@ export class FeedStore {
192
196
  // If spaceId is not provided in request (e.g. feedIds query), we can't easily validate token unless we look up spaceId for feedIds.
193
197
  // Ideally spaceId should be required for token validation.
194
198
 
195
- /*
199
+ /*
196
200
  Logic:
197
201
  1. If `cursor` is present, it's `token|insertionId`.
198
202
  2. If `position` is present, it's `position` (legacy/manual).
@@ -415,116 +419,12 @@ export class FeedStore {
415
419
  return deletedRows.length;
416
420
  }).pipe(Effect.withSpan('FeedStore.deleteOldestBlocks'));
417
421
 
418
- /**
419
- * Validates that appended blocks do not introduce position or Lamport conflicts.
420
- */
421
- #validateAppendBlocks = Effect.fn('FeedStore.validateAppendBlocks')(
422
- (
423
- blocks: readonly Block[],
424
- feedPrivateIds: Map<string, number>,
425
- ): Effect.Effect<
426
- void,
427
- SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
428
- SqlClient.SqlClient
429
- > =>
430
- Effect.gen(this, function* () {
431
- const sql = yield* SqlClient.SqlClient;
432
-
433
- // Validate request-level uniqueness before writing.
434
- const positionFeedIdByKey = new Map<string, string>();
435
- const lamportFeedIdByKey = new Map<string, string>();
436
- for (const block of blocks) {
437
- const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
438
- if (!this.#options.assignPositions && block.position != null) {
439
- const positionKey = `${feedPrivateId}|${block.position}`;
440
- if (positionFeedIdByKey.has(positionKey)) {
441
- yield* Effect.fail(
442
- new DuplicateBlockPositionError({
443
- feedId: block.feedId!,
444
- position: block.position,
445
- }),
446
- );
447
- }
448
- positionFeedIdByKey.set(positionKey, block.feedId!);
449
- }
450
-
451
- const lamportKey = `${feedPrivateId}|${block.actorId}|${block.sequence}`;
452
- if (lamportFeedIdByKey.has(lamportKey)) {
453
- yield* Effect.fail(
454
- new DuplicateBlockLamportTimestampError({
455
- feedId: block.feedId!,
456
- actorId: block.actorId,
457
- sequence: block.sequence,
458
- }),
459
- );
460
- }
461
- lamportFeedIdByKey.set(lamportKey, block.feedId!);
462
- }
463
-
464
- // Bulk-validate against persisted rows with exact tuple matching.
465
- const lamportConditions = blocks.map((block) => {
466
- const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
467
- return sql`(feedPrivateId = ${feedPrivateId} AND actorId = ${block.actorId} AND sequence = ${block.sequence})`;
468
- });
469
- if (lamportConditions.length > 0) {
470
- const existingLamports = yield* sql<{ feedPrivateId: number; actorId: string; sequence: number }>`
471
- SELECT feedPrivateId, actorId, sequence
472
- FROM blocks
473
- WHERE ${sql.or(lamportConditions)}
474
- LIMIT 1
475
- `;
476
- const lamportConflict = existingLamports[0];
477
- if (lamportConflict) {
478
- const key = `${lamportConflict.feedPrivateId}|${lamportConflict.actorId}|${lamportConflict.sequence}`;
479
- yield* Effect.fail(
480
- new DuplicateBlockLamportTimestampError({
481
- feedId: lamportFeedIdByKey.get(key)!,
482
- actorId: lamportConflict.actorId,
483
- sequence: lamportConflict.sequence,
484
- }),
485
- );
486
- }
487
- }
488
-
489
- if (!this.#options.assignPositions) {
490
- const positionedBlocks = blocks.filter((block) => block.position != null);
491
- const positionConditions = positionedBlocks.map((block) => {
492
- const feedPrivateId = feedPrivateIds.get(block.feedId!)!;
493
- return sql`(feedPrivateId = ${feedPrivateId} AND position = ${block.position})`;
494
- });
495
- if (positionConditions.length > 0) {
496
- const existingPositions = yield* sql<{ feedPrivateId: number; position: number }>`
497
- SELECT feedPrivateId, position
498
- FROM blocks
499
- WHERE position IS NOT NULL
500
- AND (${sql.or(positionConditions)})
501
- LIMIT 1
502
- `;
503
- const positionConflict = existingPositions[0];
504
- if (positionConflict) {
505
- const key = `${positionConflict.feedPrivateId}|${positionConflict.position}`;
506
- yield* Effect.fail(
507
- new DuplicateBlockPositionError({
508
- feedId: positionFeedIdByKey.get(key)!,
509
- position: positionConflict.position,
510
- }),
511
- );
512
- }
513
- }
514
- }
515
- }),
516
- );
517
-
518
422
  /**
519
423
  * Appends blocks for a space/namespace and optionally assigns global positions.
520
424
  */
521
425
  append = (
522
426
  request: AppendRequest,
523
- ): Effect.Effect<
524
- AppendResponse,
525
- SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
526
- SqlClient.SqlClient | SqlTransaction.SqlTransaction
527
- > =>
427
+ ): Effect.Effect<AppendResponse, SqlError.SqlError, SqlClient.SqlClient | SqlTransaction.SqlTransaction> =>
528
428
  Effect.gen(this, function* () {
529
429
  if (!request.spaceId) {
530
430
  return yield* Effect.die(new Error('spaceId required for append'));
@@ -567,8 +467,6 @@ export class FeedStore {
567
467
  { concurrency: 'unbounded' },
568
468
  );
569
469
 
570
- yield* this.#validateAppendBlocks(request.blocks, feedPrivateIds);
571
-
572
470
  // 2. Get max position per namespace ONCE (not per block).
573
471
  const maxPositions = new Map<string, number>();
574
472
  if (this.#options.assignPositions) {
@@ -623,11 +521,7 @@ export class FeedStore {
623
521
  appendLocal = Effect.fn('Feed.appendLocal')(
624
522
  (
625
523
  messages: { spaceId: string; feedId: string; feedNamespace: string; data: Uint8Array }[],
626
- ): Effect.Effect<
627
- Block[],
628
- SqlError.SqlError | DuplicateBlockPositionError | DuplicateBlockLamportTimestampError,
629
- SqlClient.SqlClient | SqlTransaction.SqlTransaction
630
- > =>
524
+ ): Effect.Effect<Block[], SqlError.SqlError, SqlClient.SqlClient | SqlTransaction.SqlTransaction> =>
631
525
  Effect.gen(this, function* () {
632
526
  const sql = yield* SqlClient.SqlClient;
633
527
 
@@ -753,6 +647,23 @@ export class FeedStore {
753
647
  Effect.gen(this, function* () {
754
648
  const sql = yield* SqlClient.SqlClient;
755
649
  for (const block of request.blocks) {
650
+ // Fold the conflict check into the UPDATE itself: only write when the row is
651
+ // unset or already at the requested position. RETURNING tells us whether the
652
+ // write took effect, so we avoid the per-block SELECT round-trip on the
653
+ // common path. Conflicts (rare) take the slow path below.
654
+ const updated = yield* sql<{ position: number | null }>`
655
+ UPDATE blocks SET position = ${block.position}
656
+ WHERE feedPrivateId = (
657
+ SELECT feedPrivateId FROM feeds
658
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
659
+ )
660
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
661
+ AND (position IS NULL OR position = ${block.position})
662
+ RETURNING position
663
+ `;
664
+ if (updated.length > 0) {
665
+ continue;
666
+ }
756
667
  const existing = yield* sql<{ position: number | null }>`
757
668
  SELECT position FROM blocks
758
669
  WHERE feedPrivateId = (
@@ -773,16 +684,56 @@ export class FeedStore {
773
684
  }),
774
685
  );
775
686
  }
776
- yield* sql`
777
- UPDATE blocks SET position = ${block.position}
778
- WHERE feedPrivateId = (
779
- SELECT feedPrivateId FROM feeds
780
- WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
781
- )
782
- AND actorId = ${block.actorId} AND sequence = ${block.sequence}
783
- `;
784
687
  }
785
688
  }).pipe(Effect.withSpan('FeedStore.setPosition'));
689
+
690
+ /**
691
+ * Gets all feeds and their blocks for a space, organized by feed ID and namespace.
692
+ * Used for space archive export.
693
+ */
694
+ getAllFeedsForSpace = (opts: {
695
+ spaceId: SpaceId;
696
+ }): Effect.Effect<
697
+ Array<{
698
+ feedId: string;
699
+ feedNamespace: string;
700
+ blocks: Block[];
701
+ }>,
702
+ SqlError.SqlError,
703
+ SqlClient.SqlClient
704
+ > =>
705
+ Effect.gen(this, function* () {
706
+ const sql = yield* SqlClient.SqlClient;
707
+
708
+ const feeds = yield* sql<{ feedId: string; feedNamespace: string }>`
709
+ SELECT DISTINCT feedId, feedNamespace FROM feeds WHERE spaceId = ${opts.spaceId}
710
+ `;
711
+
712
+ const result: Array<{ feedId: string; feedNamespace: string; blocks: Block[] }> = [];
713
+
714
+ for (const feed of feeds) {
715
+ const blocks = yield* sql<Block>`
716
+ SELECT blocks.*, feeds.feedId, feeds.feedNamespace
717
+ FROM blocks
718
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
719
+ WHERE feeds.spaceId = ${opts.spaceId}
720
+ AND feeds.feedId = ${feed.feedId}
721
+ AND feeds.feedNamespace = ${feed.feedNamespace}
722
+ ORDER BY blocks.sequence ASC
723
+ `;
724
+
725
+ result.push({
726
+ feedId: feed.feedId,
727
+ feedNamespace: feed.feedNamespace,
728
+ blocks: blocks.map((row) => ({
729
+ ...row,
730
+ data: new Uint8Array(row.data),
731
+ })),
732
+ });
733
+ }
734
+
735
+ return result;
736
+ }).pipe(Effect.withSpan('FeedStore.getAllFeedsForSpace'));
786
737
  }
787
738
 
788
739
  const encodeCursor = (token: string, insertionId: number) => FeedCursor.make(`${token}|${insertionId}`);