@dxos/feed 0.8.4-main.2244d791bb → 0.8.4-main.3fbcb4aa9b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -260,4 +260,3 @@ All timestamps (e.g., `insertionTimestamp`) are Unix timestamps in milliseconds.
260
260
  ### TODOs
261
261
 
262
262
  - [ ] Sync protocol only works if each namespace is synced separately.
263
-
@@ -1,21 +1,5 @@
1
1
  // src/errors.ts
2
2
  import { BaseError } from "@dxos/errors";
3
- var DuplicateBlockPositionError = class extends BaseError.extend("DuplicateBlockPositionError") {
4
- constructor(args) {
5
- super({
6
- message: `Non-unique block position detected (feedId=${args.feedId} position=${args.position}).`,
7
- context: args
8
- });
9
- }
10
- };
11
- var DuplicateBlockLamportTimestampError = class extends BaseError.extend("DuplicateBlockLamportTimestampError") {
12
- constructor(args) {
13
- super({
14
- message: `Non-unique block Lamport timestamp detected (feedId=${args.feedId} actorId=${args.actorId} sequence=${args.sequence}).`,
15
- context: args
16
- });
17
- }
18
- };
19
3
  var PositionConflictError = class extends BaseError.extend("PositionConflictError") {
20
4
  constructor(args) {
21
5
  super({
@@ -93,7 +77,9 @@ var FeedStore = class {
93
77
  const rows = yield* sql`
94
78
  SELECT feedPrivateId FROM feeds WHERE spaceId = ${spaceId} AND feedId = ${feedId}
95
79
  `;
96
- if (rows.length > 0) return rows[0].feedPrivateId;
80
+ if (rows.length > 0) {
81
+ return rows[0].feedPrivateId;
82
+ }
97
83
  const newRows = yield* sql`
98
84
  INSERT INTO feeds (spaceId, feedId, feedNamespace) VALUES (${spaceId}, ${feedId}, ${namespace}) RETURNING feedPrivateId
99
85
  `;
@@ -105,7 +91,9 @@ var FeedStore = class {
105
91
  #ensureCursorToken = Effect.fn("Feed.ensureCursorToken")((spaceId) => Effect.gen(this, function* () {
106
92
  const sql = yield* SqlClient.SqlClient;
107
93
  const rows = yield* sql`SELECT token FROM cursor_tokens WHERE spaceId = ${spaceId}`;
108
- if (rows.length > 0) return rows[0].token;
94
+ if (rows.length > 0) {
95
+ return rows[0].token;
96
+ }
109
97
  const token = crypto.randomUUID().replace(/-/g, "").slice(0, 6);
110
98
  yield* sql`INSERT INTO cursor_tokens (spaceId, token) VALUES (${spaceId}, ${token})`;
111
99
  return token;
@@ -302,81 +290,6 @@ var FeedStore = class {
302
290
  return deletedRows.length;
303
291
  }).pipe(Effect.withSpan("FeedStore.deleteOldestBlocks"));
304
292
  /**
305
- * Validates that appended blocks do not introduce position or Lamport conflicts.
306
- */
307
- #validateAppendBlocks = Effect.fn("FeedStore.validateAppendBlocks")((blocks, feedPrivateIds) => Effect.gen(this, function* () {
308
- const sql = yield* SqlClient.SqlClient;
309
- const positionFeedIdByKey = /* @__PURE__ */ new Map();
310
- const lamportFeedIdByKey = /* @__PURE__ */ new Map();
311
- for (const block of blocks) {
312
- const feedPrivateId = feedPrivateIds.get(block.feedId);
313
- if (!this.#options.assignPositions && block.position != null) {
314
- const positionKey = `${feedPrivateId}|${block.position}`;
315
- if (positionFeedIdByKey.has(positionKey)) {
316
- yield* Effect.fail(new DuplicateBlockPositionError({
317
- feedId: block.feedId,
318
- position: block.position
319
- }));
320
- }
321
- positionFeedIdByKey.set(positionKey, block.feedId);
322
- }
323
- const lamportKey = `${feedPrivateId}|${block.actorId}|${block.sequence}`;
324
- if (lamportFeedIdByKey.has(lamportKey)) {
325
- yield* Effect.fail(new DuplicateBlockLamportTimestampError({
326
- feedId: block.feedId,
327
- actorId: block.actorId,
328
- sequence: block.sequence
329
- }));
330
- }
331
- lamportFeedIdByKey.set(lamportKey, block.feedId);
332
- }
333
- const lamportConditions = blocks.map((block) => {
334
- const feedPrivateId = feedPrivateIds.get(block.feedId);
335
- return sql`(feedPrivateId = ${feedPrivateId} AND actorId = ${block.actorId} AND sequence = ${block.sequence})`;
336
- });
337
- if (lamportConditions.length > 0) {
338
- const existingLamports = yield* sql`
339
- SELECT feedPrivateId, actorId, sequence
340
- FROM blocks
341
- WHERE ${sql.or(lamportConditions)}
342
- LIMIT 1
343
- `;
344
- const lamportConflict = existingLamports[0];
345
- if (lamportConflict) {
346
- const key = `${lamportConflict.feedPrivateId}|${lamportConflict.actorId}|${lamportConflict.sequence}`;
347
- yield* Effect.fail(new DuplicateBlockLamportTimestampError({
348
- feedId: lamportFeedIdByKey.get(key),
349
- actorId: lamportConflict.actorId,
350
- sequence: lamportConflict.sequence
351
- }));
352
- }
353
- }
354
- if (!this.#options.assignPositions) {
355
- const positionedBlocks = blocks.filter((block) => block.position != null);
356
- const positionConditions = positionedBlocks.map((block) => {
357
- const feedPrivateId = feedPrivateIds.get(block.feedId);
358
- return sql`(feedPrivateId = ${feedPrivateId} AND position = ${block.position})`;
359
- });
360
- if (positionConditions.length > 0) {
361
- const existingPositions = yield* sql`
362
- SELECT feedPrivateId, position
363
- FROM blocks
364
- WHERE position IS NOT NULL
365
- AND (${sql.or(positionConditions)})
366
- LIMIT 1
367
- `;
368
- const positionConflict = existingPositions[0];
369
- if (positionConflict) {
370
- const key = `${positionConflict.feedPrivateId}|${positionConflict.position}`;
371
- yield* Effect.fail(new DuplicateBlockPositionError({
372
- feedId: positionFeedIdByKey.get(key),
373
- position: positionConflict.position
374
- }));
375
- }
376
- }
377
- }
378
- }));
379
- /**
380
293
  * Appends blocks for a space/namespace and optionally assigns global positions.
381
294
  */
382
295
  append = (request) => Effect.gen(this, function* () {
@@ -408,7 +321,6 @@ var FeedStore = class {
408
321
  }), {
409
322
  concurrency: "unbounded"
410
323
  });
411
- yield* this.#validateAppendBlocks(request.blocks, feedPrivateIds);
412
324
  const maxPositions = /* @__PURE__ */ new Map();
413
325
  if (this.#options.assignPositions) {
414
326
  const maxPosResult = yield* sql`
@@ -553,6 +465,19 @@ var FeedStore = class {
553
465
  setPosition = (request) => Effect.gen(this, function* () {
554
466
  const sql = yield* SqlClient.SqlClient;
555
467
  for (const block of request.blocks) {
468
+ const updated = yield* sql`
469
+ UPDATE blocks SET position = ${block.position}
470
+ WHERE feedPrivateId = (
471
+ SELECT feedPrivateId FROM feeds
472
+ WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
473
+ )
474
+ AND actorId = ${block.actorId} AND sequence = ${block.sequence}
475
+ AND (position IS NULL OR position = ${block.position})
476
+ RETURNING position
477
+ `;
478
+ if (updated.length > 0) {
479
+ continue;
480
+ }
556
481
  const existing = yield* sql`
557
482
  SELECT position FROM blocks
558
483
  WHERE feedPrivateId = (
@@ -571,16 +496,39 @@ var FeedStore = class {
571
496
  requestedPosition: block.position
572
497
  }));
573
498
  }
574
- yield* sql`
575
- UPDATE blocks SET position = ${block.position}
576
- WHERE feedPrivateId = (
577
- SELECT feedPrivateId FROM feeds
578
- WHERE spaceId = ${request.spaceId} AND feedId = ${block.feedId} AND feedNamespace = ${block.feedNamespace}
579
- )
580
- AND actorId = ${block.actorId} AND sequence = ${block.sequence}
581
- `;
582
499
  }
583
500
  }).pipe(Effect.withSpan("FeedStore.setPosition"));
501
+ /**
502
+ * Gets all feeds and their blocks for a space, organized by feed ID and namespace.
503
+ * Used for space archive export.
504
+ */
505
+ getAllFeedsForSpace = (opts) => Effect.gen(this, function* () {
506
+ const sql = yield* SqlClient.SqlClient;
507
+ const feeds = yield* sql`
508
+ SELECT DISTINCT feedId, feedNamespace FROM feeds WHERE spaceId = ${opts.spaceId}
509
+ `;
510
+ const result = [];
511
+ for (const feed of feeds) {
512
+ const blocks = yield* sql`
513
+ SELECT blocks.*, feeds.feedId, feeds.feedNamespace
514
+ FROM blocks
515
+ JOIN feeds ON blocks.feedPrivateId = feeds.feedPrivateId
516
+ WHERE feeds.spaceId = ${opts.spaceId}
517
+ AND feeds.feedId = ${feed.feedId}
518
+ AND feeds.feedNamespace = ${feed.feedNamespace}
519
+ ORDER BY blocks.sequence ASC
520
+ `;
521
+ result.push({
522
+ feedId: feed.feedId,
523
+ feedNamespace: feed.feedNamespace,
524
+ blocks: blocks.map((row) => ({
525
+ ...row,
526
+ data: new Uint8Array(row.data)
527
+ }))
528
+ });
529
+ }
530
+ return result;
531
+ }).pipe(Effect.withSpan("FeedStore.getAllFeedsForSpace"));
584
532
  };
585
533
  var encodeCursor = (token, insertionId) => FeedCursor.make(`${token}|${insertionId}`);
586
534
  var decodeCursor = (cursor) => {
@@ -593,6 +541,19 @@ var decodeCursor = (cursor) => {
593
541
 
594
542
  // src/sync-server.ts
595
543
  import * as Effect2 from "effect/Effect";
544
+ import { log } from "@dxos/log";
545
+ var __dxlog_file = "/__w/dxos/dxos/packages/core/echo/feed/src/sync-server.ts";
546
+ var inboundSummary = (message) => ({
547
+ tag: message._tag,
548
+ senderPeerId: message.senderPeerId,
549
+ recipientPeerId: message.recipientPeerId,
550
+ requestId: "requestId" in message ? message.requestId : void 0,
551
+ spaceId: "spaceId" in message ? message.spaceId : void 0,
552
+ feedNamespace: "feedNamespace" in message ? message.feedNamespace : void 0,
553
+ blockCount: "blocks" in message ? message.blocks.length : void 0,
554
+ position: "position" in message ? message.position : void 0,
555
+ limit: "limit" in message ? message.limit : void 0
556
+ });
596
557
  var SyncServer = class {
597
558
  #peerId;
598
559
  #feedStore;
@@ -605,8 +566,12 @@ var SyncServer = class {
605
566
  /**
606
567
  * Receive a message from a client. Handles QueryRequest and AppendRequest; sends response via sendMessage with correct peer ids.
607
568
  */
608
- handleMessage(message) {
569
+ handleMessage(ctx, message) {
609
570
  const self = this;
571
+ log("feed sync server received message", {
572
+ peerId: self.#peerId,
573
+ ...inboundSummary(message)
574
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 33, S: this });
610
575
  const recipientPeerId = message.senderPeerId;
611
576
  const withPeerIds = (payload) => ({
612
577
  ...payload,
@@ -618,29 +583,70 @@ var SyncServer = class {
618
583
  const req = message;
619
584
  return Effect2.gen(function* () {
620
585
  const response = yield* self.#feedStore.query(req);
621
- yield* self.#sendMessage(withPeerIds({
586
+ log("feed sync server query completed", {
587
+ peerId: self.#peerId,
588
+ recipientPeerId,
589
+ requestId: req.requestId,
590
+ spaceId: req.spaceId,
591
+ feedNamespace: req.feedNamespace,
592
+ returnedBlocks: response.blocks.length,
593
+ hasMore: response.hasMore
594
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 49, S: this });
595
+ yield* self.#sendMessage(ctx, withPeerIds({
622
596
  _tag: "QueryResponse",
623
597
  ...response
624
598
  }));
625
- }).pipe(Effect2.catchAll((err) => self.#sendMessage(withPeerIds({
626
- _tag: "Error",
627
- message: err instanceof Error ? err.message : String(err)
628
- }))));
599
+ }).pipe(Effect2.catchAll((err) => {
600
+ log("feed sync server query failed", {
601
+ peerId: self.#peerId,
602
+ recipientPeerId,
603
+ requestId: req.requestId,
604
+ spaceId: req.spaceId,
605
+ feedNamespace: req.feedNamespace,
606
+ cause: err instanceof Error ? err.message : String(err)
607
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 63, S: this });
608
+ return self.#sendMessage(ctx, withPeerIds({
609
+ _tag: "Error",
610
+ message: err instanceof Error ? err.message : String(err)
611
+ }));
612
+ }));
629
613
  }
630
614
  case "AppendRequest": {
631
615
  const req = message;
632
616
  return Effect2.gen(function* () {
633
617
  const response = yield* self.#feedStore.append(req);
634
- yield* self.#sendMessage(withPeerIds({
618
+ log("feed sync server append completed", {
619
+ peerId: self.#peerId,
620
+ recipientPeerId,
621
+ requestId: req.requestId,
622
+ spaceId: req.spaceId,
623
+ feedNamespace: req.feedNamespace,
624
+ assignedPositions: response.positions.length
625
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 82, S: this });
626
+ yield* self.#sendMessage(ctx, withPeerIds({
635
627
  _tag: "AppendResponse",
636
628
  ...response
637
629
  }));
638
- }).pipe(Effect2.catchAll((err) => self.#sendMessage(withPeerIds({
639
- _tag: "Error",
640
- message: err instanceof Error ? err.message : String(err)
641
- }))));
630
+ }).pipe(Effect2.catchAll((err) => {
631
+ log("feed sync server append failed", {
632
+ peerId: self.#peerId,
633
+ recipientPeerId,
634
+ requestId: req.requestId,
635
+ spaceId: req.spaceId,
636
+ feedNamespace: req.feedNamespace,
637
+ cause: err instanceof Error ? err.message : String(err)
638
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 95, S: this });
639
+ return self.#sendMessage(ctx, withPeerIds({
640
+ _tag: "Error",
641
+ message: err instanceof Error ? err.message : String(err)
642
+ }));
643
+ }));
642
644
  }
643
645
  default:
646
+ log("feed sync server ignoring unsupported message", {
647
+ peerId: self.#peerId,
648
+ tag: message._tag
649
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 110, S: this });
644
650
  return Effect2.void;
645
651
  }
646
652
  }
@@ -650,6 +656,9 @@ var SyncServer = class {
650
656
  import * as Array from "effect/Array";
651
657
  import * as Deferred from "effect/Deferred";
652
658
  import * as Effect3 from "effect/Effect";
659
+ import { ContextDisposedError } from "@dxos/context";
660
+ import { log as log2 } from "@dxos/log";
661
+ var __dxlog_file2 = "/__w/dxos/dxos/packages/core/echo/feed/src/sync-client.ts";
653
662
  var SyncClient = class {
654
663
  #peerId;
655
664
  #serverPeerId;
@@ -681,16 +690,36 @@ var SyncClient = class {
681
690
  */
682
691
  handleMessage(message) {
683
692
  const requestId = "requestId" in message && message.requestId != null ? String(message.requestId) : void 0;
684
- if (requestId == null) return Effect3.void;
693
+ if (requestId == null) {
694
+ log2.trace("feed sync client response ignored (no request id)", {
695
+ tag: message._tag,
696
+ senderPeerId: message.senderPeerId
697
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 45, S: this });
698
+ return Effect3.void;
699
+ }
685
700
  const deferred = this.#handlers.get(requestId);
686
- if (deferred == null) return Effect3.void;
701
+ if (deferred == null) {
702
+ log2.trace("feed sync client response ignored (no pending rpc)", {
703
+ requestId,
704
+ tag: message._tag,
705
+ senderPeerId: message.senderPeerId
706
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 53, S: this });
707
+ return Effect3.void;
708
+ }
687
709
  this.#handlers.delete(requestId);
710
+ log2("feed sync client rpc completed", {
711
+ requestId,
712
+ tag: message._tag,
713
+ blockCount: "blocks" in message ? message.blocks.length : void 0,
714
+ positionCount: "positions" in message ? message.positions.length : void 0,
715
+ error: message._tag === "Error" ? message.message : void 0
716
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 61, S: this });
688
717
  if (message._tag === "Error") {
689
718
  return Effect3.andThen(Deferred.fail(deferred, new Error(message.message)), () => Effect3.void);
690
719
  }
691
720
  return Effect3.andThen(Deferred.succeed(deferred, message), () => Effect3.void);
692
721
  }
693
- pull(opts) {
722
+ pull(ctx, opts) {
694
723
  const self = this;
695
724
  return Effect3.gen(function* () {
696
725
  const lastPulledPosition = yield* self.#feedStore.getSyncState({
@@ -700,6 +729,13 @@ var SyncClient = class {
700
729
  const requestId = crypto.randomUUID();
701
730
  const deferred = yield* Deferred.make();
702
731
  self.#handlers.set(requestId, deferred);
732
+ const cleanupDispose = ctx.disposed ? () => {
733
+ } : ctx.onDispose(() => {
734
+ Effect3.runFork(Deferred.fail(deferred, new ContextDisposedError()));
735
+ });
736
+ if (ctx.disposed) {
737
+ yield* Deferred.fail(deferred, new ContextDisposedError());
738
+ }
703
739
  const request = {
704
740
  _tag: "QueryRequest",
705
741
  requestId,
@@ -708,10 +744,25 @@ var SyncClient = class {
708
744
  position: lastPulledPosition,
709
745
  limit: opts.limit
710
746
  };
711
- yield* self.#sendMessage(self.#withPeerIds(request));
712
- const message = yield* Deferred.await(deferred);
747
+ log2("feed sync client pull rpc sending", {
748
+ requestId,
749
+ spaceId: opts.spaceId,
750
+ feedNamespace: opts.feedNamespace,
751
+ afterPosition: lastPulledPosition,
752
+ limit: opts.limit
753
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 97, S: this });
754
+ yield* self.#sendMessage(ctx, self.#withPeerIds(request));
755
+ const message = yield* Effect3.ensuring(Deferred.await(deferred), Effect3.sync(() => {
756
+ cleanupDispose();
757
+ self.#handlers.delete(requestId);
758
+ }));
713
759
  const response = yield* self.#expectResponse(requestId, message, "QueryResponse");
714
760
  if (response.blocks.length === 0) {
761
+ log2.trace("feed sync client pull done (empty batch)", {
762
+ requestId,
763
+ spaceId: opts.spaceId,
764
+ feedNamespace: opts.feedNamespace
765
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 111, S: this });
715
766
  return {
716
767
  done: true
717
768
  };
@@ -727,12 +778,20 @@ var SyncClient = class {
727
778
  feedNamespace: opts.feedNamespace,
728
779
  lastPulledPosition: maxPulledPosition
729
780
  });
781
+ log2("feed sync client pull applied batch", {
782
+ requestId,
783
+ spaceId: opts.spaceId,
784
+ feedNamespace: opts.feedNamespace,
785
+ batchSize: response.blocks.length,
786
+ hasMore: response.hasMore,
787
+ maxPulledPosition
788
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 132, S: this });
730
789
  return {
731
790
  done: false
732
791
  };
733
792
  });
734
793
  }
735
- push(opts) {
794
+ push(ctx, opts) {
736
795
  const self = this;
737
796
  return Effect3.gen(function* () {
738
797
  const unpositioned = yield* self.#feedStore.query({
@@ -742,6 +801,10 @@ var SyncClient = class {
742
801
  limit: opts.limit
743
802
  });
744
803
  if (unpositioned.blocks.length === 0) {
804
+ log2.trace("feed sync client push skipped (nothing to send)", {
805
+ spaceId: opts.spaceId,
806
+ feedNamespace: opts.feedNamespace
807
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 155, S: this });
745
808
  return {
746
809
  done: true
747
810
  };
@@ -749,6 +812,13 @@ var SyncClient = class {
749
812
  const requestId = crypto.randomUUID();
750
813
  const deferred = yield* Deferred.make();
751
814
  self.#handlers.set(requestId, deferred);
815
+ const cleanupDispose = ctx.disposed ? () => {
816
+ } : ctx.onDispose(() => {
817
+ Effect3.runFork(Deferred.fail(deferred, new ContextDisposedError()));
818
+ });
819
+ if (ctx.disposed) {
820
+ yield* Deferred.fail(deferred, new ContextDisposedError());
821
+ }
752
822
  const request = {
753
823
  _tag: "AppendRequest",
754
824
  requestId,
@@ -756,8 +826,17 @@ var SyncClient = class {
756
826
  feedNamespace: opts.feedNamespace,
757
827
  blocks: unpositioned.blocks
758
828
  };
759
- yield* self.#sendMessage(self.#withPeerIds(request));
760
- const message = yield* Deferred.await(deferred);
829
+ log2("feed sync client push rpc sending", {
830
+ requestId,
831
+ spaceId: opts.spaceId,
832
+ feedNamespace: opts.feedNamespace,
833
+ blockCount: unpositioned.blocks.length
834
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 179, S: this });
835
+ yield* self.#sendMessage(ctx, self.#withPeerIds(request));
836
+ const message = yield* Effect3.ensuring(Deferred.await(deferred), Effect3.sync(() => {
837
+ cleanupDispose();
838
+ self.#handlers.delete(requestId);
839
+ }));
761
840
  const response = yield* self.#expectResponse(requestId, message, "AppendResponse");
762
841
  yield* self.#feedStore.setPosition({
763
842
  spaceId: opts.spaceId,
@@ -769,6 +848,12 @@ var SyncClient = class {
769
848
  position
770
849
  }))
771
850
  });
851
+ log2("feed sync client push positions applied", {
852
+ requestId,
853
+ spaceId: opts.spaceId,
854
+ feedNamespace: opts.feedNamespace,
855
+ positionCount: response.positions.length
856
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 201, S: this });
772
857
  return {
773
858
  done: false
774
859
  };
@@ -787,11 +872,9 @@ var SyncClient = class {
787
872
  };
788
873
 
789
874
  export {
790
- DuplicateBlockPositionError,
791
- DuplicateBlockLamportTimestampError,
792
875
  PositionConflictError,
793
876
  FeedStore,
794
877
  SyncServer,
795
878
  SyncClient
796
879
  };
797
- //# sourceMappingURL=chunk-ZL7JEUUQ.mjs.map
880
+ //# sourceMappingURL=chunk-YYASRWLE.mjs.map