@optimystic/db-core 1.10.1 → 1.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/dist/src/cluster/structs.d.ts +48 -7
  2. package/dist/src/cluster/structs.d.ts.map +1 -1
  3. package/dist/src/cluster/structs.js +33 -13
  4. package/dist/src/cluster/structs.js.map +1 -1
  5. package/dist/src/cohort-topic/coldstart.d.ts +8 -4
  6. package/dist/src/cohort-topic/coldstart.d.ts.map +1 -1
  7. package/dist/src/cohort-topic/coldstart.js +3 -3
  8. package/dist/src/cohort-topic/coldstart.js.map +1 -1
  9. package/dist/src/cohort-topic/member-engine.d.ts +5 -1
  10. package/dist/src/cohort-topic/member-engine.d.ts.map +1 -1
  11. package/dist/src/cohort-topic/member-engine.js.map +1 -1
  12. package/dist/src/cohort-topic/registration/store.d.ts +8 -1
  13. package/dist/src/cohort-topic/registration/store.d.ts.map +1 -1
  14. package/dist/src/cohort-topic/registration/store.js +57 -0
  15. package/dist/src/cohort-topic/registration/store.js.map +1 -1
  16. package/dist/src/cohort-topic/willingness.d.ts +4 -3
  17. package/dist/src/cohort-topic/willingness.d.ts.map +1 -1
  18. package/dist/src/cohort-topic/willingness.js.map +1 -1
  19. package/dist/src/cohort-topic/wire/primitives.d.ts +5 -0
  20. package/dist/src/cohort-topic/wire/primitives.d.ts.map +1 -1
  21. package/dist/src/cohort-topic/wire/primitives.js +5 -0
  22. package/dist/src/cohort-topic/wire/primitives.js.map +1 -1
  23. package/dist/src/cohort-topic/wire/validate.d.ts.map +1 -1
  24. package/dist/src/cohort-topic/wire/validate.js +1 -3
  25. package/dist/src/cohort-topic/wire/validate.js.map +1 -1
  26. package/dist/src/collection/action.d.ts +15 -1
  27. package/dist/src/collection/action.d.ts.map +1 -1
  28. package/dist/src/collection/action.js.map +1 -1
  29. package/dist/src/collection/collection.d.ts.map +1 -1
  30. package/dist/src/collection/collection.js +60 -5
  31. package/dist/src/collection/collection.js.map +1 -1
  32. package/dist/src/collection/struct.d.ts +6 -2
  33. package/dist/src/collection/struct.d.ts.map +1 -1
  34. package/dist/src/collection/struct.js.map +1 -1
  35. package/dist/src/index.d.ts +1 -0
  36. package/dist/src/index.d.ts.map +1 -1
  37. package/dist/src/index.js +1 -0
  38. package/dist/src/index.js.map +1 -1
  39. package/dist/src/matchmaking/arrival-push.d.ts +35 -0
  40. package/dist/src/matchmaking/arrival-push.d.ts.map +1 -0
  41. package/dist/src/matchmaking/arrival-push.js +56 -0
  42. package/dist/src/matchmaking/arrival-push.js.map +1 -0
  43. package/dist/src/matchmaking/config.d.ts +12 -2
  44. package/dist/src/matchmaking/config.d.ts.map +1 -1
  45. package/dist/src/matchmaking/config.js +8 -2
  46. package/dist/src/matchmaking/config.js.map +1 -1
  47. package/dist/src/matchmaking/index.d.ts +3 -2
  48. package/dist/src/matchmaking/index.d.ts.map +1 -1
  49. package/dist/src/matchmaking/index.js +3 -2
  50. package/dist/src/matchmaking/index.js.map +1 -1
  51. package/dist/src/matchmaking/seeker.d.ts +5 -2
  52. package/dist/src/matchmaking/seeker.d.ts.map +1 -1
  53. package/dist/src/matchmaking/seeker.js +6 -1
  54. package/dist/src/matchmaking/seeker.js.map +1 -1
  55. package/dist/src/matchmaking/wire.d.ts +66 -3
  56. package/dist/src/matchmaking/wire.d.ts.map +1 -1
  57. package/dist/src/matchmaking/wire.js +79 -11
  58. package/dist/src/matchmaking/wire.js.map +1 -1
  59. package/dist/src/network/i-key-network.d.ts +8 -0
  60. package/dist/src/network/i-key-network.d.ts.map +1 -1
  61. package/dist/src/network/index.d.ts +1 -0
  62. package/dist/src/network/index.d.ts.map +1 -1
  63. package/dist/src/network/index.js +1 -0
  64. package/dist/src/network/index.js.map +1 -1
  65. package/dist/src/network/stale-failure.d.ts +12 -3
  66. package/dist/src/network/stale-failure.d.ts.map +1 -1
  67. package/dist/src/network/stale-failure.js +14 -1
  68. package/dist/src/network/stale-failure.js.map +1 -1
  69. package/dist/src/network/struct.d.ts +54 -4
  70. package/dist/src/network/struct.d.ts.map +1 -1
  71. package/dist/src/network/struct.js.map +1 -1
  72. package/dist/src/network/transaction-expired-error.d.ts +44 -0
  73. package/dist/src/network/transaction-expired-error.d.ts.map +1 -0
  74. package/dist/src/network/transaction-expired-error.js +77 -0
  75. package/dist/src/network/transaction-expired-error.js.map +1 -0
  76. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  77. package/dist/src/testing/test-transactor.js +28 -46
  78. package/dist/src/testing/test-transactor.js.map +1 -1
  79. package/dist/src/transaction/coordinator.d.ts +4 -1
  80. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  81. package/dist/src/transaction/coordinator.js +31 -5
  82. package/dist/src/transaction/coordinator.js.map +1 -1
  83. package/dist/src/transaction/index.d.ts +1 -1
  84. package/dist/src/transaction/index.d.ts.map +1 -1
  85. package/dist/src/transaction/index.js +1 -1
  86. package/dist/src/transaction/index.js.map +1 -1
  87. package/dist/src/transaction/transaction.d.ts +27 -0
  88. package/dist/src/transaction/transaction.d.ts.map +1 -1
  89. package/dist/src/transaction/transaction.js +27 -0
  90. package/dist/src/transaction/transaction.js.map +1 -1
  91. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  92. package/dist/src/transactor/network-transactor.js +32 -39
  93. package/dist/src/transactor/network-transactor.js.map +1 -1
  94. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  95. package/dist/src/transactor/transactor-source.js +6 -2
  96. package/dist/src/transactor/transactor-source.js.map +1 -1
  97. package/dist/src/utility/format-instant.d.ts +8 -0
  98. package/dist/src/utility/format-instant.d.ts.map +1 -0
  99. package/dist/src/utility/format-instant.js +11 -0
  100. package/dist/src/utility/format-instant.js.map +1 -0
  101. package/package.json +1 -1
  102. package/src/cluster/structs.ts +72 -14
  103. package/src/cohort-topic/coldstart.ts +8 -4
  104. package/src/cohort-topic/member-engine.ts +5 -1
  105. package/src/cohort-topic/registration/store.ts +68 -0
  106. package/src/cohort-topic/willingness.ts +4 -3
  107. package/src/cohort-topic/wire/primitives.ts +6 -0
  108. package/src/cohort-topic/wire/validate.ts +1 -3
  109. package/src/collection/action.ts +15 -1
  110. package/src/collection/collection.ts +63 -5
  111. package/src/collection/struct.ts +6 -2
  112. package/src/index.ts +1 -0
  113. package/src/matchmaking/arrival-push.ts +66 -0
  114. package/src/matchmaking/config.ts +17 -2
  115. package/src/matchmaking/index.ts +3 -2
  116. package/src/matchmaking/seeker.ts +11 -3
  117. package/src/matchmaking/wire.ts +134 -10
  118. package/src/network/i-key-network.ts +9 -0
  119. package/src/network/index.ts +1 -0
  120. package/src/network/stale-failure.ts +16 -3
  121. package/src/network/struct.ts +57 -4
  122. package/src/network/transaction-expired-error.ts +81 -0
  123. package/src/testing/test-transactor.ts +28 -40
  124. package/src/transaction/coordinator.ts +37 -7
  125. package/src/transaction/index.ts +2 -1
  126. package/src/transaction/transaction.ts +28 -0
  127. package/src/transactor/network-transactor.ts +36 -43
  128. package/src/transactor/transactor-source.ts +6 -2
  129. package/src/utility/format-instant.ts +10 -0
@@ -8,9 +8,12 @@
8
8
  * local soft state; cross-member replication runs over cohort gossip in a later ticket.
9
9
  */
10
10
 
11
+ import { createLogger } from "../../logger.js";
11
12
  import { bytesKey } from "./bytes.js";
12
13
  import type { RegistrationRecord, RegistrationStore } from "./types.js";
13
14
 
15
+ const log = createLogger("cohort-topic:registration");
16
+
14
17
  class InMemoryRegistrationStore implements RegistrationStore {
15
18
  /** topicKey → (participantKey → record). Inner maps are pruned when they empty. */
16
19
  private readonly byTopic = new Map<string, Map<string, RegistrationRecord>>();
@@ -79,3 +82,68 @@ class InMemoryRegistrationStore implements RegistrationStore {
79
82
  export function createRegistrationStore(): RegistrationStore {
80
83
  return new InMemoryRegistrationStore();
81
84
  }
85
+
86
+ /** Called once per record that a `put` made present where the store held none for its participant. */
87
+ export type RecordAddedListener = (rec: RegistrationRecord) => void;
88
+
89
+ /**
90
+ * A store that reports each absent→present `put`. Every way a record reaches a member — local
91
+ * admission, a gossip merge, a rotation-handoff pull — goes through `put`, so this is the one place
92
+ * a "a registration arrived here" signal can be taken without missing a path. A `put` that replaces
93
+ * a held record (a renewal touch, a re-attach restamp, a re-merge) reports nothing.
94
+ */
95
+ class RecordAdditionObserver implements RegistrationStore {
96
+ constructor(
97
+ private readonly inner: RegistrationStore,
98
+ private readonly onAdded: RecordAddedListener,
99
+ ) {}
100
+
101
+ put(rec: RegistrationRecord): void {
102
+ const added = this.inner.getByParticipant(rec.topicId, rec.participantId) === undefined;
103
+ this.inner.put(rec);
104
+ if (added) {
105
+ this.notify(rec);
106
+ }
107
+ }
108
+
109
+ getByParticipant(topicId: Uint8Array, participantId: Uint8Array): RegistrationRecord | undefined {
110
+ return this.inner.getByParticipant(topicId, participantId);
111
+ }
112
+
113
+ listByTopic(topicId: Uint8Array): readonly RegistrationRecord[] {
114
+ return this.inner.listByTopic(topicId);
115
+ }
116
+
117
+ listAll(): readonly RegistrationRecord[] {
118
+ return this.inner.listAll();
119
+ }
120
+
121
+ delete(topicId: Uint8Array, participantId: Uint8Array): void {
122
+ this.inner.delete(topicId, participantId);
123
+ }
124
+
125
+ directParticipants(topicId: Uint8Array): number {
126
+ return this.inner.directParticipants(topicId);
127
+ }
128
+
129
+ evictStale(now: number): readonly RegistrationRecord[] {
130
+ return this.inner.evictStale(now);
131
+ }
132
+
133
+ /** A listener fault must not abort the admission or gossip merge that called `put`. */
134
+ private notify(rec: RegistrationRecord): void {
135
+ try {
136
+ this.onAdded(rec);
137
+ } catch (err) {
138
+ log("record-added listener threw for topic %s participant %s: %o", bytesKey(rec.topicId), bytesKey(rec.participantId), err);
139
+ }
140
+ }
141
+ }
142
+
143
+ /**
144
+ * Wrap `store` so `onAdded` fires after each `put` that makes a `(topicId, participantId)` record
145
+ * present where none was held. A listener throw is logged, never propagated into `put`.
146
+ */
147
+ export function observeRecordAdditions(store: RegistrationStore, onAdded: RecordAddedListener): RegistrationStore {
148
+ return new RecordAdditionObserver(store, onAdded);
149
+ }
@@ -133,9 +133,10 @@ export interface WillingnessConfig {
133
133
  /** Cohort size `k` (drives the default quorum). Default 16. */
134
134
  cohortSize?: number;
135
135
  /**
136
- * Members that must be willing to serve a tier for the cohort to take it on. Default: strict
137
- * majority of `cohortSize` (`⌊k/2⌋ + 1`). Not pinned by `docs/cohort-topic.md` — §Tier ladder
138
- * only requires "a quorum"; configurable here, revisitable by the Edge/Core policy ticket.
136
+ * Members that must be willing to serve a tier for the cohort to take it on (the admission quorum,
137
+ * `docs/cohort-topic.md` §Admission quorum). Default: strict majority of `cohortSize`
138
+ * ({@link defaultQuorum}). Deliberately not the threshold-signature `minSigs = k − x` — see the doc
139
+ * for why the two differ and when to revisit the default.
139
140
  */
140
141
  quorum?: number;
141
142
  /** Load bucket at/above which a tier is shed. Default {@link DEFAULT_OVERLOAD_BUCKET} (6). */
@@ -23,6 +23,12 @@ import { b64urlToBytes } from "./codec.js";
23
23
  */
24
24
  export const COORD_BYTES = 32;
25
25
 
26
+ /**
27
+ * Correlation-id byte width: the 16-byte nonce the cohort-topic substrate mints per walk probe / renew,
28
+ * and the matchmaking seeker registration id an arrival push is bound to.
29
+ */
30
+ export const CORRELATION_BYTES = 16;
31
+
26
32
  /** Thrown for any malformed, oversized, or structurally invalid cohort-topic frame. */
27
33
  export class CohortWireError extends Error {
28
34
  constructor(message: string) {
@@ -16,6 +16,7 @@ import {
16
16
  b64urlField,
17
17
  b64urlFixedLen,
18
18
  COORD_BYTES,
19
+ CORRELATION_BYTES,
19
20
  failWire as fail,
20
21
  optBool,
21
22
  optFiniteNumber,
@@ -78,9 +79,6 @@ function treeTier(value: number, what: string): number {
78
79
  return value;
79
80
  }
80
81
 
81
- /** Correlation-id byte width (a 16-byte nonce minted per walk probe / renew). */
82
- const CORRELATION_BYTES = 16;
83
-
84
82
  /**
85
83
  * Ceiling on a decoded `rootKey` (the root key of a root-placed topic). A root key is a block's routing
86
84
  * key — the UTF-8 of its block id, 43 bytes for a generated id — so 256 leaves room for a named block
@@ -26,7 +26,21 @@ export type ActionContext = {
26
26
  committed: ActionRev[];
27
27
  /** The latest known revision number */
28
28
  rev: number;
29
- /** Optional uncommitted pending action ID */
29
+ /** Optional uncommitted pending action to read through. A repo overlays this action's pending
30
+ * transform on each block where it holds a pending record of it, and answers every other block
31
+ * exactly as a plain read with the same context would; the overlay answer's `state.pendings`
32
+ * names the action, the plain one's does not.
33
+ *
34
+ * No production code sets it. It is reserved for tentative reads in long-lived pend (backlog
35
+ * `feat-long-lived-pend-completes-as-members-appear`), where a read that observes a
36
+ * pended-but-uncommitted revision makes the reading transaction tentative too. It is
37
+ * deliberately not used for early abort: the only certain abort signal — a latest revision at
38
+ * or past the pended one under another action — is already on a plain read's `state.latest`.
39
+ * Whoever sets it must keep the overlay answer out of every cache as committed content: mark it
40
+ * `mayRetain: false` through `TransactorSource.describeServed`, and record no read dependency
41
+ * for it. The content is no committed revision, and the answer's `materialized` revision names
42
+ * the base under the overlay, not the content (see "Staged Edits Keep Their Base" in
43
+ * `docs/internals.md`). */
30
44
  actionId?: ActionId;
31
45
  };
32
46
 
@@ -13,18 +13,20 @@ import { TransactorSource, answeredBlock, servedRevision } from "../transactor/t
13
13
  import { BlockFloors } from "../transactor/block-floors.js";
14
14
  import type { WriteDurability } from "../network/struct.js";
15
15
  import { mergeDurability } from "../network/durability.js";
16
- import { highestStaleAt } from "../network/stale-failure.js";
16
+ import { highestStaleAt, isExpiryFailure } from "../network/stale-failure.js";
17
+ import { TransactionExpiredError } from "../network/transaction-expired-error.js";
17
18
  import type { CollectionHeaderBlock, CollectionId, ICollection, SyncOptions, TornActionReason } from "./index.js";
18
19
  import { CollectionHeaderVanishedError, SyncRetryExhaustedError, SyncRevisionStalledError, TornActionError } from "./struct.js";
19
20
  import type { ActionContext } from "./action.js";
20
21
  import { actionIdAt } from "./action.js";
21
22
  import type { ReadDependency } from "../transaction/transaction.js";
22
- import { clampPriority } from "../transaction/transaction.js";
23
+ import { clampPriority, SlotHoldAfterLosses } from "../transaction/transaction.js";
23
24
  import { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
24
25
  import { randomBytes } from '@noble/hashes/utils.js';
25
26
  import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
26
27
  import { Latches } from "../utility/latches.js";
27
28
  import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
29
+ import type { JitteredBackoffConfig, RandFn } from "../utility/backoff.js";
28
30
  import { createLogger } from "../logger.js";
29
31
 
30
32
  const log = createLogger('collection');
@@ -116,6 +118,32 @@ const DefaultMaxBackoffMs = 5000;
116
118
  * spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
117
119
  const DefaultMaxStalledAttempts = 2;
118
120
 
121
+ /**
122
+ * The sleep before the refresh that follows a write's `failures`-th consecutive loss.
123
+ *
124
+ * The jittered backoff exists to separate a herd of writers that lost the SAME race, so they do not
125
+ * re-collide on the next tick. A loss to a rival that had already committed (the refusal carried a
126
+ * confirmed `staleAt`) is not a herd, and once this write has taken `SlotHoldAfterLosses` such losses
127
+ * (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
128
+ * action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
129
+ * Both sides count the same refusals against the same constant, so the writer stops sleeping on the
130
+ * loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
131
+ * confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
132
+ *
133
+ * NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member the
134
+ * refused pends did not reach) this retries without sleeping and so spends the same attempt budget
135
+ * sooner, at about twice one writer's pend rate (measured in the plan stage of
136
+ * `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
137
+ * delayed the same give-up.
138
+ */
139
+ function retryBackoffMs(failures: number, confirmedLosses: number, latestConfirmed: boolean,
140
+ config: JitteredBackoffConfig, rand?: RandFn): number {
141
+ if (latestConfirmed && confirmedLosses >= SlotHoldAfterLosses) {
142
+ return 0;
143
+ }
144
+ return jitteredBackoffMs(failures - 1, config, rand);
145
+ }
146
+
119
147
  export type CollectionInitOptions<TAction> = {
120
148
  modules: Record<ActionType, ActionHandler<TAction>>;
121
149
  createHeaderBlock: (id: BlockId, store: BlockStore<IBlock>) => IBlock;
@@ -742,6 +770,11 @@ export class Collection<TAction> implements ICollection<TAction> {
742
770
  // documented as never a retryability signal (docs/internals.md). It is the same kind of use
743
771
  // `syncAttempts`' stall check makes: it can only END a retry, never start one.
744
772
  const rivalConfirmed = result.staleAt !== undefined || (result.missing?.length ?? 0) > 0;
773
+ // NOTE: an expiry refusal of the re-send (`isExpiryFailure`) lands in the "can clear" arm and
774
+ // is re-sent each round, though it clears only if a clock is corrected meanwhile. Reaching it
775
+ // needs the clocks to start disagreeing between a write's attempts, since an expiry on the
776
+ // attempt itself ends the write before any refresh. If it is ever seen burning a budget,
777
+ // settle it at once beside the confirmed rival.
745
778
  // NOTE: a confirmed rival settles at once, even with retry budget left: re-sending is
746
779
  // pointless, but the lineage question is asked only this once, so a member that is silent
747
780
  // just now makes the answer `final: false`. If unsettled outcomes ever show up often under
@@ -1764,6 +1797,10 @@ export class Collection<TAction> implements ICollection<TAction> {
1764
1797
  // responder to have re-confirmed the number this round, not merely an older observation
1765
1798
  // left standing in `lastStaleAt`.
1766
1799
  let lastFailureConfirmedStaleAt = false;
1800
+ // Refused attempts of this write that carried their own staleAt. Never reset: a member counts
1801
+ // its stale refusals per action for the action's whole life, and this is the writer's view of
1802
+ // that count (see retryBackoffMs).
1803
+ let confirmedLosses = 0;
1767
1804
  // Consecutive refreshes that moved `getNextRev()` nowhere at all while a confirmed revision
1768
1805
  // stood at or above it.
1769
1806
  let consecutiveStalls = 0;
@@ -1895,6 +1932,19 @@ export class Collection<TAction> implements ICollection<TAction> {
1895
1932
  // First attempt has consecutiveFailures == 0, so priority 0 — the common pend is unchanged.
1896
1933
  const attempt = await this.source.transact(tracker.transforms, actionId, newRev, this.id, addResult.tailPath.block.header.id, clampPriority(consecutiveFailures), blockDigests, baseRevs);
1897
1934
  if (!attempt.success) {
1935
+ if (isExpiryFailure(attempt)) {
1936
+ // The next attempt's expiration comes from the same clock, so it would be refused the
1937
+ // same way; stop here by name. Nothing on this instance has been touched — the attempt
1938
+ // ran on a snapshot tracker — so the staged actions are still staged for a sync after
1939
+ // the clock is fixed.
1940
+ // NOTE: an expiry refusing a commit SWEEP (`NetworkTransactor.commitTailThenSweep`)
1941
+ // arrives after the log tail was stored, and no refresh follows this throw, so that
1942
+ // attempt's log entry stays behind with none of its data — the state the budget-exhausted
1943
+ // exit below leaves too. Needs a sweep cohort whose clocks disagree while the tail's agree;
1944
+ // tracked with the other leftover-entry exits in
1945
+ // tickets/backlog/bug-a-refused-write-can-leave-its-log-entry-behind.
1946
+ throw new TransactionExpiredError(attempt.expired);
1947
+ }
1898
1948
  consecutiveFailures++;
1899
1949
  lastReason = attempt.reason ?? lastReason;
1900
1950
  // Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
@@ -1902,6 +1952,7 @@ export class Collection<TAction> implements ICollection<TAction> {
1902
1952
  // producers and the transactor's aggregation already use.
1903
1953
  lastStaleAt = highestStaleAt([lastStaleAt, attempt.staleAt]);
1904
1954
  lastFailureConfirmedStaleAt = attempt.staleAt !== undefined;
1955
+ if (lastFailureConfirmedStaleAt) confirmedLosses++;
1905
1956
  // Give up once the consecutive no-progress budget is exhausted, so a transactor that
1906
1957
  // persistently rejects the sync can no longer hold the collection latch forever.
1907
1958
  // NOTE: this also bounds the legitimate `pending`-wait case (retrying the same action
@@ -1937,17 +1988,23 @@ export class Collection<TAction> implements ICollection<TAction> {
1937
1988
  // is byte-identical — see logAppendBlockIds — so what is at stake is the second entry,
1938
1989
  // not two versions of one tail block.) Each round is a failure against the same
1939
1990
  // no-progress budget as a refused attempt.
1991
+ // Whether the failure just handled was a loss to a rival already durably committed. Only
1992
+ // the attempt's own refusal can say so; a refresh refused for a cause that can clear
1993
+ // (below) is a race against something still in flight, and clears it.
1994
+ let latestConfirmed = lastFailureConfirmedStaleAt;
1940
1995
  for (;;) {
1941
1996
  // Back off before every retry (any stale failure — reason/missing/pending), growing
1942
1997
  // exponentially from the base delay up to the cap, with proportional random jitter so a
1943
1998
  // herd of clients that lost the same race does not re-collide on the next tick (see
1944
- // utility/backoff.ts). The abortable sleep lets an aborted sync reject promptly instead
1945
- // of finishing the sleep.
1999
+ // utility/backoff.ts) — except once this write has lost to committed rivals often enough
2000
+ // that a member holds the next slot for it (see retryBackoffMs). The abortable sleep lets
2001
+ // an aborted sync reject promptly instead of finishing the sleep.
1946
2002
  // NOTE: the `missing`/`reason` conflict paths now pay this backoff too (they previously
1947
2003
  // retried with zero delay); that is what stops the persistent-`reason` hot spin. If a
1948
2004
  // high-contention workload ever shows this base delay as recovery latency, lower
1949
2005
  // baseBackoffMs for that caller rather than reintroducing the zero-delay retry.
1950
- const delay = jitteredBackoffMs(consecutiveFailures - 1, { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
2006
+ const delay = retryBackoffMs(consecutiveFailures, confirmedLosses, latestConfirmed,
2007
+ { baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
1951
2008
  await abortableDelay(delay, signal);
1952
2009
  try {
1953
2010
  // Fetch latest state - updateInternal() will call replayActions() if there are conflicts.
@@ -1986,6 +2043,7 @@ export class Collection<TAction> implements ICollection<TAction> {
1986
2043
  // is what escapes: the log holds an entry for a write that was not saved.
1987
2044
  consecutiveFailures++;
1988
2045
  lastReason = err.detail;
2046
+ latestConfirmed = false;
1989
2047
  if (consecutiveFailures >= maxAttempts
1990
2048
  || (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs)) {
1991
2049
  throw err;
@@ -19,9 +19,13 @@ export interface SyncOptions {
19
19
  * of the attempt count — a progress-agnostic ceiling. Unset means no deadline. */
20
20
  deadlineMs?: number;
21
21
  /** Base backoff delay in ms applied before the first retry; subsequent retries grow the delay
22
- * exponentially up to {@link maxBackoffMs}. Default 100. */
22
+ * exponentially up to {@link maxBackoffMs}. Default 100. Not applied once `SlotHoldAfterLosses` of the
23
+ * write's refused attempts carried a confirmed committed revision (`StaleFailure.staleAt`) and the
24
+ * latest refusal is one of them: a member then holds the next slot for this write,
25
+ * and the retry follows at once rather than spending the hold asleep. */
23
26
  baseBackoffMs?: number;
24
- /** Upper bound on any single backoff sleep, in ms. Default 5000. */
27
+ /** Upper bound on any single backoff sleep, in ms. Default 5000. Like {@link baseBackoffMs}, not
28
+ * applied to a retry that follows at once because a member holds the slot for this write. */
25
29
  maxBackoffMs?: number;
26
30
  /** Optional abort signal. Checked at the top of each loop iteration and raced against the
27
31
  * backoff sleep, so an aborted sync rejects promptly (with an AbortError) rather than finishing
package/src/index.ts CHANGED
@@ -14,6 +14,7 @@ export * from "./transaction/index.js";
14
14
  export * from "./transactor/index.js";
15
15
  export * from "./transform/index.js";
16
16
  export * from "./utility/canonical-json.js";
17
+ export * from "./utility/format-instant.js";
17
18
  export * from "./utility/groupby.js";
18
19
  export * from "./utility/hash-string.js";
19
20
  export * from "./utility/latches.js";
@@ -0,0 +1,66 @@
1
+ /**
2
+ * Matchmaking — arrival-push fan-out selection (db-core, pure).
3
+ *
4
+ * Decides which push-opted seekers a newly arrived provider is pushed to (`docs/matchmaking.md`
5
+ * §Fairness — FCFS by `attachedAt`, fan-out bounded by `capacityBudget`). Every cohort member runs the
6
+ * same selection over the same gossip-replicated records and then delivers only to the selected seekers
7
+ * whose slot primary it is, so the cohort as a whole notifies about `capacityBudget` seekers rather than
8
+ * that many per member.
9
+ *
10
+ * Pure: no clock, no I/O. The db-p2p cohort driver decodes the records, keeps the targets it is primary
11
+ * for, coalesces, signs and delivers.
12
+ */
13
+
14
+ import { matchesFilter } from "./capability-filter.js";
15
+ import type { LocalProviderRegistration, LocalSeekerRegistration } from "./query-eval.js";
16
+
17
+ /** A decoded seeker registration held at this cohort, considered as a push target. */
18
+ export type ArrivalPushCandidate = LocalSeekerRegistration;
19
+
20
+ /**
21
+ * Seekers to notify about `provider`, longest-waiting first: of the push-opted seekers whose filter the
22
+ * provider matches and that attached no later than it did, the first `capacityBudget` by `attachedAt`
23
+ * (ties broken by `participantId`, so every member computes the same order). Generic so the caller gets
24
+ * back its own richer candidate objects.
25
+ *
26
+ * The `attachedAt` rule exists for a member that receives the whole record set at once (catching up by
27
+ * gossip, or pulling records in a rotation handoff): without it, that member would push every existing
28
+ * provider to every seeker. A provider that attached before a seeker was answerable by that seeker's own
29
+ * first query, and its safety poll still finds it.
30
+ *
31
+ * NOTE: accepted tradeoff — the ranking reads only replicated state, so a seeker already satisfied,
32
+ * finished, or answering `unknown_seeker` to its own primary keeps holding a fan-out slot until its
33
+ * record leaves the cohort (withdraw on finish, or its `seeker_ttl` expiry), under-filling that arrival's fan-out;
34
+ * members whose gossip views differ can also over- or under-fill by a few. Excluding them would mean
35
+ * gossiping per-seeker push state; the seeker's safety poll covers what is missed. Revisit if push
36
+ * under-fill shows up as seekers routinely falling back to the safety poll on a busy topic.
37
+ */
38
+ export function selectArrivalPushTargets<S extends ArrivalPushCandidate>(
39
+ provider: LocalProviderRegistration,
40
+ seekers: readonly S[],
41
+ ): S[] {
42
+ const budget = provider.payload.capacityBudget;
43
+ if (budget === 0) {
44
+ return [];
45
+ }
46
+ return seekers
47
+ .filter((s) => isPushTarget(provider, s))
48
+ .sort(byLongestWaiting)
49
+ .slice(0, budget);
50
+ }
51
+
52
+ /** True iff `seeker` holds a push binding, matches `provider`, and was waiting when it arrived. */
53
+ function isPushTarget(provider: LocalProviderRegistration, seeker: ArrivalPushCandidate): boolean {
54
+ return seeker.payload.pushOnArrival === true
55
+ && seeker.payload.correlationId !== undefined
56
+ && matchesFilter(provider.payload, seeker.payload.filter)
57
+ && provider.attachedAt >= seeker.attachedAt;
58
+ }
59
+
60
+ /** Ascending `attachedAt`, then code-unit order of `participantId` (locale-independent, so members agree). */
61
+ function byLongestWaiting(a: ArrivalPushCandidate, b: ArrivalPushCandidate): number {
62
+ if (a.attachedAt !== b.attachedAt) {
63
+ return a.attachedAt - b.attachedAt;
64
+ }
65
+ return a.participantId < b.participantId ? -1 : a.participantId > b.participantId ? 1 : 0;
66
+ }
@@ -10,8 +10,9 @@
10
10
  * The TTL/limit constants are wire-stable and do **not** depend on simulator findings; the hang-out
11
11
  * rows DO (folded via `fold-simulator-findings-into-design-docs`) — they carry the simulator-validated
12
12
  * defaults (`contention_factor_cap = 4.0`, `mean_want_count = 3`, …; see `docs/matchmaking.md`
13
- * §Configuration "Defaults validated by simulator"). The `push_*` rows belong to the arrival-push
14
- * path (a separate slice) and are intentionally not modeled here.
13
+ * §Configuration "Defaults validated by simulator"). Of the arrival-push rows, `push_safety_poll_ms` is
14
+ * seeker-side and lives on {@link HangOutConfig}; `push_coalesce_ms` is the one cohort-side knob and
15
+ * lives on {@link ArrivalPushConfig}.
15
16
  */
16
17
 
17
18
  import type { NodeProfile } from "../cohort-topic/tiers.js";
@@ -61,12 +62,26 @@ export interface HangOutConfig {
61
62
  readonly contentionFactorCap: number;
62
63
  /** Hang-out poll cadence on the non-push path. */
63
64
  readonly requeryIntervalMs: number;
65
+ /** Sparse safety-poll cadence beneath the pushes, for a seeker that set `pushOnArrival`. */
66
+ readonly pushSafetyPollMs: number;
64
67
  }
65
68
 
66
69
  /** The default hang-out decision config (simulator-validated). */
67
70
  export const DEFAULT_HANG_OUT_CONFIG: HangOutConfig = {
68
71
  contentionFactorCap: CONTENTION_FACTOR_CAP,
69
72
  requeryIntervalMs: REQUERY_INTERVAL_MS,
73
+ pushSafetyPollMs: PUSH_SAFETY_POLL_MS,
74
+ };
75
+
76
+ /** Cohort-side arrival-push tuning (`docs/matchmaking.md` §Coalescing). */
77
+ export interface ArrivalPushConfig {
78
+ /** Window the pushing member batches fresh matchable arrivals per seeker before flushing one push. */
79
+ readonly coalesceMs: number;
80
+ }
81
+
82
+ /** The default arrival-push config. */
83
+ export const DEFAULT_ARRIVAL_PUSH_CONFIG: ArrivalPushConfig = {
84
+ coalesceMs: PUSH_COALESCE_MS,
70
85
  };
71
86
 
72
87
  /** The full matchmaking config, with the documented defaults. */
@@ -4,8 +4,8 @@
4
4
  * See `docs/matchmaking.md`. The db-core layer owns the transport-agnostic pieces: wire codecs +
5
5
  * seeker-side entry re-validation, the stable topic-anchor derivation, configuration, both
6
6
  * registration roles' decision/state (provider attach/renew/self-throttle; seeker short-TTL
7
- * registration), the cohort-side capability filter + pure query evaluation, and the seeker
8
- * hang-out-vs-continue decision engine. The db-p2p layer wires these to the cohort-topic substrate
7
+ * registration), the cohort-side capability filter + pure query evaluation, the arrival-push fan-out
8
+ * selection, and the seeker hang-out-vs-continue decision engine. The db-p2p layer wires these to the cohort-topic substrate
9
9
  * (managers, the cohort query handler, the seeker walk client).
10
10
  */
11
11
 
@@ -16,6 +16,7 @@ export * from "./provider.js";
16
16
  export * from "./seeker.js";
17
17
  export * from "./capability-filter.js";
18
18
  export * from "./query-eval.js";
19
+ export * from "./arrival-push.js";
19
20
  export * from "./seeker-walk.js";
20
21
  export * from "./multi-cohort-seeker.js";
21
22
  export * from "./voting-quorum.js";
@@ -15,6 +15,8 @@
15
15
  */
16
16
 
17
17
  import { randomBytes } from "@noble/hashes/utils.js";
18
+ import { bytesToB64url } from "../cohort-topic/wire/codec.js";
19
+ import { CORRELATION_BYTES } from "../cohort-topic/wire/primitives.js";
18
20
  import { seekerSigningPayload, type CapabilityFilter, type SeekerAppPayloadV1, encodeSeekerAppPayload } from "./wire.js";
19
21
 
20
22
  /** Construction inputs for a {@link MatchmakingSeeker}. */
@@ -27,11 +29,14 @@ export interface MatchmakingSeekerOptions {
27
29
  readonly contactHint: string;
28
30
  /** Optional capability filter (evaluated cohort-side in the next ticket; carried here). */
29
31
  readonly filter?: CapabilityFilter;
30
- /** Opt into arrival pushes; default false (poll path). Consumed by the next ticket. */
32
+ /** Opt into arrival pushes; default false (poll path). */
31
33
  readonly pushOnArrival?: boolean;
32
34
  /** Sign the canonical registration image; resolves the base64url signature. */
33
35
  readonly sign: (payload: Uint8Array) => Promise<string>;
34
- /** 16-byte registration correlation id (not signature-bound); default fresh CSPRNG bytes. */
36
+ /**
37
+ * 16-byte registration correlation id (not signature-bound); default fresh CSPRNG bytes. Sent in the
38
+ * payload when `pushOnArrival` is set, as the id arrival pushes are bound to.
39
+ */
35
40
  readonly correlationId?: Uint8Array;
36
41
  /** CSPRNG source (injectable for deterministic tests). Default `@noble/hashes` `randomBytes`. */
37
42
  readonly randomBytes?: (n: number) => Uint8Array;
@@ -58,7 +63,7 @@ export class MatchmakingSeeker {
58
63
  this.pushOnArrival = options.pushOnArrival;
59
64
  this.sign = options.sign;
60
65
  const rand = options.randomBytes ?? randomBytes;
61
- this.correlationId = options.correlationId ?? rand(16);
66
+ this.correlationId = options.correlationId ?? rand(CORRELATION_BYTES);
62
67
  }
63
68
 
64
69
  /** Build the signed {@link SeekerAppPayloadV1} for this seeker's registration. */
@@ -76,6 +81,9 @@ export class MatchmakingSeeker {
76
81
  if (this.pushOnArrival !== undefined) {
77
82
  payload.pushOnArrival = this.pushOnArrival;
78
83
  }
84
+ if (this.pushOnArrival === true) {
85
+ payload.correlationId = bytesToB64url(this.correlationId);
86
+ }
79
87
  return payload;
80
88
  }
81
89