@optimystic/db-core 1.10.1 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +48 -7
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js +33 -13
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/cohort-topic/coldstart.d.ts +8 -4
- package/dist/src/cohort-topic/coldstart.d.ts.map +1 -1
- package/dist/src/cohort-topic/coldstart.js +3 -3
- package/dist/src/cohort-topic/coldstart.js.map +1 -1
- package/dist/src/cohort-topic/member-engine.d.ts +5 -1
- package/dist/src/cohort-topic/member-engine.d.ts.map +1 -1
- package/dist/src/cohort-topic/member-engine.js.map +1 -1
- package/dist/src/cohort-topic/registration/store.d.ts +8 -1
- package/dist/src/cohort-topic/registration/store.d.ts.map +1 -1
- package/dist/src/cohort-topic/registration/store.js +57 -0
- package/dist/src/cohort-topic/registration/store.js.map +1 -1
- package/dist/src/cohort-topic/willingness.d.ts +4 -3
- package/dist/src/cohort-topic/willingness.d.ts.map +1 -1
- package/dist/src/cohort-topic/willingness.js.map +1 -1
- package/dist/src/cohort-topic/wire/primitives.d.ts +5 -0
- package/dist/src/cohort-topic/wire/primitives.d.ts.map +1 -1
- package/dist/src/cohort-topic/wire/primitives.js +5 -0
- package/dist/src/cohort-topic/wire/primitives.js.map +1 -1
- package/dist/src/cohort-topic/wire/validate.d.ts.map +1 -1
- package/dist/src/cohort-topic/wire/validate.js +1 -3
- package/dist/src/cohort-topic/wire/validate.js.map +1 -1
- package/dist/src/collection/action.d.ts +15 -1
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +60 -5
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +6 -2
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/matchmaking/arrival-push.d.ts +35 -0
- package/dist/src/matchmaking/arrival-push.d.ts.map +1 -0
- package/dist/src/matchmaking/arrival-push.js +56 -0
- package/dist/src/matchmaking/arrival-push.js.map +1 -0
- package/dist/src/matchmaking/config.d.ts +12 -2
- package/dist/src/matchmaking/config.d.ts.map +1 -1
- package/dist/src/matchmaking/config.js +8 -2
- package/dist/src/matchmaking/config.js.map +1 -1
- package/dist/src/matchmaking/index.d.ts +3 -2
- package/dist/src/matchmaking/index.d.ts.map +1 -1
- package/dist/src/matchmaking/index.js +3 -2
- package/dist/src/matchmaking/index.js.map +1 -1
- package/dist/src/matchmaking/seeker.d.ts +5 -2
- package/dist/src/matchmaking/seeker.d.ts.map +1 -1
- package/dist/src/matchmaking/seeker.js +6 -1
- package/dist/src/matchmaking/seeker.js.map +1 -1
- package/dist/src/matchmaking/wire.d.ts +66 -3
- package/dist/src/matchmaking/wire.d.ts.map +1 -1
- package/dist/src/matchmaking/wire.js +79 -11
- package/dist/src/matchmaking/wire.js.map +1 -1
- package/dist/src/network/i-key-network.d.ts +8 -0
- package/dist/src/network/i-key-network.d.ts.map +1 -1
- package/dist/src/network/index.d.ts +1 -0
- package/dist/src/network/index.d.ts.map +1 -1
- package/dist/src/network/index.js +1 -0
- package/dist/src/network/index.js.map +1 -1
- package/dist/src/network/stale-failure.d.ts +12 -3
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +14 -1
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +54 -4
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/network/transaction-expired-error.d.ts +44 -0
- package/dist/src/network/transaction-expired-error.d.ts.map +1 -0
- package/dist/src/network/transaction-expired-error.js +77 -0
- package/dist/src/network/transaction-expired-error.js.map +1 -0
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +28 -46
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +4 -1
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +31 -5
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/index.d.ts +1 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -1
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +27 -0
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/transaction.js +27 -0
- package/dist/src/transaction/transaction.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +32 -39
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +6 -2
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/utility/format-instant.d.ts +8 -0
- package/dist/src/utility/format-instant.d.ts.map +1 -0
- package/dist/src/utility/format-instant.js +11 -0
- package/dist/src/utility/format-instant.js.map +1 -0
- package/package.json +1 -1
- package/src/cluster/structs.ts +72 -14
- package/src/cohort-topic/coldstart.ts +8 -4
- package/src/cohort-topic/member-engine.ts +5 -1
- package/src/cohort-topic/registration/store.ts +68 -0
- package/src/cohort-topic/willingness.ts +4 -3
- package/src/cohort-topic/wire/primitives.ts +6 -0
- package/src/cohort-topic/wire/validate.ts +1 -3
- package/src/collection/action.ts +15 -1
- package/src/collection/collection.ts +63 -5
- package/src/collection/struct.ts +6 -2
- package/src/index.ts +1 -0
- package/src/matchmaking/arrival-push.ts +66 -0
- package/src/matchmaking/config.ts +17 -2
- package/src/matchmaking/index.ts +3 -2
- package/src/matchmaking/seeker.ts +11 -3
- package/src/matchmaking/wire.ts +134 -10
- package/src/network/i-key-network.ts +9 -0
- package/src/network/index.ts +1 -0
- package/src/network/stale-failure.ts +16 -3
- package/src/network/struct.ts +57 -4
- package/src/network/transaction-expired-error.ts +81 -0
- package/src/testing/test-transactor.ts +28 -40
- package/src/transaction/coordinator.ts +37 -7
- package/src/transaction/index.ts +2 -1
- package/src/transaction/transaction.ts +28 -0
- package/src/transactor/network-transactor.ts +36 -43
- package/src/transactor/transactor-source.ts +6 -2
- package/src/utility/format-instant.ts +10 -0
|
@@ -8,9 +8,12 @@
|
|
|
8
8
|
* local soft state; cross-member replication runs over cohort gossip in a later ticket.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
+
import { createLogger } from "../../logger.js";
|
|
11
12
|
import { bytesKey } from "./bytes.js";
|
|
12
13
|
import type { RegistrationRecord, RegistrationStore } from "./types.js";
|
|
13
14
|
|
|
15
|
+
const log = createLogger("cohort-topic:registration");
|
|
16
|
+
|
|
14
17
|
class InMemoryRegistrationStore implements RegistrationStore {
|
|
15
18
|
/** topicKey → (participantKey → record). Inner maps are pruned when they empty. */
|
|
16
19
|
private readonly byTopic = new Map<string, Map<string, RegistrationRecord>>();
|
|
@@ -79,3 +82,68 @@ class InMemoryRegistrationStore implements RegistrationStore {
|
|
|
79
82
|
export function createRegistrationStore(): RegistrationStore {
|
|
80
83
|
return new InMemoryRegistrationStore();
|
|
81
84
|
}
|
|
85
|
+
|
|
86
|
+
/** Called once per record that a `put` made present where the store held none for its participant. */
|
|
87
|
+
export type RecordAddedListener = (rec: RegistrationRecord) => void;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* A store that reports each absent→present `put`. Every way a record reaches a member — local
|
|
91
|
+
* admission, a gossip merge, a rotation-handoff pull — goes through `put`, so this is the one place
|
|
92
|
+
* a "a registration arrived here" signal can be taken without missing a path. A `put` that replaces
|
|
93
|
+
* a held record (a renewal touch, a re-attach restamp, a re-merge) reports nothing.
|
|
94
|
+
*/
|
|
95
|
+
class RecordAdditionObserver implements RegistrationStore {
|
|
96
|
+
constructor(
|
|
97
|
+
private readonly inner: RegistrationStore,
|
|
98
|
+
private readonly onAdded: RecordAddedListener,
|
|
99
|
+
) {}
|
|
100
|
+
|
|
101
|
+
put(rec: RegistrationRecord): void {
|
|
102
|
+
const added = this.inner.getByParticipant(rec.topicId, rec.participantId) === undefined;
|
|
103
|
+
this.inner.put(rec);
|
|
104
|
+
if (added) {
|
|
105
|
+
this.notify(rec);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
getByParticipant(topicId: Uint8Array, participantId: Uint8Array): RegistrationRecord | undefined {
|
|
110
|
+
return this.inner.getByParticipant(topicId, participantId);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
listByTopic(topicId: Uint8Array): readonly RegistrationRecord[] {
|
|
114
|
+
return this.inner.listByTopic(topicId);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
listAll(): readonly RegistrationRecord[] {
|
|
118
|
+
return this.inner.listAll();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
delete(topicId: Uint8Array, participantId: Uint8Array): void {
|
|
122
|
+
this.inner.delete(topicId, participantId);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
directParticipants(topicId: Uint8Array): number {
|
|
126
|
+
return this.inner.directParticipants(topicId);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
evictStale(now: number): readonly RegistrationRecord[] {
|
|
130
|
+
return this.inner.evictStale(now);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** A listener fault must not abort the admission or gossip merge that called `put`. */
|
|
134
|
+
private notify(rec: RegistrationRecord): void {
|
|
135
|
+
try {
|
|
136
|
+
this.onAdded(rec);
|
|
137
|
+
} catch (err) {
|
|
138
|
+
log("record-added listener threw for topic %s participant %s: %o", bytesKey(rec.topicId), bytesKey(rec.participantId), err);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Wrap `store` so `onAdded` fires after each `put` that makes a `(topicId, participantId)` record
|
|
145
|
+
* present where none was held. A listener throw is logged, never propagated into `put`.
|
|
146
|
+
*/
|
|
147
|
+
export function observeRecordAdditions(store: RegistrationStore, onAdded: RecordAddedListener): RegistrationStore {
|
|
148
|
+
return new RecordAdditionObserver(store, onAdded);
|
|
149
|
+
}
|
|
@@ -133,9 +133,10 @@ export interface WillingnessConfig {
|
|
|
133
133
|
/** Cohort size `k` (drives the default quorum). Default 16. */
|
|
134
134
|
cohortSize?: number;
|
|
135
135
|
/**
|
|
136
|
-
* Members that must be willing to serve a tier for the cohort to take it on
|
|
137
|
-
*
|
|
138
|
-
*
|
|
136
|
+
* Members that must be willing to serve a tier for the cohort to take it on (the admission quorum,
|
|
137
|
+
* `docs/cohort-topic.md` §Admission quorum). Default: strict majority of `cohortSize`
|
|
138
|
+
* ({@link defaultQuorum}). Deliberately not the threshold-signature `minSigs = k − x` — see the doc
|
|
139
|
+
* for why the two differ and when to revisit the default.
|
|
139
140
|
*/
|
|
140
141
|
quorum?: number;
|
|
141
142
|
/** Load bucket at/above which a tier is shed. Default {@link DEFAULT_OVERLOAD_BUCKET} (6). */
|
|
@@ -23,6 +23,12 @@ import { b64urlToBytes } from "./codec.js";
|
|
|
23
23
|
*/
|
|
24
24
|
export const COORD_BYTES = 32;
|
|
25
25
|
|
|
26
|
+
/**
|
|
27
|
+
* Correlation-id byte width: the 16-byte nonce the cohort-topic substrate mints per walk probe / renew,
|
|
28
|
+
* and the matchmaking seeker registration id an arrival push is bound to.
|
|
29
|
+
*/
|
|
30
|
+
export const CORRELATION_BYTES = 16;
|
|
31
|
+
|
|
26
32
|
/** Thrown for any malformed, oversized, or structurally invalid cohort-topic frame. */
|
|
27
33
|
export class CohortWireError extends Error {
|
|
28
34
|
constructor(message: string) {
|
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
b64urlField,
|
|
17
17
|
b64urlFixedLen,
|
|
18
18
|
COORD_BYTES,
|
|
19
|
+
CORRELATION_BYTES,
|
|
19
20
|
failWire as fail,
|
|
20
21
|
optBool,
|
|
21
22
|
optFiniteNumber,
|
|
@@ -78,9 +79,6 @@ function treeTier(value: number, what: string): number {
|
|
|
78
79
|
return value;
|
|
79
80
|
}
|
|
80
81
|
|
|
81
|
-
/** Correlation-id byte width (a 16-byte nonce minted per walk probe / renew). */
|
|
82
|
-
const CORRELATION_BYTES = 16;
|
|
83
|
-
|
|
84
82
|
/**
|
|
85
83
|
* Ceiling on a decoded `rootKey` (the root key of a root-placed topic). A root key is a block's routing
|
|
86
84
|
* key — the UTF-8 of its block id, 43 bytes for a generated id — so 256 leaves room for a named block
|
package/src/collection/action.ts
CHANGED
|
@@ -26,7 +26,21 @@ export type ActionContext = {
|
|
|
26
26
|
committed: ActionRev[];
|
|
27
27
|
/** The latest known revision number */
|
|
28
28
|
rev: number;
|
|
29
|
-
/** Optional uncommitted pending action
|
|
29
|
+
/** Optional uncommitted pending action to read through. A repo overlays this action's pending
|
|
30
|
+
* transform on each block where it holds a pending record of it, and answers every other block
|
|
31
|
+
* exactly as a plain read with the same context would; the overlay answer's `state.pendings`
|
|
32
|
+
* names the action, the plain one's does not.
|
|
33
|
+
*
|
|
34
|
+
* No production code sets it. It is reserved for tentative reads in long-lived pend (backlog
|
|
35
|
+
* `feat-long-lived-pend-completes-as-members-appear`), where a read that observes a
|
|
36
|
+
* pended-but-uncommitted revision makes the reading transaction tentative too. It is
|
|
37
|
+
* deliberately not used for early abort: the only certain abort signal — a latest revision at
|
|
38
|
+
* or past the pended one under another action — is already on a plain read's `state.latest`.
|
|
39
|
+
* Whoever sets it must keep the overlay answer out of every cache as committed content: mark it
|
|
40
|
+
* `mayRetain: false` through `TransactorSource.describeServed`, and record no read dependency
|
|
41
|
+
* for it. The content is no committed revision, and the answer's `materialized` revision names
|
|
42
|
+
* the base under the overlay, not the content (see "Staged Edits Keep Their Base" in
|
|
43
|
+
* `docs/internals.md`). */
|
|
30
44
|
actionId?: ActionId;
|
|
31
45
|
};
|
|
32
46
|
|
|
@@ -13,18 +13,20 @@ import { TransactorSource, answeredBlock, servedRevision } from "../transactor/t
|
|
|
13
13
|
import { BlockFloors } from "../transactor/block-floors.js";
|
|
14
14
|
import type { WriteDurability } from "../network/struct.js";
|
|
15
15
|
import { mergeDurability } from "../network/durability.js";
|
|
16
|
-
import { highestStaleAt } from "../network/stale-failure.js";
|
|
16
|
+
import { highestStaleAt, isExpiryFailure } from "../network/stale-failure.js";
|
|
17
|
+
import { TransactionExpiredError } from "../network/transaction-expired-error.js";
|
|
17
18
|
import type { CollectionHeaderBlock, CollectionId, ICollection, SyncOptions, TornActionReason } from "./index.js";
|
|
18
19
|
import { CollectionHeaderVanishedError, SyncRetryExhaustedError, SyncRevisionStalledError, TornActionError } from "./struct.js";
|
|
19
20
|
import type { ActionContext } from "./action.js";
|
|
20
21
|
import { actionIdAt } from "./action.js";
|
|
21
22
|
import type { ReadDependency } from "../transaction/transaction.js";
|
|
22
|
-
import { clampPriority } from "../transaction/transaction.js";
|
|
23
|
+
import { clampPriority, SlotHoldAfterLosses } from "../transaction/transaction.js";
|
|
23
24
|
import { ReadDependencyCollector } from "../transaction/read-dependency-collector.js";
|
|
24
25
|
import { randomBytes } from '@noble/hashes/utils.js';
|
|
25
26
|
import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
|
|
26
27
|
import { Latches } from "../utility/latches.js";
|
|
27
28
|
import { jitteredBackoffMs, abortableDelay, makeAbortError } from "../utility/backoff.js";
|
|
29
|
+
import type { JitteredBackoffConfig, RandFn } from "../utility/backoff.js";
|
|
28
30
|
import { createLogger } from "../logger.js";
|
|
29
31
|
|
|
30
32
|
const log = createLogger('collection');
|
|
@@ -116,6 +118,32 @@ const DefaultMaxBackoffMs = 5000;
|
|
|
116
118
|
* spurious `SyncRevisionStalledError` ever shows up under real contention anyway, raise this. */
|
|
117
119
|
const DefaultMaxStalledAttempts = 2;
|
|
118
120
|
|
|
121
|
+
/**
|
|
122
|
+
* The sleep before the refresh that follows a write's `failures`-th consecutive loss.
|
|
123
|
+
*
|
|
124
|
+
* The jittered backoff exists to separate a herd of writers that lost the SAME race, so they do not
|
|
125
|
+
* re-collide on the next tick. A loss to a rival that had already committed (the refusal carried a
|
|
126
|
+
* confirmed `staleAt`) is not a herd, and once this write has taken `SlotHoldAfterLosses` such losses
|
|
127
|
+
* (`confirmedLosses`, counted over the whole write as a member counts its stale refusals of one
|
|
128
|
+
* action) the members that refused it hold the next slot for it — so sleeping only spends that hold.
|
|
129
|
+
* Both sides count the same refusals against the same constant, so the writer stops sleeping on the
|
|
130
|
+
* loss after which a member holds, with nothing said over the wire. A loss the latest refusal did not
|
|
131
|
+
* confirm (`latestConfirmed` false — a race against something still in flight) keeps the jitter.
|
|
132
|
+
*
|
|
133
|
+
* NOTE: against members that do not hold (an earlier release, `slotHoldWindowMs: 0`, or a member the
|
|
134
|
+
* refused pends did not reach) this retries without sleeping and so spends the same attempt budget
|
|
135
|
+
* sooner, at about twice one writer's pend rate (measured in the plan stage of
|
|
136
|
+
* `slot-hold-for-an-aged-writer`). Accepted: the backoff bought that writer nothing either, it only
|
|
137
|
+
* delayed the same give-up.
|
|
138
|
+
*/
|
|
139
|
+
function retryBackoffMs(failures: number, confirmedLosses: number, latestConfirmed: boolean,
|
|
140
|
+
config: JitteredBackoffConfig, rand?: RandFn): number {
|
|
141
|
+
if (latestConfirmed && confirmedLosses >= SlotHoldAfterLosses) {
|
|
142
|
+
return 0;
|
|
143
|
+
}
|
|
144
|
+
return jitteredBackoffMs(failures - 1, config, rand);
|
|
145
|
+
}
|
|
146
|
+
|
|
119
147
|
export type CollectionInitOptions<TAction> = {
|
|
120
148
|
modules: Record<ActionType, ActionHandler<TAction>>;
|
|
121
149
|
createHeaderBlock: (id: BlockId, store: BlockStore<IBlock>) => IBlock;
|
|
@@ -742,6 +770,11 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
742
770
|
// documented as never a retryability signal (docs/internals.md). It is the same kind of use
|
|
743
771
|
// `syncAttempts`' stall check makes: it can only END a retry, never start one.
|
|
744
772
|
const rivalConfirmed = result.staleAt !== undefined || (result.missing?.length ?? 0) > 0;
|
|
773
|
+
// NOTE: an expiry refusal of the re-send (`isExpiryFailure`) lands in the "can clear" arm and
|
|
774
|
+
// is re-sent each round, though it clears only if a clock is corrected meanwhile. Reaching it
|
|
775
|
+
// needs the clocks to start disagreeing between a write's attempts, since an expiry on the
|
|
776
|
+
// attempt itself ends the write before any refresh. If it is ever seen burning a budget,
|
|
777
|
+
// settle it at once beside the confirmed rival.
|
|
745
778
|
// NOTE: a confirmed rival settles at once, even with retry budget left: re-sending is
|
|
746
779
|
// pointless, but the lineage question is asked only this once, so a member that is silent
|
|
747
780
|
// just now makes the answer `final: false`. If unsettled outcomes ever show up often under
|
|
@@ -1764,6 +1797,10 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1764
1797
|
// responder to have re-confirmed the number this round, not merely an older observation
|
|
1765
1798
|
// left standing in `lastStaleAt`.
|
|
1766
1799
|
let lastFailureConfirmedStaleAt = false;
|
|
1800
|
+
// Refused attempts of this write that carried their own staleAt. Never reset: a member counts
|
|
1801
|
+
// its stale refusals per action for the action's whole life, and this is the writer's view of
|
|
1802
|
+
// that count (see retryBackoffMs).
|
|
1803
|
+
let confirmedLosses = 0;
|
|
1767
1804
|
// Consecutive refreshes that moved `getNextRev()` nowhere at all while a confirmed revision
|
|
1768
1805
|
// stood at or above it.
|
|
1769
1806
|
let consecutiveStalls = 0;
|
|
@@ -1895,6 +1932,19 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1895
1932
|
// First attempt has consecutiveFailures == 0, so priority 0 — the common pend is unchanged.
|
|
1896
1933
|
const attempt = await this.source.transact(tracker.transforms, actionId, newRev, this.id, addResult.tailPath.block.header.id, clampPriority(consecutiveFailures), blockDigests, baseRevs);
|
|
1897
1934
|
if (!attempt.success) {
|
|
1935
|
+
if (isExpiryFailure(attempt)) {
|
|
1936
|
+
// The next attempt's expiration comes from the same clock, so it would be refused the
|
|
1937
|
+
// same way; stop here by name. Nothing on this instance has been touched — the attempt
|
|
1938
|
+
// ran on a snapshot tracker — so the staged actions are still staged for a sync after
|
|
1939
|
+
// the clock is fixed.
|
|
1940
|
+
// NOTE: an expiry refusing a commit SWEEP (`NetworkTransactor.commitTailThenSweep`)
|
|
1941
|
+
// arrives after the log tail was stored, and no refresh follows this throw, so that
|
|
1942
|
+
// attempt's log entry stays behind with none of its data — the state the budget-exhausted
|
|
1943
|
+
// exit below leaves too. Needs a sweep cohort whose clocks disagree while the tail's agree;
|
|
1944
|
+
// tracked with the other leftover-entry exits in
|
|
1945
|
+
// tickets/backlog/bug-a-refused-write-can-leave-its-log-entry-behind.
|
|
1946
|
+
throw new TransactionExpiredError(attempt.expired);
|
|
1947
|
+
}
|
|
1898
1948
|
consecutiveFailures++;
|
|
1899
1949
|
lastReason = attempt.reason ?? lastReason;
|
|
1900
1950
|
// Highest-wins, not last-wins: the next request has to clear EVERY holder, so a later
|
|
@@ -1902,6 +1952,7 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1902
1952
|
// producers and the transactor's aggregation already use.
|
|
1903
1953
|
lastStaleAt = highestStaleAt([lastStaleAt, attempt.staleAt]);
|
|
1904
1954
|
lastFailureConfirmedStaleAt = attempt.staleAt !== undefined;
|
|
1955
|
+
if (lastFailureConfirmedStaleAt) confirmedLosses++;
|
|
1905
1956
|
// Give up once the consecutive no-progress budget is exhausted, so a transactor that
|
|
1906
1957
|
// persistently rejects the sync can no longer hold the collection latch forever.
|
|
1907
1958
|
// NOTE: this also bounds the legitimate `pending`-wait case (retrying the same action
|
|
@@ -1937,17 +1988,23 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1937
1988
|
// is byte-identical — see logAppendBlockIds — so what is at stake is the second entry,
|
|
1938
1989
|
// not two versions of one tail block.) Each round is a failure against the same
|
|
1939
1990
|
// no-progress budget as a refused attempt.
|
|
1991
|
+
// Whether the failure just handled was a loss to a rival already durably committed. Only
|
|
1992
|
+
// the attempt's own refusal can say so; a refresh refused for a cause that can clear
|
|
1993
|
+
// (below) is a race against something still in flight, and clears it.
|
|
1994
|
+
let latestConfirmed = lastFailureConfirmedStaleAt;
|
|
1940
1995
|
for (;;) {
|
|
1941
1996
|
// Back off before every retry (any stale failure — reason/missing/pending), growing
|
|
1942
1997
|
// exponentially from the base delay up to the cap, with proportional random jitter so a
|
|
1943
1998
|
// herd of clients that lost the same race does not re-collide on the next tick (see
|
|
1944
|
-
// utility/backoff.ts)
|
|
1945
|
-
//
|
|
1999
|
+
// utility/backoff.ts) — except once this write has lost to committed rivals often enough
|
|
2000
|
+
// that a member holds the next slot for it (see retryBackoffMs). The abortable sleep lets
|
|
2001
|
+
// an aborted sync reject promptly instead of finishing the sleep.
|
|
1946
2002
|
// NOTE: the `missing`/`reason` conflict paths now pay this backoff too (they previously
|
|
1947
2003
|
// retried with zero delay); that is what stops the persistent-`reason` hot spin. If a
|
|
1948
2004
|
// high-contention workload ever shows this base delay as recovery latency, lower
|
|
1949
2005
|
// baseBackoffMs for that caller rather than reintroducing the zero-delay retry.
|
|
1950
|
-
const delay =
|
|
2006
|
+
const delay = retryBackoffMs(consecutiveFailures, confirmedLosses, latestConfirmed,
|
|
2007
|
+
{ baseMs: baseBackoffMs, capMs: maxBackoffMs }, options?.rand);
|
|
1951
2008
|
await abortableDelay(delay, signal);
|
|
1952
2009
|
try {
|
|
1953
2010
|
// Fetch latest state - updateInternal() will call replayActions() if there are conflicts.
|
|
@@ -1986,6 +2043,7 @@ export class Collection<TAction> implements ICollection<TAction> {
|
|
|
1986
2043
|
// is what escapes: the log holds an entry for a write that was not saved.
|
|
1987
2044
|
consecutiveFailures++;
|
|
1988
2045
|
lastReason = err.detail;
|
|
2046
|
+
latestConfirmed = false;
|
|
1989
2047
|
if (consecutiveFailures >= maxAttempts
|
|
1990
2048
|
|| (deadlineMs !== undefined && Date.now() - startedAt >= deadlineMs)) {
|
|
1991
2049
|
throw err;
|
package/src/collection/struct.ts
CHANGED
|
@@ -19,9 +19,13 @@ export interface SyncOptions {
|
|
|
19
19
|
* of the attempt count — a progress-agnostic ceiling. Unset means no deadline. */
|
|
20
20
|
deadlineMs?: number;
|
|
21
21
|
/** Base backoff delay in ms applied before the first retry; subsequent retries grow the delay
|
|
22
|
-
* exponentially up to {@link maxBackoffMs}. Default 100.
|
|
22
|
+
* exponentially up to {@link maxBackoffMs}. Default 100. Not applied once `SlotHoldAfterLosses` of the
|
|
23
|
+
* write's refused attempts carried a confirmed committed revision (`StaleFailure.staleAt`) and the
|
|
24
|
+
* latest refusal is one of them: a member then holds the next slot for this write,
|
|
25
|
+
* and the retry follows at once rather than spending the hold asleep. */
|
|
23
26
|
baseBackoffMs?: number;
|
|
24
|
-
/** Upper bound on any single backoff sleep, in ms. Default 5000.
|
|
27
|
+
/** Upper bound on any single backoff sleep, in ms. Default 5000. Like {@link baseBackoffMs}, not
|
|
28
|
+
* applied to a retry that follows at once because a member holds the slot for this write. */
|
|
25
29
|
maxBackoffMs?: number;
|
|
26
30
|
/** Optional abort signal. Checked at the top of each loop iteration and raced against the
|
|
27
31
|
* backoff sleep, so an aborted sync rejects promptly (with an AbortError) rather than finishing
|
package/src/index.ts
CHANGED
|
@@ -14,6 +14,7 @@ export * from "./transaction/index.js";
|
|
|
14
14
|
export * from "./transactor/index.js";
|
|
15
15
|
export * from "./transform/index.js";
|
|
16
16
|
export * from "./utility/canonical-json.js";
|
|
17
|
+
export * from "./utility/format-instant.js";
|
|
17
18
|
export * from "./utility/groupby.js";
|
|
18
19
|
export * from "./utility/hash-string.js";
|
|
19
20
|
export * from "./utility/latches.js";
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Matchmaking — arrival-push fan-out selection (db-core, pure).
|
|
3
|
+
*
|
|
4
|
+
* Decides which push-opted seekers a newly arrived provider is pushed to (`docs/matchmaking.md`
|
|
5
|
+
* §Fairness — FCFS by `attachedAt`, fan-out bounded by `capacityBudget`). Every cohort member runs the
|
|
6
|
+
* same selection over the same gossip-replicated records and then delivers only to the selected seekers
|
|
7
|
+
* whose slot primary it is, so the cohort as a whole notifies about `capacityBudget` seekers rather than
|
|
8
|
+
* that many per member.
|
|
9
|
+
*
|
|
10
|
+
* Pure: no clock, no I/O. The db-p2p cohort driver decodes the records, keeps the targets it is primary
|
|
11
|
+
* for, coalesces, signs and delivers.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { matchesFilter } from "./capability-filter.js";
|
|
15
|
+
import type { LocalProviderRegistration, LocalSeekerRegistration } from "./query-eval.js";
|
|
16
|
+
|
|
17
|
+
/** A decoded seeker registration held at this cohort, considered as a push target. */
|
|
18
|
+
export type ArrivalPushCandidate = LocalSeekerRegistration;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Seekers to notify about `provider`, longest-waiting first: of the push-opted seekers whose filter the
|
|
22
|
+
* provider matches and that attached no later than it did, the first `capacityBudget` by `attachedAt`
|
|
23
|
+
* (ties broken by `participantId`, so every member computes the same order). Generic so the caller gets
|
|
24
|
+
* back its own richer candidate objects.
|
|
25
|
+
*
|
|
26
|
+
* The `attachedAt` rule exists for a member that receives the whole record set at once (catching up by
|
|
27
|
+
* gossip, or pulling records in a rotation handoff): without it, that member would push every existing
|
|
28
|
+
* provider to every seeker. A provider that attached before a seeker was answerable by that seeker's own
|
|
29
|
+
* first query, and its safety poll still finds it.
|
|
30
|
+
*
|
|
31
|
+
* NOTE: accepted tradeoff — the ranking reads only replicated state, so a seeker already satisfied,
|
|
32
|
+
* finished, or answering `unknown_seeker` to its own primary keeps holding a fan-out slot until its
|
|
33
|
+
* record leaves the cohort (withdraw on finish, or its `seeker_ttl` expiry), under-filling that arrival's fan-out;
|
|
34
|
+
* members whose gossip views differ can also over- or under-fill by a few. Excluding them would mean
|
|
35
|
+
* gossiping per-seeker push state; the seeker's safety poll covers what is missed. Revisit if push
|
|
36
|
+
* under-fill shows up as seekers routinely falling back to the safety poll on a busy topic.
|
|
37
|
+
*/
|
|
38
|
+
export function selectArrivalPushTargets<S extends ArrivalPushCandidate>(
|
|
39
|
+
provider: LocalProviderRegistration,
|
|
40
|
+
seekers: readonly S[],
|
|
41
|
+
): S[] {
|
|
42
|
+
const budget = provider.payload.capacityBudget;
|
|
43
|
+
if (budget === 0) {
|
|
44
|
+
return [];
|
|
45
|
+
}
|
|
46
|
+
return seekers
|
|
47
|
+
.filter((s) => isPushTarget(provider, s))
|
|
48
|
+
.sort(byLongestWaiting)
|
|
49
|
+
.slice(0, budget);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** True iff `seeker` holds a push binding, matches `provider`, and was waiting when it arrived. */
|
|
53
|
+
function isPushTarget(provider: LocalProviderRegistration, seeker: ArrivalPushCandidate): boolean {
|
|
54
|
+
return seeker.payload.pushOnArrival === true
|
|
55
|
+
&& seeker.payload.correlationId !== undefined
|
|
56
|
+
&& matchesFilter(provider.payload, seeker.payload.filter)
|
|
57
|
+
&& provider.attachedAt >= seeker.attachedAt;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Ascending `attachedAt`, then code-unit order of `participantId` (locale-independent, so members agree). */
|
|
61
|
+
function byLongestWaiting(a: ArrivalPushCandidate, b: ArrivalPushCandidate): number {
|
|
62
|
+
if (a.attachedAt !== b.attachedAt) {
|
|
63
|
+
return a.attachedAt - b.attachedAt;
|
|
64
|
+
}
|
|
65
|
+
return a.participantId < b.participantId ? -1 : a.participantId > b.participantId ? 1 : 0;
|
|
66
|
+
}
|
|
@@ -10,8 +10,9 @@
|
|
|
10
10
|
* The TTL/limit constants are wire-stable and do **not** depend on simulator findings; the hang-out
|
|
11
11
|
* rows DO (folded via `fold-simulator-findings-into-design-docs`) — they carry the simulator-validated
|
|
12
12
|
* defaults (`contention_factor_cap = 4.0`, `mean_want_count = 3`, …; see `docs/matchmaking.md`
|
|
13
|
-
* §Configuration "Defaults validated by simulator").
|
|
14
|
-
*
|
|
13
|
+
* §Configuration "Defaults validated by simulator"). Of the arrival-push rows, `push_safety_poll_ms` is
|
|
14
|
+
* seeker-side and lives on {@link HangOutConfig}; `push_coalesce_ms` is the one cohort-side knob and
|
|
15
|
+
* lives on {@link ArrivalPushConfig}.
|
|
15
16
|
*/
|
|
16
17
|
|
|
17
18
|
import type { NodeProfile } from "../cohort-topic/tiers.js";
|
|
@@ -61,12 +62,26 @@ export interface HangOutConfig {
|
|
|
61
62
|
readonly contentionFactorCap: number;
|
|
62
63
|
/** Hang-out poll cadence on the non-push path. */
|
|
63
64
|
readonly requeryIntervalMs: number;
|
|
65
|
+
/** Sparse safety-poll cadence beneath the pushes, for a seeker that set `pushOnArrival`. */
|
|
66
|
+
readonly pushSafetyPollMs: number;
|
|
64
67
|
}
|
|
65
68
|
|
|
66
69
|
/** The default hang-out decision config (simulator-validated). */
|
|
67
70
|
export const DEFAULT_HANG_OUT_CONFIG: HangOutConfig = {
|
|
68
71
|
contentionFactorCap: CONTENTION_FACTOR_CAP,
|
|
69
72
|
requeryIntervalMs: REQUERY_INTERVAL_MS,
|
|
73
|
+
pushSafetyPollMs: PUSH_SAFETY_POLL_MS,
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/** Cohort-side arrival-push tuning (`docs/matchmaking.md` §Coalescing). */
|
|
77
|
+
export interface ArrivalPushConfig {
|
|
78
|
+
/** Window the pushing member batches fresh matchable arrivals per seeker before flushing one push. */
|
|
79
|
+
readonly coalesceMs: number;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The default arrival-push config. */
|
|
83
|
+
export const DEFAULT_ARRIVAL_PUSH_CONFIG: ArrivalPushConfig = {
|
|
84
|
+
coalesceMs: PUSH_COALESCE_MS,
|
|
70
85
|
};
|
|
71
86
|
|
|
72
87
|
/** The full matchmaking config, with the documented defaults. */
|
package/src/matchmaking/index.ts
CHANGED
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* See `docs/matchmaking.md`. The db-core layer owns the transport-agnostic pieces: wire codecs +
|
|
5
5
|
* seeker-side entry re-validation, the stable topic-anchor derivation, configuration, both
|
|
6
6
|
* registration roles' decision/state (provider attach/renew/self-throttle; seeker short-TTL
|
|
7
|
-
* registration), the cohort-side capability filter + pure query evaluation,
|
|
8
|
-
* hang-out-vs-continue decision engine. The db-p2p layer wires these to the cohort-topic substrate
|
|
7
|
+
* registration), the cohort-side capability filter + pure query evaluation, the arrival-push fan-out
|
|
8
|
+
* selection, and the seeker hang-out-vs-continue decision engine. The db-p2p layer wires these to the cohort-topic substrate
|
|
9
9
|
* (managers, the cohort query handler, the seeker walk client).
|
|
10
10
|
*/
|
|
11
11
|
|
|
@@ -16,6 +16,7 @@ export * from "./provider.js";
|
|
|
16
16
|
export * from "./seeker.js";
|
|
17
17
|
export * from "./capability-filter.js";
|
|
18
18
|
export * from "./query-eval.js";
|
|
19
|
+
export * from "./arrival-push.js";
|
|
19
20
|
export * from "./seeker-walk.js";
|
|
20
21
|
export * from "./multi-cohort-seeker.js";
|
|
21
22
|
export * from "./voting-quorum.js";
|
|
@@ -15,6 +15,8 @@
|
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
import { randomBytes } from "@noble/hashes/utils.js";
|
|
18
|
+
import { bytesToB64url } from "../cohort-topic/wire/codec.js";
|
|
19
|
+
import { CORRELATION_BYTES } from "../cohort-topic/wire/primitives.js";
|
|
18
20
|
import { seekerSigningPayload, type CapabilityFilter, type SeekerAppPayloadV1, encodeSeekerAppPayload } from "./wire.js";
|
|
19
21
|
|
|
20
22
|
/** Construction inputs for a {@link MatchmakingSeeker}. */
|
|
@@ -27,11 +29,14 @@ export interface MatchmakingSeekerOptions {
|
|
|
27
29
|
readonly contactHint: string;
|
|
28
30
|
/** Optional capability filter (evaluated cohort-side in the next ticket; carried here). */
|
|
29
31
|
readonly filter?: CapabilityFilter;
|
|
30
|
-
/** Opt into arrival pushes; default false (poll path).
|
|
32
|
+
/** Opt into arrival pushes; default false (poll path). */
|
|
31
33
|
readonly pushOnArrival?: boolean;
|
|
32
34
|
/** Sign the canonical registration image; resolves the base64url signature. */
|
|
33
35
|
readonly sign: (payload: Uint8Array) => Promise<string>;
|
|
34
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* 16-byte registration correlation id (not signature-bound); default fresh CSPRNG bytes. Sent in the
|
|
38
|
+
* payload when `pushOnArrival` is set, as the id arrival pushes are bound to.
|
|
39
|
+
*/
|
|
35
40
|
readonly correlationId?: Uint8Array;
|
|
36
41
|
/** CSPRNG source (injectable for deterministic tests). Default `@noble/hashes` `randomBytes`. */
|
|
37
42
|
readonly randomBytes?: (n: number) => Uint8Array;
|
|
@@ -58,7 +63,7 @@ export class MatchmakingSeeker {
|
|
|
58
63
|
this.pushOnArrival = options.pushOnArrival;
|
|
59
64
|
this.sign = options.sign;
|
|
60
65
|
const rand = options.randomBytes ?? randomBytes;
|
|
61
|
-
this.correlationId = options.correlationId ?? rand(
|
|
66
|
+
this.correlationId = options.correlationId ?? rand(CORRELATION_BYTES);
|
|
62
67
|
}
|
|
63
68
|
|
|
64
69
|
/** Build the signed {@link SeekerAppPayloadV1} for this seeker's registration. */
|
|
@@ -76,6 +81,9 @@ export class MatchmakingSeeker {
|
|
|
76
81
|
if (this.pushOnArrival !== undefined) {
|
|
77
82
|
payload.pushOnArrival = this.pushOnArrival;
|
|
78
83
|
}
|
|
84
|
+
if (this.pushOnArrival === true) {
|
|
85
|
+
payload.correlationId = bytesToB64url(this.correlationId);
|
|
86
|
+
}
|
|
79
87
|
return payload;
|
|
80
88
|
}
|
|
81
89
|
|