@optimystic/db-core 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +336 -336
- package/dist/src/cluster/structs.d.ts +39 -1
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js +24 -0
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/collection.d.ts +17 -0
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +24 -2
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +5 -0
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +7 -0
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/network/i-peer-network.d.ts +16 -0
- package/dist/src/network/i-peer-network.d.ts.map +1 -1
- package/dist/src/network/struct.d.ts +39 -2
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js +18 -0
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +95 -8
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +121 -8
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +1 -1
- package/dist/src/transaction/transaction.js +1 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +48 -13
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +25 -2
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/package.json +1 -1
- package/src/cluster/membership.ts +85 -85
- package/src/cluster/structs.ts +43 -4
- package/src/cohort-topic/addressing.ts +120 -120
- package/src/cohort-topic/antidos/bootstrap-evidence-envelope.ts +253 -253
- package/src/cohort-topic/antidos/bootstrap-evidence.ts +106 -106
- package/src/cohort-topic/antidos/index.ts +5 -5
- package/src/cohort-topic/antidos/rate-limiter.ts +210 -210
- package/src/cohort-topic/antidos/replay-guard.ts +146 -146
- package/src/cohort-topic/antidos/topic-budget.ts +160 -160
- package/src/cohort-topic/antiflood/index.ts +2 -2
- package/src/cohort-topic/antiflood/invariants.ts +108 -108
- package/src/cohort-topic/antiflood/jitter.ts +117 -117
- package/src/cohort-topic/coldstart.ts +237 -237
- package/src/cohort-topic/dmax.ts +88 -88
- package/src/cohort-topic/gossip/bus.ts +254 -254
- package/src/cohort-topic/gossip/index.ts +3 -3
- package/src/cohort-topic/gossip/records.ts +45 -45
- package/src/cohort-topic/gossip/view.ts +91 -91
- package/src/cohort-topic/index.ts +20 -20
- package/src/cohort-topic/load/barometer.ts +134 -134
- package/src/cohort-topic/load/index.ts +1 -1
- package/src/cohort-topic/member-engine.ts +430 -430
- package/src/cohort-topic/membership/index.ts +3 -3
- package/src/cohort-topic/membership/publisher.ts +163 -163
- package/src/cohort-topic/membership/source.ts +41 -41
- package/src/cohort-topic/membership/verifier.ts +461 -461
- package/src/cohort-topic/ports.ts +157 -157
- package/src/cohort-topic/promotion.ts +405 -405
- package/src/cohort-topic/registration/bytes.ts +37 -37
- package/src/cohort-topic/registration/handoff.ts +154 -154
- package/src/cohort-topic/registration/index.ts +6 -6
- package/src/cohort-topic/registration/renewal.ts +495 -495
- package/src/cohort-topic/registration/sharding.ts +61 -61
- package/src/cohort-topic/registration/store.ts +81 -81
- package/src/cohort-topic/registration/types.ts +91 -91
- package/src/cohort-topic/ring-hash.ts +50 -50
- package/src/cohort-topic/service.ts +416 -416
- package/src/cohort-topic/sig/index.ts +2 -2
- package/src/cohort-topic/sig/payloads.ts +59 -59
- package/src/cohort-topic/sig/threshold.ts +64 -64
- package/src/cohort-topic/tiers.ts +74 -74
- package/src/cohort-topic/traffic.ts +233 -233
- package/src/cohort-topic/walk.ts +326 -326
- package/src/cohort-topic/willingness.ts +237 -237
- package/src/cohort-topic/wire/codec.ts +216 -216
- package/src/cohort-topic/wire/index.ts +18 -18
- package/src/cohort-topic/wire/payloads.ts +126 -126
- package/src/cohort-topic/wire/primitives.ts +188 -188
- package/src/cohort-topic/wire/types.ts +475 -475
- package/src/cohort-topic/wire/validate.ts +512 -512
- package/src/collection/collection-type-registry.ts +37 -37
- package/src/collection/collection.ts +25 -2
- package/src/collections/diary/diary.ts +68 -68
- package/src/collections/tree/readme.md +4 -0
- package/src/collections/tree/tree.ts +320 -312
- package/src/matchmaking/capability-filter.ts +45 -45
- package/src/matchmaking/config.ts +98 -98
- package/src/matchmaking/index.ts +21 -21
- package/src/matchmaking/multi-cohort-seeker.ts +234 -234
- package/src/matchmaking/provider.ts +123 -123
- package/src/matchmaking/query-eval.ts +105 -105
- package/src/matchmaking/seeker-walk.ts +127 -127
- package/src/matchmaking/seeker.ts +86 -86
- package/src/matchmaking/topic-anchor.ts +90 -90
- package/src/matchmaking/voting-quorum.ts +394 -394
- package/src/matchmaking/wire.ts +603 -603
- package/src/network/i-peer-network.ts +17 -0
- package/src/network/stale-failure.ts +43 -43
- package/src/network/struct.ts +41 -2
- package/src/network/types.ts +37 -37
- package/src/reactivity/backfill.ts +220 -220
- package/src/reactivity/backpressure.ts +191 -191
- package/src/reactivity/checkpoint.ts +308 -308
- package/src/reactivity/config.ts +172 -172
- package/src/reactivity/dedupe.ts +132 -132
- package/src/reactivity/forwarder.ts +87 -87
- package/src/reactivity/index.ts +34 -34
- package/src/reactivity/notification.ts +123 -123
- package/src/reactivity/policy.ts +79 -79
- package/src/reactivity/push-state.ts +310 -310
- package/src/reactivity/recover.ts +153 -153
- package/src/reactivity/replay-buffer.ts +141 -141
- package/src/reactivity/resume.ts +549 -549
- package/src/reactivity/rotation.ts +415 -415
- package/src/reactivity/subscriber.ts +132 -132
- package/src/reactivity/subscription.ts +66 -66
- package/src/reactivity/topic-anchor.ts +71 -71
- package/src/reactivity/verify.ts +73 -73
- package/src/reactivity/wire-validate.ts +13 -13
- package/src/reactivity/wire.ts +224 -224
- package/src/testing/async-wait.ts +65 -65
- package/src/testing/index.ts +2 -2
- package/src/testing/test-transactor.ts +638 -502
- package/src/transaction/errors.ts +91 -91
- package/src/transaction/operations-hash.ts +196 -196
- package/src/transaction/read-dependency-collector.ts +78 -78
- package/src/transaction/transaction.ts +1 -1
- package/src/transactor/change-notifier.ts +80 -80
- package/src/transactor/index.ts +5 -5
- package/src/transactor/network-transactor.ts +49 -14
- package/src/transactor/transactor-source.ts +25 -2
- package/src/transform/atomic-proxy.ts +92 -92
- package/src/transform/helpers.ts +159 -159
- package/src/utility/backoff.ts +95 -95
- package/src/utility/batch-coordinator.ts +191 -191
- package/dist/src/transaction/context.d.ts +0 -60
- package/dist/src/transaction/context.d.ts.map +0 -1
- package/dist/src/transaction/context.js +0 -91
- package/dist/src/transaction/context.js.map +0 -1
|
@@ -1,234 +1,234 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Matchmaking — multi-cohort sweep (db-core, pure orchestration of the hot-topic representative sample).
|
|
3
|
-
*
|
|
4
|
-
* Per `docs/matchmaking.md` §Multi-cohort sweep. When a topic is hot enough that its providers live
|
|
5
|
-
* across many tier-`d >= 1` cohorts, a seeker that wants a *representative* cross-ring sample (rather
|
|
6
|
-
* than the prefix-biased single-cohort slice) does:
|
|
7
|
-
*
|
|
8
|
-
* 1. Registers at its natural tier as usual (the single-cohort {@link import("./seeker-walk.js").decide}
|
|
9
|
-
* walk — not this module).
|
|
10
|
-
* 2. Queries the **root** cohort, which returns an {@link AggregateCountV1}: log-bucketed provider counts
|
|
11
|
-
* per tier-1 prefix shard, **threshold-signed**. A cold root that fell through to `NoState` produces
|
|
12
|
-
* no aggregate (the producer gates on tree depth — see `db-p2p/matchmaking/aggregate-counts.ts`).
|
|
13
|
-
* 3. Selects the high-population tier-1 shards ({@link selectShards}) and queries them directly, unioning
|
|
14
|
-
* the returned providers into a deduped, re-validated set.
|
|
15
|
-
*
|
|
16
|
-
* This module is **pure**: the root-aggregate fetch, the per-shard query, and the optional threshold-sig
|
|
17
|
-
* verification are injected as a {@link MultiCohortSweepPorts} port (db-p2p binds them to the matchmaking
|
|
18
|
-
* query RPCs), exactly like the single-cohort walk splits pure `decide` (here) from the db-p2p walk
|
|
19
|
-
* client. The advisory trust model is preserved end-to-end: every shard entry is re-validated with
|
|
20
|
-
* {@link verifyProviderEntry} before it counts, so a lying shard primary buys nothing.
|
|
21
|
-
*
|
|
22
|
-
* The sweep costs more RPCs than the single-cohort sample and is reserved for representativeness-over-
|
|
23
|
-
* latency use cases (voting quorums, capability fairness audits); db-p2p binds it to the voting
|
|
24
|
-
* `QuorumDiscovery.sweep` port.
|
|
25
|
-
*/
|
|
26
|
-
|
|
27
|
-
import { matchesFilter } from "./capability-filter.js";
|
|
28
|
-
import { verifyProviderEntry, type AggregateCountV1, type CapabilityFilter, type EntrySigVerifier, type ProviderEntryV1 } from "./wire.js";
|
|
29
|
-
|
|
30
|
-
/** The tier whose prefix shards the sweep ranges over (`docs/matchmaking.md` §Wire formats — typically 1). */
|
|
31
|
-
export const DEFAULT_SWEEP_TARGET_TIER = 1;
|
|
32
|
-
/** Fan-out ceiling: never query more than this many shards in one sweep (bounds RPC cost). */
|
|
33
|
-
export const DEFAULT_SWEEP_MAX_SHARDS = 16;
|
|
34
|
-
/**
|
|
35
|
-
* Multiplier on `wantCount` when accumulating bucketed shard populations. `1` because {@link logBucketCount}
|
|
36
|
-
* already rounds counts *down* — the true population is `>=` the reported sum, so the selection already
|
|
37
|
-
* over-provisions without an extra factor.
|
|
38
|
-
*/
|
|
39
|
-
export const DEFAULT_SWEEP_OVERPROVISION = 1;
|
|
40
|
-
|
|
41
|
-
/** One tier-1 prefix shard the sweep elected to query, with its (bucketed) reported population. */
|
|
42
|
-
export interface ShardSelection {
|
|
43
|
-
/** Prefix slot `0..F-1` identifying the tier-1 cohort. */
|
|
44
|
-
readonly prefixSlot: number;
|
|
45
|
-
/** The tier whose shard this is (typically 1). */
|
|
46
|
-
readonly targetTier: number;
|
|
47
|
-
/** The shard's log-bucketed reported provider count (rounds down — see {@link logBucketCount}). */
|
|
48
|
-
readonly bucketedCount: number;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
/** Inputs to {@link selectShards}. */
|
|
52
|
-
export interface SelectShardsOptions {
|
|
53
|
-
/** Providers the seeker needs (drives how many shards are unioned). */
|
|
54
|
-
readonly wantCount: number;
|
|
55
|
-
/** Which `targetTier` buckets to consider. Default {@link DEFAULT_SWEEP_TARGET_TIER}. */
|
|
56
|
-
readonly targetTier?: number;
|
|
57
|
-
/** Fan-out ceiling. Default {@link DEFAULT_SWEEP_MAX_SHARDS}. */
|
|
58
|
-
readonly maxShards?: number;
|
|
59
|
-
/** Multiplier on `wantCount`. Default {@link DEFAULT_SWEEP_OVERPROVISION}. */
|
|
60
|
-
readonly overprovision?: number;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/**
|
|
64
|
-
* Choose which tier-1 shards to query from an {@link AggregateCountV1}: the highest-population shards
|
|
65
|
-
* first (ties broken by ascending `prefixSlot` for determinism), accumulating bucketed counts until they
|
|
66
|
-
* cover `wantCount * overprovision`, capped at `maxShards`. Empty shards (`count === 0`) are skipped.
|
|
67
|
-
* Pure and deterministic.
|
|
68
|
-
*/
|
|
69
|
-
export function selectShards(aggregate: AggregateCountV1, opts: SelectShardsOptions): ShardSelection[] {
|
|
70
|
-
const targetTier = opts.targetTier ?? DEFAULT_SWEEP_TARGET_TIER;
|
|
71
|
-
const maxShards = opts.maxShards ?? DEFAULT_SWEEP_MAX_SHARDS;
|
|
72
|
-
const overprovision = opts.overprovision ?? DEFAULT_SWEEP_OVERPROVISION;
|
|
73
|
-
const need = Math.max(1, Math.ceil(opts.wantCount * overprovision));
|
|
74
|
-
|
|
75
|
-
const ranked = aggregate.bucketCounts
|
|
76
|
-
.filter((b) => b.targetTier === targetTier && b.count > 0)
|
|
77
|
-
.sort((a, b) => b.count - a.count || a.prefixSlot - b.prefixSlot);
|
|
78
|
-
|
|
79
|
-
const selected: ShardSelection[] = [];
|
|
80
|
-
let cumulative = 0;
|
|
81
|
-
for (const bucket of ranked) {
|
|
82
|
-
if (selected.length >= maxShards) {
|
|
83
|
-
break;
|
|
84
|
-
}
|
|
85
|
-
selected.push({ prefixSlot: bucket.prefixSlot, targetTier: bucket.targetTier, bucketedCount: bucket.count });
|
|
86
|
-
cumulative += bucket.count;
|
|
87
|
-
if (cumulative >= need) {
|
|
88
|
-
break;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
return selected;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/** Identifies one tier shard to query directly (db-p2p resolves it to `coord_d` and dials the cohort). */
|
|
95
|
-
export interface SweepShardQuery {
|
|
96
|
-
readonly prefixSlot: number;
|
|
97
|
-
readonly targetTier: number;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* The transport seam the sweep drives, injected by db-p2p. `fetchAggregate` queries the root cohort
|
|
102
|
-
* (resolving `undefined` when the root is cold / unpromoted and returns no {@link AggregateCountV1});
|
|
103
|
-
* `queryShard` queries one elected tier-1 cohort; `verifyAggregate` (optional) threshold-verifies the
|
|
104
|
-
* aggregate before its counts are trusted.
|
|
105
|
-
*/
|
|
106
|
-
export interface MultiCohortSweepPorts {
|
|
107
|
-
/** Query the root cohort for the aggregate; `undefined` when the root produced none (cold / unpromoted). */
|
|
108
|
-
fetchAggregate(): Promise<AggregateCountV1 | undefined>;
|
|
109
|
-
/** Threshold-verify the aggregate (db-p2p binds the cohort crypto). Omitted → trusted unconditionally. */
|
|
110
|
-
verifyAggregate?(aggregate: AggregateCountV1): boolean;
|
|
111
|
-
/** Query one elected shard; returns its advisory provider entries (the sweep re-validates each). */
|
|
112
|
-
queryShard(shard: SweepShardQuery): Promise<readonly ProviderEntryV1[]>;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/** Inputs to {@link runMultiCohortSweep}. */
|
|
116
|
-
export interface MultiCohortSweepOptions {
|
|
117
|
-
/** The matchmaking topic id (used to re-validate each forwarded entry's `registrationSig`). */
|
|
118
|
-
readonly topicId: Uint8Array;
|
|
119
|
-
/** Providers the seeker needs (drives shard selection). */
|
|
120
|
-
readonly wantCount: number;
|
|
121
|
-
/** Per-entry signature verifier (db-p2p binds `verifyPeerSig`). */
|
|
122
|
-
readonly verifyEntry: EntrySigVerifier;
|
|
123
|
-
/** Optional capability filter, re-applied over every shard's returned set. */
|
|
124
|
-
readonly filter?: CapabilityFilter;
|
|
125
|
-
/** Which `targetTier` shards to range over. Default {@link DEFAULT_SWEEP_TARGET_TIER}. */
|
|
126
|
-
readonly targetTier?: number;
|
|
127
|
-
/** Fan-out ceiling. Default {@link DEFAULT_SWEEP_MAX_SHARDS}. */
|
|
128
|
-
readonly maxShards?: number;
|
|
129
|
-
/** Multiplier on `wantCount`. Default {@link DEFAULT_SWEEP_OVERPROVISION}. */
|
|
130
|
-
readonly overprovision?: number;
|
|
131
|
-
/**
|
|
132
|
-
* Patience budget (ms) for the whole sweep; drains across the shard fan-out. Omitted ⇒ unbounded
|
|
133
|
-
* (query every selected shard, today's behaviour). Mirrors the walk leg's budget.
|
|
134
|
-
*/
|
|
135
|
-
readonly patienceMs?: number;
|
|
136
|
-
/** Wall clock (unix ms); injectable for tests. Default `Date.now`. Only consulted when `patienceMs` set. */
|
|
137
|
-
readonly clock?: () => number;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/** The assembled result of a multi-cohort sweep. */
|
|
141
|
-
export interface MultiCohortSweepResult {
|
|
142
|
-
/** The unioned, filtered, `registrationSig`-re-validated providers, deduped by `participantId`. */
|
|
143
|
-
readonly providers: ProviderEntryV1[];
|
|
144
|
-
/** The shards selected from the aggregate (empty when no aggregate / untrusted). */
|
|
145
|
-
readonly selectedShards: ShardSelection[];
|
|
146
|
-
/** How many shards were actually queried (== `selectedShards.length` on success). */
|
|
147
|
-
readonly shardsQueried: number;
|
|
148
|
-
/** Whether the root produced an aggregate at all (`false` for a cold / unpromoted root). */
|
|
149
|
-
readonly aggregateAvailable: boolean;
|
|
150
|
-
/** Whether the aggregate's threshold signature verified (`false` when absent or invalid). */
|
|
151
|
-
readonly aggregateTrusted: boolean;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/** An empty result (no aggregate, or an aggregate that failed verification). */
|
|
155
|
-
function emptyResult(aggregateAvailable: boolean, aggregateTrusted: boolean): MultiCohortSweepResult {
|
|
156
|
-
return { providers: [], selectedShards: [], shardsQueried: 0, aggregateAvailable, aggregateTrusted };
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/**
|
|
160
|
-
* Run the multi-cohort sweep (`docs/matchmaking.md` §Multi-cohort sweep): fetch the root aggregate,
|
|
161
|
-
* threshold-verify it, select high-population shards, query each, and union the deduped + re-validated
|
|
162
|
-
* providers. A cold root (no aggregate) or an aggregate that fails verification yields an empty set so
|
|
163
|
-
* the caller falls back to the single-cohort sample. Each shard entry is filtered and
|
|
164
|
-
* `registrationSig`-re-validated before it counts — the cohort vouches only for "the set I held".
|
|
165
|
-
*
|
|
166
|
-
* When `opts.patienceMs` is supplied, a wall-clock deadline is fixed at entry and the shard fan-out
|
|
167
|
-
* stops as soon as the budget drains — mirroring the walk leg's patience model. This is
|
|
168
|
-
* "stop starting new shard queries": an in-flight `queryShard` call is not cancelled mid-flight.
|
|
169
|
-
* When `opts.patienceMs` is absent, every elected shard is queried (today's behaviour).
|
|
170
|
-
*/
|
|
171
|
-
export async function runMultiCohortSweep(ports: MultiCohortSweepPorts, opts: MultiCohortSweepOptions): Promise<MultiCohortSweepResult> {
|
|
172
|
-
// Budget: optional wall-clock deadline mirroring the walk leg. Only consulted when patienceMs is set.
|
|
173
|
-
const clock = opts.clock ?? ((): number => Date.now());
|
|
174
|
-
const deadline = opts.patienceMs !== undefined ? clock() + opts.patienceMs : undefined;
|
|
175
|
-
const remaining = (): number => (deadline !== undefined ? Math.max(0, deadline - clock()) : Infinity);
|
|
176
|
-
// If the budget is already drained on entry, skip even the aggregate RPC — the walk already consumed
|
|
177
|
-
// all patience and there is no point spending an RPC on an aggregate we have no time to act on.
|
|
178
|
-
if (deadline !== undefined && remaining() <= 0) {
|
|
179
|
-
return emptyResult(false, false);
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
const aggregate = await ports.fetchAggregate();
|
|
183
|
-
if (aggregate === undefined) {
|
|
184
|
-
return emptyResult(false, false);
|
|
185
|
-
}
|
|
186
|
-
// `aggregateTrusted` is asserted only when a verifier actually ran and passed. With no verifier injected
|
|
187
|
-
// the sweep still proceeds — every shard entry is `registrationSig`-re-validated below, so a forged
|
|
188
|
-
// aggregate can at worst mis-steer shard selection (wasted RPCs / a thinner sample), never inject
|
|
189
|
-
// providers — but the flag stays honest rather than claiming a trust that was never established.
|
|
190
|
-
const aggregateTrusted = ports.verifyAggregate !== undefined;
|
|
191
|
-
if (ports.verifyAggregate !== undefined && !ports.verifyAggregate(aggregate)) {
|
|
192
|
-
return emptyResult(true, false);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const selectShardOpts: SelectShardsOptions = { wantCount: opts.wantCount };
|
|
196
|
-
if (opts.targetTier !== undefined) {
|
|
197
|
-
(selectShardOpts as { targetTier: number }).targetTier = opts.targetTier;
|
|
198
|
-
}
|
|
199
|
-
if (opts.maxShards !== undefined) {
|
|
200
|
-
(selectShardOpts as { maxShards: number }).maxShards = opts.maxShards;
|
|
201
|
-
}
|
|
202
|
-
if (opts.overprovision !== undefined) {
|
|
203
|
-
(selectShardOpts as { overprovision: number }).overprovision = opts.overprovision;
|
|
204
|
-
}
|
|
205
|
-
const selected = selectShards(aggregate, selectShardOpts);
|
|
206
|
-
|
|
207
|
-
const matched = new Map<string, ProviderEntryV1>();
|
|
208
|
-
let shardsQueried = 0;
|
|
209
|
-
for (const shard of selected) {
|
|
210
|
-
// Stop starting new shard queries once the budget has drained.
|
|
211
|
-
if (remaining() <= 0) {
|
|
212
|
-
break;
|
|
213
|
-
}
|
|
214
|
-
const entries = await ports.queryShard({ prefixSlot: shard.prefixSlot, targetTier: shard.targetTier });
|
|
215
|
-
shardsQueried++;
|
|
216
|
-
for (const entry of entries) {
|
|
217
|
-
if (!matchesFilter(entry, opts.filter)) {
|
|
218
|
-
continue;
|
|
219
|
-
}
|
|
220
|
-
if (!verifyProviderEntry(opts.topicId, entry, opts.verifyEntry)) {
|
|
221
|
-
continue;
|
|
222
|
-
}
|
|
223
|
-
matched.set(entry.participantId, entry);
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
return {
|
|
228
|
-
providers: [...matched.values()],
|
|
229
|
-
selectedShards: selected,
|
|
230
|
-
shardsQueried,
|
|
231
|
-
aggregateAvailable: true,
|
|
232
|
-
aggregateTrusted,
|
|
233
|
-
};
|
|
234
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Matchmaking — multi-cohort sweep (db-core, pure orchestration of the hot-topic representative sample).
|
|
3
|
+
*
|
|
4
|
+
* Per `docs/matchmaking.md` §Multi-cohort sweep. When a topic is hot enough that its providers live
|
|
5
|
+
* across many tier-`d >= 1` cohorts, a seeker that wants a *representative* cross-ring sample (rather
|
|
6
|
+
* than the prefix-biased single-cohort slice) does:
|
|
7
|
+
*
|
|
8
|
+
* 1. Registers at its natural tier as usual (the single-cohort {@link import("./seeker-walk.js").decide}
|
|
9
|
+
* walk — not this module).
|
|
10
|
+
* 2. Queries the **root** cohort, which returns an {@link AggregateCountV1}: log-bucketed provider counts
|
|
11
|
+
* per tier-1 prefix shard, **threshold-signed**. A cold root that fell through to `NoState` produces
|
|
12
|
+
* no aggregate (the producer gates on tree depth — see `db-p2p/matchmaking/aggregate-counts.ts`).
|
|
13
|
+
* 3. Selects the high-population tier-1 shards ({@link selectShards}) and queries them directly, unioning
|
|
14
|
+
* the returned providers into a deduped, re-validated set.
|
|
15
|
+
*
|
|
16
|
+
* This module is **pure**: the root-aggregate fetch, the per-shard query, and the optional threshold-sig
|
|
17
|
+
* verification are injected as a {@link MultiCohortSweepPorts} port (db-p2p binds them to the matchmaking
|
|
18
|
+
* query RPCs), exactly like the single-cohort walk splits pure `decide` (here) from the db-p2p walk
|
|
19
|
+
* client. The advisory trust model is preserved end-to-end: every shard entry is re-validated with
|
|
20
|
+
* {@link verifyProviderEntry} before it counts, so a lying shard primary buys nothing.
|
|
21
|
+
*
|
|
22
|
+
* The sweep costs more RPCs than the single-cohort sample and is reserved for representativeness-over-
|
|
23
|
+
* latency use cases (voting quorums, capability fairness audits); db-p2p binds it to the voting
|
|
24
|
+
* `QuorumDiscovery.sweep` port.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { matchesFilter } from "./capability-filter.js";
|
|
28
|
+
import { verifyProviderEntry, type AggregateCountV1, type CapabilityFilter, type EntrySigVerifier, type ProviderEntryV1 } from "./wire.js";
|
|
29
|
+
|
|
30
|
+
/** The tier whose prefix shards the sweep ranges over (`docs/matchmaking.md` §Wire formats — typically 1). */
|
|
31
|
+
export const DEFAULT_SWEEP_TARGET_TIER = 1;
|
|
32
|
+
/** Fan-out ceiling: never query more than this many shards in one sweep (bounds RPC cost). */
|
|
33
|
+
export const DEFAULT_SWEEP_MAX_SHARDS = 16;
|
|
34
|
+
/**
|
|
35
|
+
* Multiplier on `wantCount` when accumulating bucketed shard populations. `1` because {@link logBucketCount}
|
|
36
|
+
* already rounds counts *down* — the true population is `>=` the reported sum, so the selection already
|
|
37
|
+
* over-provisions without an extra factor.
|
|
38
|
+
*/
|
|
39
|
+
export const DEFAULT_SWEEP_OVERPROVISION = 1;
|
|
40
|
+
|
|
41
|
+
/** One tier-1 prefix shard the sweep elected to query, with its (bucketed) reported population. */
|
|
42
|
+
export interface ShardSelection {
|
|
43
|
+
/** Prefix slot `0..F-1` identifying the tier-1 cohort. */
|
|
44
|
+
readonly prefixSlot: number;
|
|
45
|
+
/** The tier whose shard this is (typically 1). */
|
|
46
|
+
readonly targetTier: number;
|
|
47
|
+
/** The shard's log-bucketed reported provider count (rounds down — see {@link logBucketCount}). */
|
|
48
|
+
readonly bucketedCount: number;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Inputs to {@link selectShards}. */
|
|
52
|
+
export interface SelectShardsOptions {
|
|
53
|
+
/** Providers the seeker needs (drives how many shards are unioned). */
|
|
54
|
+
readonly wantCount: number;
|
|
55
|
+
/** Which `targetTier` buckets to consider. Default {@link DEFAULT_SWEEP_TARGET_TIER}. */
|
|
56
|
+
readonly targetTier?: number;
|
|
57
|
+
/** Fan-out ceiling. Default {@link DEFAULT_SWEEP_MAX_SHARDS}. */
|
|
58
|
+
readonly maxShards?: number;
|
|
59
|
+
/** Multiplier on `wantCount`. Default {@link DEFAULT_SWEEP_OVERPROVISION}. */
|
|
60
|
+
readonly overprovision?: number;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Choose which tier-1 shards to query from an {@link AggregateCountV1}: the highest-population shards
|
|
65
|
+
* first (ties broken by ascending `prefixSlot` for determinism), accumulating bucketed counts until they
|
|
66
|
+
* cover `wantCount * overprovision`, capped at `maxShards`. Empty shards (`count === 0`) are skipped.
|
|
67
|
+
* Pure and deterministic.
|
|
68
|
+
*/
|
|
69
|
+
export function selectShards(aggregate: AggregateCountV1, opts: SelectShardsOptions): ShardSelection[] {
|
|
70
|
+
const targetTier = opts.targetTier ?? DEFAULT_SWEEP_TARGET_TIER;
|
|
71
|
+
const maxShards = opts.maxShards ?? DEFAULT_SWEEP_MAX_SHARDS;
|
|
72
|
+
const overprovision = opts.overprovision ?? DEFAULT_SWEEP_OVERPROVISION;
|
|
73
|
+
const need = Math.max(1, Math.ceil(opts.wantCount * overprovision));
|
|
74
|
+
|
|
75
|
+
const ranked = aggregate.bucketCounts
|
|
76
|
+
.filter((b) => b.targetTier === targetTier && b.count > 0)
|
|
77
|
+
.sort((a, b) => b.count - a.count || a.prefixSlot - b.prefixSlot);
|
|
78
|
+
|
|
79
|
+
const selected: ShardSelection[] = [];
|
|
80
|
+
let cumulative = 0;
|
|
81
|
+
for (const bucket of ranked) {
|
|
82
|
+
if (selected.length >= maxShards) {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
selected.push({ prefixSlot: bucket.prefixSlot, targetTier: bucket.targetTier, bucketedCount: bucket.count });
|
|
86
|
+
cumulative += bucket.count;
|
|
87
|
+
if (cumulative >= need) {
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return selected;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Identifies one tier shard to query directly (db-p2p resolves it to `coord_d` and dials the cohort). */
|
|
95
|
+
export interface SweepShardQuery {
|
|
96
|
+
readonly prefixSlot: number;
|
|
97
|
+
readonly targetTier: number;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* The transport seam the sweep drives, injected by db-p2p. `fetchAggregate` queries the root cohort
|
|
102
|
+
* (resolving `undefined` when the root is cold / unpromoted and returns no {@link AggregateCountV1});
|
|
103
|
+
* `queryShard` queries one elected tier-1 cohort; `verifyAggregate` (optional) threshold-verifies the
|
|
104
|
+
* aggregate before its counts are trusted.
|
|
105
|
+
*/
|
|
106
|
+
export interface MultiCohortSweepPorts {
|
|
107
|
+
/** Query the root cohort for the aggregate; `undefined` when the root produced none (cold / unpromoted). */
|
|
108
|
+
fetchAggregate(): Promise<AggregateCountV1 | undefined>;
|
|
109
|
+
/** Threshold-verify the aggregate (db-p2p binds the cohort crypto). Omitted → trusted unconditionally. */
|
|
110
|
+
verifyAggregate?(aggregate: AggregateCountV1): boolean;
|
|
111
|
+
/** Query one elected shard; returns its advisory provider entries (the sweep re-validates each). */
|
|
112
|
+
queryShard(shard: SweepShardQuery): Promise<readonly ProviderEntryV1[]>;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Inputs to {@link runMultiCohortSweep}. */
|
|
116
|
+
export interface MultiCohortSweepOptions {
|
|
117
|
+
/** The matchmaking topic id (used to re-validate each forwarded entry's `registrationSig`). */
|
|
118
|
+
readonly topicId: Uint8Array;
|
|
119
|
+
/** Providers the seeker needs (drives shard selection). */
|
|
120
|
+
readonly wantCount: number;
|
|
121
|
+
/** Per-entry signature verifier (db-p2p binds `verifyPeerSig`). */
|
|
122
|
+
readonly verifyEntry: EntrySigVerifier;
|
|
123
|
+
/** Optional capability filter, re-applied over every shard's returned set. */
|
|
124
|
+
readonly filter?: CapabilityFilter;
|
|
125
|
+
/** Which `targetTier` shards to range over. Default {@link DEFAULT_SWEEP_TARGET_TIER}. */
|
|
126
|
+
readonly targetTier?: number;
|
|
127
|
+
/** Fan-out ceiling. Default {@link DEFAULT_SWEEP_MAX_SHARDS}. */
|
|
128
|
+
readonly maxShards?: number;
|
|
129
|
+
/** Multiplier on `wantCount`. Default {@link DEFAULT_SWEEP_OVERPROVISION}. */
|
|
130
|
+
readonly overprovision?: number;
|
|
131
|
+
/**
|
|
132
|
+
* Patience budget (ms) for the whole sweep; drains across the shard fan-out. Omitted ⇒ unbounded
|
|
133
|
+
* (query every selected shard, today's behaviour). Mirrors the walk leg's budget.
|
|
134
|
+
*/
|
|
135
|
+
readonly patienceMs?: number;
|
|
136
|
+
/** Wall clock (unix ms); injectable for tests. Default `Date.now`. Only consulted when `patienceMs` set. */
|
|
137
|
+
readonly clock?: () => number;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The assembled result of a multi-cohort sweep. */
|
|
141
|
+
export interface MultiCohortSweepResult {
|
|
142
|
+
/** The unioned, filtered, `registrationSig`-re-validated providers, deduped by `participantId`. */
|
|
143
|
+
readonly providers: ProviderEntryV1[];
|
|
144
|
+
/** The shards selected from the aggregate (empty when no aggregate / untrusted). */
|
|
145
|
+
readonly selectedShards: ShardSelection[];
|
|
146
|
+
/** How many shards were actually queried (== `selectedShards.length` on success). */
|
|
147
|
+
readonly shardsQueried: number;
|
|
148
|
+
/** Whether the root produced an aggregate at all (`false` for a cold / unpromoted root). */
|
|
149
|
+
readonly aggregateAvailable: boolean;
|
|
150
|
+
/** Whether the aggregate's threshold signature verified (`false` when absent or invalid). */
|
|
151
|
+
readonly aggregateTrusted: boolean;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** An empty result (no aggregate, or an aggregate that failed verification). */
|
|
155
|
+
function emptyResult(aggregateAvailable: boolean, aggregateTrusted: boolean): MultiCohortSweepResult {
|
|
156
|
+
return { providers: [], selectedShards: [], shardsQueried: 0, aggregateAvailable, aggregateTrusted };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Run the multi-cohort sweep (`docs/matchmaking.md` §Multi-cohort sweep): fetch the root aggregate,
|
|
161
|
+
* threshold-verify it, select high-population shards, query each, and union the deduped + re-validated
|
|
162
|
+
* providers. A cold root (no aggregate) or an aggregate that fails verification yields an empty set so
|
|
163
|
+
* the caller falls back to the single-cohort sample. Each shard entry is filtered and
|
|
164
|
+
* `registrationSig`-re-validated before it counts — the cohort vouches only for "the set I held".
|
|
165
|
+
*
|
|
166
|
+
* When `opts.patienceMs` is supplied, a wall-clock deadline is fixed at entry and the shard fan-out
|
|
167
|
+
* stops as soon as the budget drains — mirroring the walk leg's patience model. This is
|
|
168
|
+
* "stop starting new shard queries": an in-flight `queryShard` call is not cancelled mid-flight.
|
|
169
|
+
* When `opts.patienceMs` is absent, every elected shard is queried (today's behaviour).
|
|
170
|
+
*/
|
|
171
|
+
export async function runMultiCohortSweep(ports: MultiCohortSweepPorts, opts: MultiCohortSweepOptions): Promise<MultiCohortSweepResult> {
|
|
172
|
+
// Budget: optional wall-clock deadline mirroring the walk leg. Only consulted when patienceMs is set.
|
|
173
|
+
const clock = opts.clock ?? ((): number => Date.now());
|
|
174
|
+
const deadline = opts.patienceMs !== undefined ? clock() + opts.patienceMs : undefined;
|
|
175
|
+
const remaining = (): number => (deadline !== undefined ? Math.max(0, deadline - clock()) : Infinity);
|
|
176
|
+
// If the budget is already drained on entry, skip even the aggregate RPC — the walk already consumed
|
|
177
|
+
// all patience and there is no point spending an RPC on an aggregate we have no time to act on.
|
|
178
|
+
if (deadline !== undefined && remaining() <= 0) {
|
|
179
|
+
return emptyResult(false, false);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const aggregate = await ports.fetchAggregate();
|
|
183
|
+
if (aggregate === undefined) {
|
|
184
|
+
return emptyResult(false, false);
|
|
185
|
+
}
|
|
186
|
+
// `aggregateTrusted` is asserted only when a verifier actually ran and passed. With no verifier injected
|
|
187
|
+
// the sweep still proceeds — every shard entry is `registrationSig`-re-validated below, so a forged
|
|
188
|
+
// aggregate can at worst mis-steer shard selection (wasted RPCs / a thinner sample), never inject
|
|
189
|
+
// providers — but the flag stays honest rather than claiming a trust that was never established.
|
|
190
|
+
const aggregateTrusted = ports.verifyAggregate !== undefined;
|
|
191
|
+
if (ports.verifyAggregate !== undefined && !ports.verifyAggregate(aggregate)) {
|
|
192
|
+
return emptyResult(true, false);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const selectShardOpts: SelectShardsOptions = { wantCount: opts.wantCount };
|
|
196
|
+
if (opts.targetTier !== undefined) {
|
|
197
|
+
(selectShardOpts as { targetTier: number }).targetTier = opts.targetTier;
|
|
198
|
+
}
|
|
199
|
+
if (opts.maxShards !== undefined) {
|
|
200
|
+
(selectShardOpts as { maxShards: number }).maxShards = opts.maxShards;
|
|
201
|
+
}
|
|
202
|
+
if (opts.overprovision !== undefined) {
|
|
203
|
+
(selectShardOpts as { overprovision: number }).overprovision = opts.overprovision;
|
|
204
|
+
}
|
|
205
|
+
const selected = selectShards(aggregate, selectShardOpts);
|
|
206
|
+
|
|
207
|
+
const matched = new Map<string, ProviderEntryV1>();
|
|
208
|
+
let shardsQueried = 0;
|
|
209
|
+
for (const shard of selected) {
|
|
210
|
+
// Stop starting new shard queries once the budget has drained.
|
|
211
|
+
if (remaining() <= 0) {
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
const entries = await ports.queryShard({ prefixSlot: shard.prefixSlot, targetTier: shard.targetTier });
|
|
215
|
+
shardsQueried++;
|
|
216
|
+
for (const entry of entries) {
|
|
217
|
+
if (!matchesFilter(entry, opts.filter)) {
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (!verifyProviderEntry(opts.topicId, entry, opts.verifyEntry)) {
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
matched.set(entry.participantId, entry);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
return {
|
|
228
|
+
providers: [...matched.values()],
|
|
229
|
+
selectedShards: selected,
|
|
230
|
+
shardsQueried,
|
|
231
|
+
aggregateAvailable: true,
|
|
232
|
+
aggregateTrusted,
|
|
233
|
+
};
|
|
234
|
+
}
|