@bitsocial/pubsub-votes 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -0
- package/README.md +212 -0
- package/dist/chain/bucket.d.ts +13 -0
- package/dist/chain/bucket.js +24 -0
- package/dist/chain/ticker.d.ts +15 -0
- package/dist/chain/ticker.js +25 -0
- package/dist/chain/types.d.ts +90 -0
- package/dist/chain/types.js +1 -0
- package/dist/checkpoint/codec.d.ts +54 -0
- package/dist/checkpoint/codec.js +99 -0
- package/dist/client/root-puller.d.ts +49 -0
- package/dist/client/root-puller.js +140 -0
- package/dist/client/voter.d.ts +225 -0
- package/dist/client/voter.js +1195 -0
- package/dist/crdt/codec.d.ts +41 -0
- package/dist/crdt/codec.js +137 -0
- package/dist/crdt/crdt.d.ts +22 -0
- package/dist/crdt/crdt.js +127 -0
- package/dist/crdt/store.d.ts +8 -0
- package/dist/crdt/store.js +23 -0
- package/dist/crdt/types.d.ts +87 -0
- package/dist/crdt/types.js +1 -0
- package/dist/encoding/canonical.d.ts +22 -0
- package/dist/encoding/canonical.js +26 -0
- package/dist/errors.d.ts +72 -0
- package/dist/errors.js +111 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.js +39 -0
- package/dist/rules/constant.d.ts +14 -0
- package/dist/rules/constant.js +18 -0
- package/dist/rules/erc20-balance.d.ts +30 -0
- package/dist/rules/erc20-balance.js +44 -0
- package/dist/rules/erc721-min-balance.d.ts +18 -0
- package/dist/rules/erc721-min-balance.js +56 -0
- package/dist/rules/registry.d.ts +42 -0
- package/dist/rules/registry.js +61 -0
- package/dist/rules/types.d.ts +74 -0
- package/dist/rules/types.js +1 -0
- package/dist/schema/common.d.ts +25 -0
- package/dist/schema/common.js +24 -0
- package/dist/schema/criteria.d.ts +78 -0
- package/dist/schema/criteria.js +76 -0
- package/dist/schema/directory.d.ts +42 -0
- package/dist/schema/directory.js +52 -0
- package/dist/schema/votes.d.ts +55 -0
- package/dist/schema/votes.js +116 -0
- package/dist/signer/eip712.d.ts +103 -0
- package/dist/signer/eip712.js +85 -0
- package/dist/signer/types.d.ts +31 -0
- package/dist/signer/types.js +1 -0
- package/dist/storage/browser.d.ts +3 -0
- package/dist/storage/browser.js +110 -0
- package/dist/storage/memory.d.ts +10 -0
- package/dist/storage/memory.js +56 -0
- package/dist/storage/node.d.ts +5 -0
- package/dist/storage/node.js +106 -0
- package/dist/storage/types.d.ts +46 -0
- package/dist/storage/types.js +1 -0
- package/dist/store/indexeddb.d.ts +9 -0
- package/dist/store/indexeddb.js +72 -0
- package/dist/store/memory.d.ts +15 -0
- package/dist/store/memory.js +22 -0
- package/dist/store/select.d.ts +15 -0
- package/dist/store/select.js +64 -0
- package/dist/store/sqlite.d.ts +11 -0
- package/dist/store/sqlite.js +68 -0
- package/dist/store/types.d.ts +57 -0
- package/dist/store/types.js +1 -0
- package/dist/tally/tally.d.ts +44 -0
- package/dist/tally/tally.js +89 -0
- package/dist/tally/types.d.ts +51 -0
- package/dist/tally/types.js +13 -0
- package/dist/topic.d.ts +20 -0
- package/dist/topic.js +28 -0
- package/dist/transport/accepted-dedup.d.ts +30 -0
- package/dist/transport/accepted-dedup.js +34 -0
- package/dist/transport/announce/browser.d.ts +9 -0
- package/dist/transport/announce/browser.js +14 -0
- package/dist/transport/announce/node.d.ts +38 -0
- package/dist/transport/announce/node.js +162 -0
- package/dist/transport/announce/types.d.ts +74 -0
- package/dist/transport/announce/types.js +16 -0
- package/dist/transport/bundle-store.d.ts +11 -0
- package/dist/transport/bundle-store.js +34 -0
- package/dist/transport/chase.d.ts +83 -0
- package/dist/transport/chase.js +88 -0
- package/dist/transport/gossip-validator.d.ts +107 -0
- package/dist/transport/gossip-validator.js +99 -0
- package/dist/transport/helia.d.ts +41 -0
- package/dist/transport/helia.js +98 -0
- package/dist/transport/integration/harness.d.ts +60 -0
- package/dist/transport/integration/harness.js +262 -0
- package/dist/transport/messages.d.ts +97 -0
- package/dist/transport/messages.js +117 -0
- package/dist/transport/rate-limit.d.ts +11 -0
- package/dist/transport/rate-limit.js +20 -0
- package/dist/transport/transport.d.ts +20 -0
- package/dist/transport/transport.js +35 -0
- package/dist/transport/types.d.ts +165 -0
- package/dist/transport/types.js +1 -0
- package/dist/verify/background.d.ts +81 -0
- package/dist/verify/background.js +236 -0
- package/dist/verify/bundle.d.ts +58 -0
- package/dist/verify/bundle.js +84 -0
- package/dist/verify/cache.d.ts +48 -0
- package/dist/verify/cache.js +62 -0
- package/dist/verify/constraints.d.ts +16 -0
- package/dist/verify/constraints.js +35 -0
- package/dist/verify/gate-result-cache.d.ts +65 -0
- package/dist/verify/gate-result-cache.js +91 -0
- package/dist/verify/name-resolution-cache.d.ts +59 -0
- package/dist/verify/name-resolution-cache.js +64 -0
- package/dist/verify/signature.d.ts +9 -0
- package/dist/verify/signature.js +55 -0
- package/dist/verify/types.d.ts +101 -0
- package/dist/verify/types.js +1 -0
- package/package.json +77 -0
package/README.md
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# @bitsocial/pubsub-votes
|
|
2
|
+
|
|
3
|
+
Trustless, leaderless voting over libp2p pubsub, designed to run on top of a host node's shared libp2p/Helia instance.
|
|
4
|
+
|
|
5
|
+
> **Status: engine, reactive facade, and live-delta transport implemented and unit-tested.** The zod schemas, canonical dag-cbor encoding, topic derivation, the verify pipeline (signature + constraints + on-chain gate + community-name resolution), the LWW winner-set CRDT with its binary bundle codec, the tally, the transport's **validate-before-forward gossip gate** over **inline bundle deltas**, and the **root-record checkpoint sync** (on-demand encode, suppressed 10-minute topic heartbeat, libp2p-fetch pull, divergent roots chased via directed bitswap) are all implemented — so the reactive `PubsubVoter` / `Contest` (`createContest`) / `ContestVote` (`createContestVote`) facade is live. The gate runs the full validity pipeline on the message bytes in an async gossipsub topic validator *before* re-forwarding, so an invalid bundle (bad signature, wallet the gate rejects, squatted name) is never propagated and `reject` scores the sender. Cold-join checkpoint bundles instead admit on the synchronous offline checks and settle their **deferred chain checks in the background, batched via multicall3** — the tally renders immediately with per-row `chainVerified`/`nameResolved` flags and refines as they land, and a node's own checkpoint only ever serves fully verified bundles (see [DESIGN.md, Background chain verification](./DESIGN.md#background-chain-verification)). **Keeping a live vote from decaying is the consuming client's job** — this library publishes each vote once and exposes `republishIntervalBuckets` so the client can schedule its own refreshes (see [DESIGN.md, Republishing is the client's job](./DESIGN.md#republishing-is-the-clients-job-not-this-librarys)). What remains is host-side (pkc-js registering gossipsub + `@libp2p/fetch` on the shared node) — see [ROADMAP.md](./ROADMAP.md), [DESIGN.md](./DESIGN.md), the [Transport gate](./DESIGN.md#transport-gossipsub-topic--validation), and [open questions](./DESIGN.md#open-questions).
|
|
6
|
+
|
|
7
|
+
## What it is for
|
|
8
|
+
|
|
9
|
+
The first consumer is [5chan](https://github.com/bitsocialnet/5chan), a serverless, adminless imageboard on the Bitsocial protocol. 5chan has a [competitive directory system](https://github.com/bitsocialnet/5chan/blob/master/README.md#competitive-directory-system): many communities compete for each directory slot (for example, multiple "Business & Finance" communities), but only the highest-voted one appears on the homepage. Today those assignments are curated by hand through pull requests to [`5chan-directories.json`](https://github.com/bitsocialnet/lists/blob/master/5chan-directories.json). This library is the planned replacement: directory voting that is decided by holders rather than by maintainers, with no server to trust.
|
|
10
|
+
|
|
11
|
+
The same engine generalizes to the original use case in [pkc-js issue #25](https://github.com/pkcprotocol/pkc-js/issues/25) (a default-communities list voted on over pubsub) and to any future Bitsocial client that needs holder-weighted, censorship-resistant curation.
|
|
12
|
+
|
|
13
|
+
## Why a separate library (not in pkc-js)
|
|
14
|
+
|
|
15
|
+
[pkc-js](https://github.com/pkcprotocol/pkc-js) (Public Key Communities) is the protocol layer: communities, publications, the challenge exchange. Voting is application/governance layer. Keeping it separate means:
|
|
16
|
+
|
|
17
|
+
- Chain-RPC and governance churn stay out of pkc-js core. pkc-js deliberately touches chains only for name resolution; it has no balance lookups, no chainTicker-to-RPC mapping, and no off-chain vote signing or verification. This library owns all of that.
|
|
18
|
+
- The engine is reusable across clients and contests.
|
|
19
|
+
- The core (`schema/`, `verify/`, `crdt/`, `tally/`) is transport-agnostic and unit-testable without a network. libp2p only appears in `transport/`.
|
|
20
|
+
|
|
21
|
+
This library does not start its own node. It consumes the host's running Helia node directly — no adapter — and drives that node's gossipsub service and blockstore itself. The node must carry a pubsub service at `libp2p.services.pubsub` (a plain Helia node does not — register e.g. `@chainsafe/libp2p-gossipsub`), a usable `blockstore`, and a libp2p fetch service at `libp2p.services.fetch` (register `@libp2p/fetch` — the checkpoint root-record pull rides it); construction throws `MissingPubsubError` / `MissingBlockstoreError` / `MissingFetchError` otherwise. With pkc-js today that node is reached at `pkc.clients.libp2pJsClients[key]._helia`; a version-stable accessor on pkc-js is a planned follow-up (see [DESIGN.md, Deferred pkc-js work](./DESIGN.md#deferred-pkc-js-work)).
|
|
22
|
+
|
|
23
|
+
## Design at a glance
|
|
24
|
+
|
|
25
|
+
- **Settings live in the topic.** `topic = "bitsocial-votes/" + CID(dag-cbor(criteria))`. Two peers on the same topic provably ran identical rules, so the network validates itself with no intermediary.
|
|
26
|
+
- **Votes are a state-based grow-only CRDT.** A signed `Votes` bundle is a standalone dag-cbor block (no parent links); each wallet gossips its own bundle **inline as a live delta**, validated straight from the message bytes — no fetch toward the publisher. State is a last-write-wins set keyed by wallet, so aggregation is a monotonic union: a peer can omit a vote but can never subtract one that an honest peer serves. Cold start and gap-fill exchange a tiny **root record** (libp2p-fetch pull + a slow topic heartbeat) and pull the checkpoint blocks behind it via directed bitswap from its advertisers.
|
|
27
|
+
- **The gate and weight are data, not code.** A fixed rule registry (mirroring pkc-js's challenge registry) maps a `type` string to a verifier. v1 ships exactly the NFT path — an `erc721-min-balance` gate `rule` (5chan Pass) and `constant` weight (1 pass = 1 vote). Balance-derived (token-weighted) voting is deferred; see [ROADMAP.md](./ROADMAP.md).
|
|
28
|
+
|
|
29
|
+
See [DESIGN.md](./DESIGN.md) for the full rationale, including how this resists vote-dropping and how criteria upgrades fork cleanly.
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
The library never starts a node and never takes a host SDK (there is no `pkc` argument). A host passes its own running Helia node in directly and injects its seams into a single `PubsubVoter`:
|
|
34
|
+
|
|
35
|
+
| Seam | Type | Required | Purpose |
|
|
36
|
+
|---|---|---|---|
|
|
37
|
+
| `helia` | `HeliaInstance` | yes | the host's running Helia node; must carry a gossipsub service at `libp2p.services.pubsub` (else `MissingPubsubError`) and a `blockstore` (else `MissingBlockstoreError`) |
|
|
38
|
+
| `chains` | `ChainClientFactory` | yes | builds a viem `PublicClient` per chain; rules read through it for the gate and weight |
|
|
39
|
+
| `signer` | `VoteSigner` | no | the voting wallet's address + EIP-712 ballot signing; omit for a read-only voter |
|
|
40
|
+
| `nameResolvers` | `NameResolver[]` | no | community-name resolvers (same interface and instances as pkc-js's `nameResolvers`, e.g. `@bitsocial/bso-resolver` for `name.bso`); each vote's `community.name` claim is verified through them — inline at the forward-gate for live votes, in the background verifier for cold-join admits — and a bundle whose name resolves to a different `publicKey` than claimed is dropped/evicted |
|
|
41
|
+
| `dataPath` | `string \| false` | no | directory for the voter's persistent caches (gate results + name resolutions), the pkc-js `dataPath` equivalent. Node default: `{cwd}/.bitsocial-pubsub-votes` (better-sqlite3 under `{dataPath}/lru-storage/`); in the browser the path is ignored and the caches live in IndexedDB. Pass `false` for in-memory-only (the pkc-js `noData` equivalent). A restart re-serves settled gate reads and fresh name resolutions from the store instead of the RPC |
|
|
42
|
+
|
|
43
|
+
A contest is addressed by its **full criteria document**, passed to `createContest` / `createContestVote`. The document is strictly validated there (`CriteriaSchema` + the rule registry), and its canonical bytes derive the topic — so the exact document every participant shares is the only contest configuration that exists.
|
|
44
|
+
|
|
45
|
+
### Construct a voter
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
import { PubsubVoter } from "@bitsocial/pubsub-votes";
|
|
49
|
+
|
|
50
|
+
const voter = new PubsubVoter({
|
|
51
|
+
helia, // the host's Helia node; needs a gossipsub service at libp2p.services.pubsub + a blockstore
|
|
52
|
+
chains: viemChainFactory(), // ({ chain, config }) => viem PublicClient
|
|
53
|
+
signer: mySigner, // optional; omit → read-only voter
|
|
54
|
+
nameResolvers: [bsoResolver], // optional; verifies community-name claims (e.g. @bitsocial/bso-resolver)
|
|
55
|
+
dataPath: "/path/to/data" // optional; persistent-cache directory (default {cwd}/.bitsocial-pubsub-votes; false → in-memory)
|
|
56
|
+
});
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Construction throws `MissingPubsubError`, `MissingBlockstoreError`, or `MissingFetchError` if the node lacks a usable pubsub service, blockstore, or libp2p fetch service — the library fails fast rather than letting a later `publish`/`subscribe`/`fetch` fail obscurely. ("Bitswap" is not a separately checkable property — it is a block broker wired beneath `blockstore` — so the validated guarantee is a well-formed blockstore, the surface bitswap retrieves through. The fetch service carries the checkpoint root-record pull; the library registers its own responder on it.)
|
|
60
|
+
|
|
61
|
+
### Read a tally reactively (no signer needed)
|
|
62
|
+
|
|
63
|
+
`createContest` mints a per-contest read object; `update()` starts syncing and it emits `update` (carrying a fresh `tally`) and `error`, just like a plebbit-js `subplebbit`:
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
const contest = await voter.createContest({ criteria }); // criteria: the contest's full document (strictly validated here)
|
|
67
|
+
contest.on("update", () => render(contest.tally)); // tally rides the object; recomputed before each emit
|
|
68
|
+
contest.on("error", (err) => showConnectivityWarning(err)); // tally chain read failed, or the background verifier's RPC/resolver is down (retrying)
|
|
69
|
+
await contest.update(); // join the topic, cold-start, begin emitting
|
|
70
|
+
// const fresh = await contest.getTally(); // or force a fresh read, bypassing the cache
|
|
71
|
+
// await contest.stop(); // leave the topic
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Each ranking row carries one flag **per deferred verification operation** (mirroring pkc-js's
|
|
75
|
+
`nameResolved`), and every background settlement re-fires `update` — so a leaderboard can render
|
|
76
|
+
provisional rows immediately and refine them in place:
|
|
77
|
+
|
|
78
|
+
```ts
|
|
79
|
+
contest.on("update", () => {
|
|
80
|
+
for (const row of contest.tally?.ranking ?? []) {
|
|
81
|
+
// row.community: { name?: string, publicKey: string } — identity is ALWAYS publicKey.
|
|
82
|
+
// Show the name only once it has been checked against the registry.
|
|
83
|
+
const label = row.community.name && row.nameResolved ? row.community.name : row.community.publicKey;
|
|
84
|
+
// row.chainVerified: true once EVERY contributing vote's on-chain gate read confirmed.
|
|
85
|
+
// false means "still being read in the background", never "failed" — a vote that fails
|
|
86
|
+
// a deferred check is evicted and the row recounted instead.
|
|
87
|
+
renderRow(label, row.weight, row.chainVerified ? "verified" : "verifying…");
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
A cold join **renders fast and refines**: checkpoint bundles are admitted after the synchronous offline checks (signature + constraints), so the first tally arrives with `chainVerified: false` rows, and the background verifier then batches the deferred gate reads (one multicall per bucket) and name resolutions — each settlement re-fires `update` with the flags flipped. See [DESIGN.md, Background chain verification](./DESIGN.md#background-chain-verification).
|
|
93
|
+
|
|
94
|
+
Repeated `createContest` calls with byte-identical criteria return the same `Contest` (engines are keyed by topic, the criteria CID).
|
|
95
|
+
|
|
96
|
+
### Publish or withdraw a vote (needs a signer)
|
|
97
|
+
|
|
98
|
+
`createContestVote` mints a publishable ballot; `publish()` signs and broadcasts it once and emits `publishingstatechange`, like a plebbit-js publication:
|
|
99
|
+
|
|
100
|
+
```ts
|
|
101
|
+
const vote = await voter.createContestVote({ criteria, votes: [{ community: { publicKey: "12D3KooW..." }, vote: 1 }] });
|
|
102
|
+
vote.on("publishingstatechange", (state) => console.log(state)); // stopped → signing → publishing → succeeded (or failed)
|
|
103
|
+
const { bundle, recipientCount } = await vote.publish(); // the signed VotesBundle + how many peers gossipsub sent it directly to
|
|
104
|
+
|
|
105
|
+
// Withdraw (active): publish an empty ballot; it supersedes the prior vote under LWW.
|
|
106
|
+
await (await voter.createContestVote({ criteria, votes: [] })).publish();
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
A community's identity is its `publicKey`. The optional `name` is the community's resolvable domain (e.g. `memes.bso`) — unique per community, never a free label: the schema requires a TLD, the name is resolved through the injected `nameResolvers` (inline at the forward-gate for live votes, in the background verifier for cold-join admits), and any bundle whose name resolves to a different `publicKey` than claimed is dropped/evicted. Bundles must also name pairwise-distinct `community.publicKey`s. See [DESIGN.md, Votes wire](./DESIGN.md#votes-wire).
|
|
110
|
+
|
|
111
|
+
`recipientCount` is the peer-reach hint gossipsub reports: how many peers it sent the vote *directly* to at publish time (first-hop fan-out, filtered for send failures) — **not** total network reach, and **not** an acceptance confirmation, since each recipient still runs the forward-gate before re-forwarding. Treat it as a coarse "did this reach anyone?" signal. Note that gossipsub *rejects* the publish with `NoPeersSubscribedToTopic` when it would reach zero peers (common right after joining, before the mesh grafts), unless the host enables `allowPublishToZeroTopicPeers` — so a resolved `recipientCount === 0` only occurs under that host setting; otherwise a no-reach publish surfaces as a thrown error (and a `failed` state).
|
|
112
|
+
|
|
113
|
+
`publish()` on a voter built without a `signer` throws `ReadOnlyError` (and emits an `error`).
|
|
114
|
+
|
|
115
|
+
### Republishing is the client's job
|
|
116
|
+
|
|
117
|
+
A vote is not permanent: a bundle is valid only for `voteExpiryBuckets` after its `blockNumber`, so a live vote must be re-published before it decays. **This library does not do that automatically** — it publishes each vote once and the consuming client decides when (or whether) to refresh. To refresh, just `createContestVote(...).publish()` again; a new bundle at the current bucket supersedes the old one. To stop, simply stop refreshing and let the vote lapse. The library gives you what you need to schedule it — all pure, no chain reads:
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
import { republishIntervalBuckets } from "@bitsocial/pubsub-votes";
|
|
121
|
+
|
|
122
|
+
const cadence = republishIntervalBuckets(criteria); // ceil(voteExpiryBuckets / 2) — the recommended cadence, in buckets
|
|
123
|
+
// A vote sampled at bucket b (bundle.blockNumber / criteria.blocksPerBucket) expires once the
|
|
124
|
+
// current bucket exceeds b + criteria.voteExpiryBuckets; refresh before then.
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
See [DESIGN.md, Republishing is the client's job](./DESIGN.md#republishing-is-the-clients-job-not-this-librarys) for why an always-on re-signer was deliberately kept out of a library that runs on the host's shared node.
|
|
128
|
+
|
|
129
|
+
### Many contests (a 5chan-style directory)
|
|
130
|
+
|
|
131
|
+
One criteria document is one contest (one topic). A directory host authors its documents however it likes — e.g. a local JSONC manifest of shared defaults merged per slot, as in [5chan-directory-criteria.jsonc](./5chan-directory-criteria.jsonc) and [examples/5chan.ts](./examples/5chan.ts) — but what participants must share **byte-identically** is the finished documents themselves (the topic is their CID), so distribute those, not an authoring format. Then create each contest:
|
|
132
|
+
|
|
133
|
+
```ts
|
|
134
|
+
const contests = await Promise.all(allCriteria.map((criteria) => voter.createContest({ criteria }))); // → Contest[]
|
|
135
|
+
for (const contest of contests) await contest.update(); // a full host joins + serves the whole directory
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
There is no separate seeder API: a node that joins a topic (via `update()` or `publish()`) automatically serves that contest's checkpoint root record over libp2p-fetch — the responder registers itself on the first joined topic and unregisters when the last is left. A seeder is just a client that joins everything.
|
|
139
|
+
|
|
140
|
+
### Lifecycle (`stop` / `destroy`)
|
|
141
|
+
|
|
142
|
+
`stop()` leaves every joined topic but keeps the voter **reusable** — each `Contest` can `update()` again and you can `createContest` afterward. `destroy()` is **terminal** (like pkc-js): it leaves every topic, unregisters the fetch responder, and marks the voter and its contests dead — any later `createContest`/`createContestVote`, or a pre-existing `Contest.update()`/`ContestVote.publish()`, throws `VoterDestroyedError`. Construct a new `PubsubVoter` to participate again. (There is no store to dispose — republishing is the client's concern.)
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
const voter = new PubsubVoter({ helia, chains, signer });
|
|
146
|
+
// … create + update contests, app runs …
|
|
147
|
+
await voter.destroy(); // terminal: leave all topics, unregister the responder, forbid reuse
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Pure helpers (no node, no network)
|
|
151
|
+
|
|
152
|
+
```ts
|
|
153
|
+
import { topicFor } from "@bitsocial/pubsub-votes";
|
|
154
|
+
|
|
155
|
+
const topic = await topicFor(criteria); // "bitsocial-votes/" + CID(dag-cbor(criteria))
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Full, type-checked call patterns for a pkc-js host, a plebbit/seedit host, and a read-only consumer are in [examples/](./examples/).
|
|
159
|
+
|
|
160
|
+
### Custom rules
|
|
161
|
+
|
|
162
|
+
The gate and weight are a single flat registry of rules, one `type` per file, mirroring the pkc-js challenge registry. Each rule owns its option schema and is evaluated at the bundle's bucket block. Chain-reading rules get `ctx.chain` — the viem `PublicClient` for their `options.chain` — and write their own reads (`readContract`, `getBalance`, ...), pinning each call to the sampled block with `blockNumber: BigInt(ctx.blockNumber)`. There is **one kind**: `evaluate → { score: bigint }`, a non-negative score where `0n` means "does not qualify" (a result object, not a bare `bigint`, so slot-specific fields can be added later). The criteria has two *slots* drawing from the one registry — the **rule** slot treats the score as a gate (`> 0n` admits), the **weight** slot as the vote's magnitude. A wallet's vote counts as `rule.score > 0n ? weight.score : 0n`. A rule that needs a threshold returns `0n` below it (so `erc721-min-balance`'s optional `min` gates), which lets the same rule serve either slot.
|
|
163
|
+
|
|
164
|
+
Built-ins: `erc721-min-balance` (v1) and `constant` (v1). A host adds or shadows rules by `type` via the `rules` option — this is how clients like 5chan or seedit register custom rules without forking the library:
|
|
165
|
+
|
|
166
|
+
```ts
|
|
167
|
+
import { PubsubVoter, type Rule } from "@bitsocial/pubsub-votes";
|
|
168
|
+
import { z } from "zod";
|
|
169
|
+
|
|
170
|
+
const seeditModAllowlist: Rule<{ type: "seedit-mod-allowlist"; allow: string[] }> = {
|
|
171
|
+
type: "seedit-mod-allowlist",
|
|
172
|
+
optionsSchema: z.object({ type: z.literal("seedit-mod-allowlist"), allow: z.array(z.string()) }),
|
|
173
|
+
async evaluate({ options, walletAddress }) {
|
|
174
|
+
return { score: options.allow.includes(walletAddress) ? 1n : 0n }; // gate: 1n admits, 0n rejects
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
|
|
178
|
+
const voter = new PubsubVoter({
|
|
179
|
+
helia, chains,
|
|
180
|
+
rules: { "seedit-mod-allowlist": seeditModAllowlist } // flat map; shadows/extends built-ins by `type`
|
|
181
|
+
});
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
A custom `type` becomes part of `dag-cbor(criteria)`, so it is provably pinned to the topic it runs on, and a client that does not implement a `type` named in `criteria.requires.rules` throws `UnknownRuleError` and recuses itself rather than miscounting.
|
|
185
|
+
|
|
186
|
+
### Weighted voting (deferred)
|
|
187
|
+
|
|
188
|
+
v1 ships `constant` weight (one Pass, one vote) **on purpose** — it resists whale dominance and downvote weaponization. Balance-derived, token-weighted voting (Pass gate + BSO weight via `erc20-balance`) is a designed-but-unshipped capability: the rule path and result shape leave room for it with no engine change, but it is not in the v1 built-ins and carries open governance/abuse and lazy-tally questions. See [ROADMAP.md](./ROADMAP.md) and [DESIGN.md, Future improvements](./DESIGN.md#future-improvements).
|
|
189
|
+
|
|
190
|
+
## Layout
|
|
191
|
+
|
|
192
|
+
```
|
|
193
|
+
src/
|
|
194
|
+
schema/ zod schemas (criteria, votes, shared wire primitives) + inferred types
|
|
195
|
+
encoding/ canonical dag-cbor encoding [implemented]
|
|
196
|
+
topic.ts topic = "bitsocial-votes/" + CID(dag-cbor) [implemented]
|
|
197
|
+
signer/ VoteSigner seam + EIP-712 ballot typed data [implemented]
|
|
198
|
+
client/ reactive facade: PubsubVoter + Contest (createContest) + ContestVote (createContestVote) [implemented]
|
|
199
|
+
errors.ts ReadOnly/MissingPubsub/MissingBlockstore/MissingFetch/... [implemented]
|
|
200
|
+
rules/ one file per `type` + registry/resolver [implemented]
|
|
201
|
+
chain/ ChainClient = viem PublicClient + bucket math [implemented]
|
|
202
|
+
verify/ signature + constraints + full BundleVerifier + verdict cache [implemented]
|
|
203
|
+
crdt/ state-based LWW winner-set: union, binary bundle codec, in-memory store [implemented]
|
|
204
|
+
checkpoint/ deterministic checkpoint codec (root manifest + size-capped chunks) [implemented]
|
|
205
|
+
transport/ async validate-before-forward gossip gate + message codec (inline bundle / root record) + root chase + transport [implemented]
|
|
206
|
+
tally/ deterministic aggregation over pre-validated bundles [implemented]
|
|
207
|
+
index.ts public entry: re-exports + facade + design types
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
## License
|
|
211
|
+
|
|
212
|
+
GPL-3.0-or-later, matching 5chan.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { BucketMath } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Bucket math for one criteria's `blocksPerBucket`.
|
|
4
|
+
*
|
|
5
|
+
* bucketForBlock(block) = Math.floor(block / blocksPerBucket)
|
|
6
|
+
* sampleBlockForBucket(bucket) = bucket * blocksPerBucket (the bucket boundary block)
|
|
7
|
+
*
|
|
8
|
+
* Every verifier prices balances at one sample block per bucket so votes cannot
|
|
9
|
+
* flip-flop mid-bucket and every client agrees. v1 uses the bucket boundary (the head
|
|
10
|
+
* rounded down to `blocksPerBucket`); the boundary lags the head by up to a full bucket,
|
|
11
|
+
* which also places it past any realistic reorg depth (see DESIGN.md "Tally", "CRDT").
|
|
12
|
+
*/
|
|
13
|
+
export declare function makeBucketMath(blocksPerBucket: number): BucketMath;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bucket math for one criteria's `blocksPerBucket`.
|
|
3
|
+
*
|
|
4
|
+
* bucketForBlock(block) = Math.floor(block / blocksPerBucket)
|
|
5
|
+
* sampleBlockForBucket(bucket) = bucket * blocksPerBucket (the bucket boundary block)
|
|
6
|
+
*
|
|
7
|
+
* Every verifier prices balances at one sample block per bucket so votes cannot
|
|
8
|
+
* flip-flop mid-bucket and every client agrees. v1 uses the bucket boundary (the head
|
|
9
|
+
* rounded down to `blocksPerBucket`); the boundary lags the head by up to a full bucket,
|
|
10
|
+
* which also places it past any realistic reorg depth (see DESIGN.md "Tally", "CRDT").
|
|
11
|
+
*/
|
|
12
|
+
export function makeBucketMath(blocksPerBucket) {
|
|
13
|
+
if (!Number.isInteger(blocksPerBucket) || blocksPerBucket <= 0) {
|
|
14
|
+
throw new RangeError(`blocksPerBucket must be a positive integer, got ${blocksPerBucket}`);
|
|
15
|
+
}
|
|
16
|
+
return {
|
|
17
|
+
bucketForBlock(blockNumber) {
|
|
18
|
+
return Math.floor(blockNumber / blocksPerBucket);
|
|
19
|
+
},
|
|
20
|
+
sampleBlockForBucket(bucket) {
|
|
21
|
+
return bucket * blocksPerBucket;
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Criteria, RuleRef } from "../schema/criteria.js";
|
|
2
|
+
/**
|
|
3
|
+
* Resolve which chain a rule reads. A rule's parsed options may name a
|
|
4
|
+
* `chain` ticker (e.g. `erc721-min-balance` -> "base"); a chainless rule (e.g.
|
|
5
|
+
* `constant`) names none, so callers fall back to the first configured chain. Shared by the
|
|
6
|
+
* verifier, the tally, and the facade so the fallback rule stays identical everywhere.
|
|
7
|
+
*/
|
|
8
|
+
/** The `chain` ticker named in a rule's parsed options, or `undefined` if none. */
|
|
9
|
+
export declare function chainTickerOf(options: unknown): string | undefined;
|
|
10
|
+
/**
|
|
11
|
+
* The chain ticker a rule ref uses: its own `chain` option, else the first chain in
|
|
12
|
+
* `requires.chains`. Throws if neither exists (a chainless rule with no configured
|
|
13
|
+
* chains cannot be read).
|
|
14
|
+
*/
|
|
15
|
+
export declare function tickerForRef(criteria: Criteria, ref: RuleRef, options: unknown): string;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Resolve which chain a rule reads. A rule's parsed options may name a
|
|
4
|
+
* `chain` ticker (e.g. `erc721-min-balance` -> "base"); a chainless rule (e.g.
|
|
5
|
+
* `constant`) names none, so callers fall back to the first configured chain. Shared by the
|
|
6
|
+
* verifier, the tally, and the facade so the fallback rule stays identical everywhere.
|
|
7
|
+
*/
|
|
8
|
+
/** The `chain` ticker named in a rule's parsed options, or `undefined` if none. */
|
|
9
|
+
export function chainTickerOf(options) {
|
|
10
|
+
const parsed = z.object({ chain: z.string().min(1) }).safeParse(options);
|
|
11
|
+
return parsed.success ? parsed.data.chain : undefined;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The chain ticker a rule ref uses: its own `chain` option, else the first chain in
|
|
15
|
+
* `requires.chains`. Throws if neither exists (a chainless rule with no configured
|
|
16
|
+
* chains cannot be read).
|
|
17
|
+
*/
|
|
18
|
+
export function tickerForRef(criteria, ref, options) {
|
|
19
|
+
const ticker = chainTickerOf(options) ?? Object.keys(criteria.requires.chains)[0];
|
|
20
|
+
if (!ticker) {
|
|
21
|
+
throw new Error(`criteria rule "${ref.type}" names no chain and requires.chains is empty; ` +
|
|
22
|
+
`cannot resolve a chain client to read it`);
|
|
23
|
+
}
|
|
24
|
+
return ticker;
|
|
25
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { PublicClient } from "viem";
|
|
2
|
+
import type { ChainConfig } from "../schema/criteria.js";
|
|
3
|
+
/**
|
|
4
|
+
* Chain access.
|
|
5
|
+
*
|
|
6
|
+
* A `ChainClient` is just a viem `PublicClient`. The library does not wrap it in a
|
|
7
|
+
* curated read API (no `balanceOfErc20`/`balanceOfErc721` helpers): every rule
|
|
8
|
+
* writes its own reads with the full viem surface (`readContract`, `getBalance`,
|
|
9
|
+
* `call`, multicall, ...) against whatever ABI it needs. That keeps custom
|
|
10
|
+
* rules unconstrained — a host can read any contract shape without waiting for
|
|
11
|
+
* a helper to be added here.
|
|
12
|
+
*
|
|
13
|
+
* Reads must be pinned to an explicit historical block (the sampled block for a
|
|
14
|
+
* bundle's bucket) so every verifier prices the same state — rules pass
|
|
15
|
+
* `blockNumber: BigInt(ctx.blockNumber)` to each viem call. viem is allowed in the
|
|
16
|
+
* core (it carries no libp2p/helia import); only `src/transport/` touches the node.
|
|
17
|
+
*
|
|
18
|
+
* pkc-js has no equivalent: it touches chains only for name resolution and has no
|
|
19
|
+
* balance reads or chainTicker-to-RPC mapping. All of this is net-new here.
|
|
20
|
+
*/
|
|
21
|
+
export type ChainClient = PublicClient;
|
|
22
|
+
/** chainTicker -> client, built from `criteria.requires.chains`. */
|
|
23
|
+
export type ChainClients = Record<string, ChainClient>;
|
|
24
|
+
/**
|
|
25
|
+
* Factory the host provides: turn a chain config into a viem `PublicClient`
|
|
26
|
+
* (typically `createPublicClient({ transport: http(config.rpcUrls[0]) })`).
|
|
27
|
+
* Declared here so the public API can describe how chain clients are supplied.
|
|
28
|
+
*/
|
|
29
|
+
export type ChainClientFactory = (args: {
|
|
30
|
+
chain: string;
|
|
31
|
+
config: ChainConfig;
|
|
32
|
+
}) => ChainClient;
|
|
33
|
+
/**
|
|
34
|
+
* A community-name resolver the host injects (`PubsubVoterOptions.nameResolvers`). The
|
|
35
|
+
* shape is structurally identical to pkc-js's `NameResolverInterface`, so a host passes
|
|
36
|
+
* the very same instances it already gives pkc-js (e.g. `@bitsocial/bso-resolver`'s
|
|
37
|
+
* `BsoResolver`, which resolves `name.bso` through the `bitsocial` text record) —
|
|
38
|
+
* declared here rather than imported so this library depends on no resolver package.
|
|
39
|
+
*
|
|
40
|
+
* The tally uses it to verify a vote's `community.name` claim: resolve the name and drop
|
|
41
|
+
* the bundle when it does not resolve or resolves to a different `publicKey` than the
|
|
42
|
+
* vote claims (see DESIGN.md "Tally"). `resolve` returning `undefined` means the name
|
|
43
|
+
* has no record. `resolve` accepts an optional `blockNumber` to pin the read to a
|
|
44
|
+
* canonical historical block (bso-resolver#3, since resolved upstream); when omitted it
|
|
45
|
+
* resolves at head. v1 still resolves at head: the registry lives on its own chain, so
|
|
46
|
+
* pinning also needs a canonical per-bucket block *on the registry's chain* — that
|
|
47
|
+
* multi-chain block-selection half is still open. See DESIGN.md "Open questions",
|
|
48
|
+
* "Pinned-block name resolution".
|
|
49
|
+
*/
|
|
50
|
+
export interface NameResolver {
|
|
51
|
+
/** Identifies this resolver instance (e.g. "bso-viem"). */
|
|
52
|
+
key: string;
|
|
53
|
+
/** The backing provider label (e.g. "viem"). */
|
|
54
|
+
provider: string;
|
|
55
|
+
/** Resolve a name to its record; `undefined` when the name has no record. */
|
|
56
|
+
resolve: (opts: {
|
|
57
|
+
name: string;
|
|
58
|
+
/**
|
|
59
|
+
* Pin the text-record read to a canonical historical block; resolves at head
|
|
60
|
+
* when omitted. v1 leaves it unset (head) until per-bucket block selection on
|
|
61
|
+
* the registry's chain lands — see the interface note above.
|
|
62
|
+
*/
|
|
63
|
+
blockNumber?: bigint;
|
|
64
|
+
abortSignal?: AbortSignal;
|
|
65
|
+
}) => Promise<{
|
|
66
|
+
publicKey: string;
|
|
67
|
+
[key: string]: string;
|
|
68
|
+
} | undefined>;
|
|
69
|
+
/** True when this resolver handles the name's TLD (e.g. ends with ".bso"). */
|
|
70
|
+
canResolve: (opts: {
|
|
71
|
+
name: string;
|
|
72
|
+
}) => boolean;
|
|
73
|
+
destroy?: () => Promise<void>;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Bucket math (documented here, implemented later).
|
|
77
|
+
*
|
|
78
|
+
* bucketForBlock(block) = Math.floor(block / blocksPerBucket)
|
|
79
|
+
* sampleBlockForBucket(bucket) = the canonical block at which balances are read
|
|
80
|
+
* for that bucket
|
|
81
|
+
*
|
|
82
|
+
* Using one sample block per bucket is what stops votes from flip-flopping mid
|
|
83
|
+
* bucket and what makes every verifier agree. The exact sample-block rule (bucket
|
|
84
|
+
* start, or a block derived from a blockhash to resist flash-loan timing) is a
|
|
85
|
+
* tuning decision recorded in DESIGN.md.
|
|
86
|
+
*/
|
|
87
|
+
export interface BucketMath {
|
|
88
|
+
bucketForBlock(blockNumber: number): number;
|
|
89
|
+
sampleBlockForBucket(bucket: number): number;
|
|
90
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { CID } from "multiformats/cid";
|
|
2
|
+
import type { VotesBundle } from "../schema/votes.js";
|
|
3
|
+
/**
|
|
4
|
+
* Checkpoint codec — the compacted snapshot of a topic's current LWW winners (one bundle per
|
|
5
|
+
* wallet), used for fast cold-start and storage compaction (see DESIGN.md "Checkpoints"). Pure and
|
|
6
|
+
* network-free: it turns a winner set into content-addressed blocks and back, so it is unit-testable
|
|
7
|
+
* offline. Fetching/publishing those blocks over libp2p is a separate, host-blocked concern.
|
|
8
|
+
*
|
|
9
|
+
* Format — a shallow depth-2 pagination DAG (NOT a history DAG: no parent links, regenerated fresh
|
|
10
|
+
* each cut):
|
|
11
|
+
* - winners are sorted **ascending by `address`** (unique per wallet under LWW, so no tie);
|
|
12
|
+
* - the sorted bundles are packed, **full and inlined**, into chunk blocks by a size-cap fill:
|
|
13
|
+
* append bundles into a chunk until the next would push the chunk's inlined-bundle bytes over
|
|
14
|
+
* `maxChunkBytes`, then start a new chunk (a single oversized bundle still forms its own chunk);
|
|
15
|
+
* - a root block lists the chunk CIDs: `{ chunks: CID[] }`.
|
|
16
|
+
*
|
|
17
|
+
* Canonical dag-cbor + the address sort + the size-cap rule make the bytes a pure function of the
|
|
18
|
+
* winner set, so any two seeders with the same view produce the **same root CID** and their blocks
|
|
19
|
+
* dedupe. The byte layout is pinned by the fixed test vector in `codec.test.ts`.
|
|
20
|
+
*/
|
|
21
|
+
/** One content-addressed checkpoint block (a chunk, or the root manifest). */
|
|
22
|
+
export interface CheckpointBlock {
|
|
23
|
+
cid: CID;
|
|
24
|
+
bytes: Uint8Array;
|
|
25
|
+
}
|
|
26
|
+
/** The result of a checkpoint cut: the root CID plus every block to store (chunks + root). */
|
|
27
|
+
export interface EncodedCheckpoint {
|
|
28
|
+
root: CID;
|
|
29
|
+
/**
|
|
30
|
+
* The chunk-CID index (the root manifest's contents), in root order. Exposed so the
|
|
31
|
+
* root record can carry it (see DESIGN.md "Checkpoints", "Block pull"): a cold joiner
|
|
32
|
+
* handed a verified chunk index skips the root-manifest bitswap round-trip and pulls
|
|
33
|
+
* every chunk in parallel. `CID(encodeCanonical({ chunks })) === root`, so it is
|
|
34
|
+
* self-verifying against the root and cannot be a new trust vector.
|
|
35
|
+
*/
|
|
36
|
+
chunks: CID[];
|
|
37
|
+
blocks: CheckpointBlock[];
|
|
38
|
+
}
|
|
39
|
+
/** Default chunk ceiling (bytes of inlined bundles per chunk), just under a 1 MiB block. */
|
|
40
|
+
export declare const DEFAULT_MAX_CHUNK_BYTES: number;
|
|
41
|
+
/**
|
|
42
|
+
* Encode a winner set into checkpoint blocks. `winners` should be the CRDT's current (non-expired)
|
|
43
|
+
* LWW winners; order does not matter (they are sorted here). Returns the root CID and every block to
|
|
44
|
+
* persist. Deterministic: identical winners + `maxChunkBytes` ⇒ identical bytes and root CID.
|
|
45
|
+
*/
|
|
46
|
+
export declare function encodeCheckpoint(winners: VotesBundle[], maxChunkBytes?: number): Promise<EncodedCheckpoint>;
|
|
47
|
+
/**
|
|
48
|
+
* Decode a checkpoint back into its inlined winner bundles, given a way to fetch each block by CID
|
|
49
|
+
* (blockstore/bitswap). This is a *structural* decode — each bundle is schema-validated (wire shape,
|
|
50
|
+
* B58 key, distinct communities) but NOT signature-verified; the caller re-verifies each through the
|
|
51
|
+
* same verifier the gate uses before merging (a single seeder cannot forge or hide a vote — see
|
|
52
|
+
* DESIGN.md "Checkpoints"). Throws if a referenced block is unavailable or malformed.
|
|
53
|
+
*/
|
|
54
|
+
export declare function decodeCheckpoint(root: CID, getBlock: (cid: CID) => Promise<Uint8Array | undefined>, knownChunks?: CID[]): Promise<VotesBundle[]>;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { CID } from "multiformats/cid";
|
|
2
|
+
import { sha256 } from "multiformats/hashes/sha2";
|
|
3
|
+
import * as dagCbor from "@ipld/dag-cbor";
|
|
4
|
+
import { encodeCanonical, dagCborCode } from "../encoding/canonical.js";
|
|
5
|
+
import { encodeBundle, toWireBundle, fromWireBundle } from "../crdt/codec.js";
|
|
6
|
+
/** Default chunk ceiling (bytes of inlined bundles per chunk), just under a 1 MiB block. */
|
|
7
|
+
export const DEFAULT_MAX_CHUNK_BYTES = 1 << 20;
|
|
8
|
+
async function blockFor(bytes) {
|
|
9
|
+
const digest = await sha256.digest(bytes);
|
|
10
|
+
return { cid: CID.createV1(dagCborCode, digest), bytes };
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The root-manifest block for a chunk-CID list: `{ chunks }` canonically encoded and hashed. Its
|
|
14
|
+
* CID is the checkpoint root, so re-deriving it from a chunk list proves the list belongs to a
|
|
15
|
+
* given root — the local check that lets a joiner trust a piggybacked chunk index (see
|
|
16
|
+
* {@link decodeCheckpoint}) without fetching the manifest.
|
|
17
|
+
*/
|
|
18
|
+
async function checkpointRootBlock(chunks) {
|
|
19
|
+
const root = { chunks };
|
|
20
|
+
return blockFor(encodeCanonical(root));
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Encode a winner set into checkpoint blocks. `winners` should be the CRDT's current (non-expired)
|
|
24
|
+
* LWW winners; order does not matter (they are sorted here). Returns the root CID and every block to
|
|
25
|
+
* persist. Deterministic: identical winners + `maxChunkBytes` ⇒ identical bytes and root CID.
|
|
26
|
+
*/
|
|
27
|
+
export async function encodeCheckpoint(winners, maxChunkBytes = DEFAULT_MAX_CHUNK_BYTES) {
|
|
28
|
+
// Lowercase before comparing so the sort equals raw byte order regardless of how a caller
|
|
29
|
+
// cased an address — casing is presentation, the wire form is lowercase bytes.
|
|
30
|
+
const sorted = [...winners]
|
|
31
|
+
.map((bundle) => ({ bundle, key: bundle.address.toLowerCase() }))
|
|
32
|
+
.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
|
|
33
|
+
.map((entry) => entry.bundle);
|
|
34
|
+
// Size-cap fill on the inlined-bundle byte total (a pure function of the bundles, so seeders
|
|
35
|
+
// agree without encoding a partial chunk each step).
|
|
36
|
+
const chunks = [];
|
|
37
|
+
let current = [];
|
|
38
|
+
let currentBytes = 0;
|
|
39
|
+
for (const bundle of sorted) {
|
|
40
|
+
const size = encodeBundle(bundle).length;
|
|
41
|
+
if (current.length > 0 && currentBytes + size > maxChunkBytes) {
|
|
42
|
+
chunks.push(current);
|
|
43
|
+
current = [];
|
|
44
|
+
currentBytes = 0;
|
|
45
|
+
}
|
|
46
|
+
current.push(bundle);
|
|
47
|
+
currentBytes += size;
|
|
48
|
+
}
|
|
49
|
+
if (current.length > 0)
|
|
50
|
+
chunks.push(current);
|
|
51
|
+
const blocks = [];
|
|
52
|
+
const chunkCids = [];
|
|
53
|
+
for (const chunk of chunks) {
|
|
54
|
+
// Chunks inline the same binary wire objects the bundle block uses (see crdt/codec.ts),
|
|
55
|
+
// so the binary-field byte saving multiplies across every inlined winner.
|
|
56
|
+
const block = await blockFor(encodeCanonical(chunk.map(toWireBundle)));
|
|
57
|
+
blocks.push(block);
|
|
58
|
+
chunkCids.push(block.cid);
|
|
59
|
+
}
|
|
60
|
+
const rootBlock = await checkpointRootBlock(chunkCids);
|
|
61
|
+
blocks.push(rootBlock);
|
|
62
|
+
return { root: rootBlock.cid, chunks: chunkCids, blocks };
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Decode a checkpoint back into its inlined winner bundles, given a way to fetch each block by CID
|
|
66
|
+
* (blockstore/bitswap). This is a *structural* decode — each bundle is schema-validated (wire shape,
|
|
67
|
+
* B58 key, distinct communities) but NOT signature-verified; the caller re-verifies each through the
|
|
68
|
+
* same verifier the gate uses before merging (a single seeder cannot forge or hide a vote — see
|
|
69
|
+
* DESIGN.md "Checkpoints"). Throws if a referenced block is unavailable or malformed.
|
|
70
|
+
*/
|
|
71
|
+
export async function decodeCheckpoint(root, getBlock, knownChunks) {
|
|
72
|
+
// If the caller supplies a chunk index (piggybacked on the root record — see DESIGN.md
|
|
73
|
+
// "Block pull"), trust it only after re-deriving the manifest and checking its CID equals
|
|
74
|
+
// `root`: a lie fails this local check and falls back to the manifest fetch, so the index is
|
|
75
|
+
// an optimization, never a new trust vector. When it verifies, the root-manifest bitswap
|
|
76
|
+
// round-trip is skipped entirely.
|
|
77
|
+
let chunks;
|
|
78
|
+
if (knownChunks !== undefined && (await checkpointRootBlock(knownChunks)).cid.equals(root)) {
|
|
79
|
+
chunks = knownChunks;
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
const rootBytes = await getBlock(root);
|
|
83
|
+
if (!rootBytes)
|
|
84
|
+
throw new Error(`checkpoint root block ${root.toString()} is unavailable`);
|
|
85
|
+
chunks = dagCbor.decode(rootBytes).chunks;
|
|
86
|
+
}
|
|
87
|
+
// The chunks are independent blocks, so pull them concurrently (one bitswap round-trip for
|
|
88
|
+
// the whole set, not one per chunk); decode preserves root order for a deterministic result.
|
|
89
|
+
const perChunk = await Promise.all(chunks.map(async (chunkCid) => {
|
|
90
|
+
const chunkBytes = await getBlock(chunkCid);
|
|
91
|
+
if (!chunkBytes)
|
|
92
|
+
throw new Error(`checkpoint chunk block ${chunkCid.toString()} is unavailable`);
|
|
93
|
+
const wires = dagCbor.decode(chunkBytes);
|
|
94
|
+
if (!Array.isArray(wires))
|
|
95
|
+
throw new Error(`checkpoint chunk ${chunkCid.toString()} is not a bundle array`);
|
|
96
|
+
return wires.map(fromWireBundle);
|
|
97
|
+
}));
|
|
98
|
+
return perChunk.flat();
|
|
99
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { PeerId } from "@libp2p/interface";
|
|
2
|
+
import type { FetchServiceLike } from "../transport/types.js";
|
|
3
|
+
import { type FetchRootRecord } from "../transport/messages.js";
|
|
4
|
+
/**
|
|
5
|
+
* The cold-start root puller: the voter-wide seam every engine's cold-join pull goes through
|
|
6
|
+
* (see DESIGN.md "Checkpoints"). One instance per voter, because everything it guards is
|
|
7
|
+
* per-PEER, not per-topic:
|
|
8
|
+
*
|
|
9
|
+
* - **Batching.** Pulls to the same peer that arrive within {@link BATCH_WINDOW_MS} coalesce
|
|
10
|
+
* into ONE fetch stream carrying a batch key ({@link batchRootsFetchKey}), so a directory-
|
|
11
|
+
* scale join pays the ~2-RTT multistream-select negotiation once per peer instead of once
|
|
12
|
+
* per contest. A single pending topic skips the batch key entirely (the common single-board
|
|
13
|
+
* case has zero new wire surface). A responder that predates the batch key answers
|
|
14
|
+
* NOT_FOUND (or garbage), and the puller falls back to today's per-topic keys.
|
|
15
|
+
* - **Budget.** At most {@link COLD_START_PEER_FETCH_LIMIT} concurrent fetch streams per peer
|
|
16
|
+
* across ALL contests, under libp2p's default per-protocol caps (32 inbound on the
|
|
17
|
+
* responder, 64 outbound on us — both enforced PER CONNECTION per direction, so one
|
|
18
|
+
* connection's budget is exactly the scope of the remote cap; other users of a shared
|
|
19
|
+
* seeder arrive on their own connections and do not eat these slots). 24 rather than the
|
|
20
|
+
* full 32 because running at the cliff still resets: our slot frees when the response
|
|
21
|
+
* lands, but the responder only decrements its count when it sees the stream *close*, so
|
|
22
|
+
* back-to-back reuse races that bookkeeping — and the same connection can carry fetch
|
|
23
|
+
* streams the budget cannot see (the host's own IPNS-over-pubsub record fetches ride the
|
|
24
|
+
* same protocol; so would a second voter on the shared node).
|
|
25
|
+
* - **Retry.** A THROWN fetch retries with full-jittered exponential backoff until
|
|
26
|
+
* {@link COLD_START_FETCH_DEADLINE_MS} — the safety net for a responder saturated by
|
|
27
|
+
* streams the budget cannot see. While the cap is saturated every freed slot is instantly
|
|
28
|
+
* retaken, so a fixed attempt count can lose the race and strand a board (measured: no
|
|
29
|
+
* retry → 32/63 boards converge; 5 fixed retries → 53/63; retry-to-deadline → 63/63). Only
|
|
30
|
+
* a throw retries — a definitive `undefined`/`null` ("no record") returns as-is — and a
|
|
31
|
+
* pull whose contest was torn down (`isLive()` false) abandons quietly. Each attempt (not
|
|
32
|
+
* the whole loop, so a backoff sleep never holds a slot) passes through the budget; queue
|
|
33
|
+
* wait counts against the same deadline.
|
|
34
|
+
*/
|
|
35
|
+
export interface RootPuller {
|
|
36
|
+
/**
|
|
37
|
+
* Pull one topic's root record from one peer: the decoded record, or `null`/`undefined`
|
|
38
|
+
* ("no record", definitive), or a rejection (unreachable peer past the deadline, or a
|
|
39
|
+
* garbage answer). `isLive` is polled between retries and before resolving, so a contest
|
|
40
|
+
* left mid-pull abandons instead of holding work alive.
|
|
41
|
+
*/
|
|
42
|
+
pull(peer: PeerId, topic: string, isLive: () => boolean): Promise<FetchRootRecord | null | undefined>;
|
|
43
|
+
}
|
|
44
|
+
/** See the budget note on {@link RootPuller}. */
|
|
45
|
+
export declare const COLD_START_PEER_FETCH_LIMIT = 24;
|
|
46
|
+
/** See the retry note on {@link RootPuller}. */
|
|
47
|
+
export declare const COLD_START_FETCH_DEADLINE_MS = 30000;
|
|
48
|
+
/** Build the voter-wide puller over the host's fetch service. */
|
|
49
|
+
export declare function makeRootPuller(fetch: FetchServiceLike): RootPuller;
|