@optimystic/db-core 1.18.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/src/blocks/helpers.d.ts +7 -0
- package/dist/src/blocks/helpers.d.ts.map +1 -1
- package/dist/src/blocks/helpers.js +7 -1
- package/dist/src/blocks/helpers.js.map +1 -1
- package/dist/src/btree/btree.d.ts +100 -4
- package/dist/src/btree/btree.d.ts.map +1 -1
- package/dist/src/btree/btree.js +252 -90
- package/dist/src/btree/btree.js.map +1 -1
- package/dist/src/btree/node-division.d.ts +37 -0
- package/dist/src/btree/node-division.d.ts.map +1 -0
- package/dist/src/btree/node-division.js +133 -0
- package/dist/src/btree/node-division.js.map +1 -0
- package/dist/src/chain/chain.d.ts +23 -0
- package/dist/src/chain/chain.d.ts.map +1 -1
- package/dist/src/chain/chain.js +43 -6
- package/dist/src/chain/chain.js.map +1 -1
- package/dist/src/collection/collection.d.ts +38 -5
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +126 -11
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/struct.d.ts +20 -0
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +5 -3
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/log/log.d.ts +5 -1
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +7 -1
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/network/block-size.d.ts +67 -0
- package/dist/src/network/block-size.d.ts.map +1 -0
- package/dist/src/network/block-size.js +82 -0
- package/dist/src/network/block-size.js.map +1 -0
- package/dist/src/network/index.d.ts +2 -0
- package/dist/src/network/index.d.ts.map +1 -1
- package/dist/src/network/index.js +2 -0
- package/dist/src/network/index.js.map +1 -1
- package/dist/src/network/struct.d.ts +13 -11
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/network/transaction-size.d.ts +59 -0
- package/dist/src/network/transaction-size.d.ts.map +1 -0
- package/dist/src/network/transaction-size.js +74 -0
- package/dist/src/network/transaction-size.js.map +1 -0
- package/dist/src/reactivity/config.d.ts +8 -4
- package/dist/src/reactivity/config.d.ts.map +1 -1
- package/dist/src/reactivity/config.js +7 -3
- package/dist/src/reactivity/config.js.map +1 -1
- package/dist/src/reactivity/rotation.d.ts +9 -3
- package/dist/src/reactivity/rotation.d.ts.map +1 -1
- package/dist/src/reactivity/rotation.js +8 -2
- package/dist/src/reactivity/rotation.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +2 -0
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +10 -0
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +72 -3
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +320 -146
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transactor/change-notifier.d.ts +13 -4
- package/dist/src/transactor/change-notifier.d.ts.map +1 -1
- package/dist/src/transactor/change-notifier.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +57 -8
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +185 -24
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +14 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/digest.d.ts +20 -0
- package/dist/src/transform/digest.d.ts.map +1 -1
- package/dist/src/transform/digest.js +23 -3
- package/dist/src/transform/digest.js.map +1 -1
- package/package.json +1 -1
- package/src/blocks/helpers.ts +8 -1
- package/src/btree/btree.ts +266 -88
- package/src/btree/node-division.ts +155 -0
- package/src/chain/chain.ts +55 -6
- package/src/collection/collection.ts +135 -13
- package/src/collection/struct.ts +18 -0
- package/src/collections/tree/tree.ts +4 -2
- package/src/log/log.ts +7 -1
- package/src/network/block-size.ts +81 -0
- package/src/network/index.ts +2 -0
- package/src/network/struct.ts +15 -13
- package/src/network/transaction-size.ts +73 -0
- package/src/reactivity/config.ts +8 -4
- package/src/reactivity/rotation.ts +9 -3
- package/src/testing/test-transactor.ts +12 -0
- package/src/transaction/coordinator.ts +383 -154
- package/src/transactor/change-notifier.ts +13 -4
- package/src/transactor/network-transactor.ts +192 -29
- package/src/transactor/transactor.ts +16 -0
- package/src/transform/digest.ts +36 -3
package/src/network/struct.ts
CHANGED
|
@@ -83,18 +83,20 @@ export type PendRequest = ActionTransforms & {
|
|
|
83
83
|
*/
|
|
84
84
|
baseRevs?: BlockBaseRevs;
|
|
85
85
|
/**
|
|
86
|
-
* The writer's word that this pend
|
|
87
|
-
* is
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
86
|
+
* The writer's word that this pend, together with the other pends sent with it in one request
|
|
87
|
+
* (`IRepo.pendMany`), is the action's whole reservation — every block the action will commit is in
|
|
88
|
+
* that one request — and that the action's commit goes as one request to the coordinator this pend
|
|
89
|
+
* goes to. Only then may a coordinator hold back the pend record's final delivery to the other
|
|
90
|
+
* members and send it inside the commit's first request (`ClusterCoordinator.mayDeferPendDelivery`
|
|
91
|
+
* in db-p2p): a member that refuses any of the record's pends late refuses that one commit record,
|
|
92
|
+
* before anything of the action is durable. A send that pends in several requests before committing
|
|
93
|
+
* any — several collections one request each under one action id, or one collection split across
|
|
94
|
+
* coordinators — must never carry it: one request's late refusal would surface while another
|
|
95
|
+
* request's commit may already have landed, tearing the action.
|
|
95
96
|
*
|
|
96
|
-
* Set by `TransactorSource.transact
|
|
97
|
-
* one collection
|
|
97
|
+
* Set by `TransactorSource.transact`, and by `TransactionCoordinator` for a transaction of exactly
|
|
98
|
+
* one collection or for every request of a `pendMany` (whose commit goes through `commitMany`);
|
|
99
|
+
* `NetworkTransactor.pend` keeps it only on a request carrying the whole pend.
|
|
98
100
|
* `Collection.completeOwnEntry` re-sends one participant of a half-landed multi-collection action
|
|
99
101
|
* through `transact`, and carries it too: that re-send pends and commits its one collection on its
|
|
100
102
|
* own, so its late refusal stops only its own commit, as an on-time refusal would.
|
|
@@ -310,8 +312,8 @@ export type CommitRequest = ActionBlocks & {
|
|
|
310
312
|
// cost: `StorageRepo.persistProofIfContentMatches` retains no `BlockCommitProof` for a block that
|
|
311
313
|
// declared no digest (`commit:proof-undeclared`), and a block with no proof is refused by any
|
|
312
314
|
// receiver running the default `requirePushCertificate: true` (`push:reject-uncertified
|
|
313
|
-
// reason=no-proof`). Such a block stays readable and
|
|
314
|
-
//
|
|
315
|
+
// reason=no-proof`). Such a block stays readable and still repairs by corroboration while two or
|
|
316
|
+
// more holders remain, but it can never GAIN a
|
|
315
317
|
// holder by push — so spread-on-churn and cohort-growth healing silently stop maintaining its
|
|
316
318
|
// replication factor. Kept optional anyway because: (a) required-but-nullable is the strongest form
|
|
317
319
|
// available and still permits `undefined`, so it does not make the bad state unrepresentable, only
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { ActionId } from "../collection/action.js";
|
|
2
|
+
import type { PendRequest } from "./struct.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The most bytes one transaction's pend may encode to: 7.5 MiB, measured by {@link jsonEncodedBytes}
|
|
6
|
+
* over the {@link PendRequest}s that travel together in one consensus record — transforms, validation
|
|
7
|
+
* payload and all. That is the sum of the batch when a transaction's collections travel together
|
|
8
|
+
* (`IRepo.pendMany`), and one collection's pend when they cannot and each is pended on its own.
|
|
9
|
+
*
|
|
10
|
+
* A pend's payload travels whole in one frame on every hop it takes: writer → coordinator on the repo
|
|
11
|
+
* protocol, then coordinator → each other cohort member on the cluster protocol, and back as the
|
|
12
|
+
* member's echoed answer. Both protocols decode under one frame cap (db-p2p's
|
|
13
|
+
* `MAX_BLOCK_MESSAGE_BYTES`), which is this limit plus a reserve for the envelope around the pend, so a
|
|
14
|
+
* transaction under this limit fits every frame it rides in. The limit is the same at every group
|
|
15
|
+
* size: a solo cohort takes no cluster hop, but is held to it all the same, so a write that commits
|
|
16
|
+
* on one machine does not start failing when a second joins.
|
|
17
|
+
*
|
|
18
|
+
* A writer with more to save than this splits it into several transactions.
|
|
19
|
+
*/
|
|
20
|
+
export const MAX_TRANSACTION_BYTES = 7.5 * 1024 * 1024;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The bytes `value` encodes to on the wire: the UTF-8 length of its `JSON.stringify`. JSON is the
|
|
24
|
+
* encoding every request and response frame of the repo and cluster protocols carries, so this is the
|
|
25
|
+
* measure a frame cap is judged by.
|
|
26
|
+
*/
|
|
27
|
+
export function jsonEncodedBytes(value: unknown): number {
|
|
28
|
+
return new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A transaction too large to send: its pend encodes to more than {@link MAX_TRANSACTION_BYTES}.
|
|
33
|
+
* Raised before anything is sent, so nothing was reserved and nothing needs cancelling. Not a
|
|
34
|
+
* `StaleFailure` and never retried — the same transaction is the same size on every attempt. The
|
|
35
|
+
* writer has to split the change into smaller transactions.
|
|
36
|
+
*/
|
|
37
|
+
export class TransactionTooLargeError extends Error {
|
|
38
|
+
constructor(
|
|
39
|
+
readonly actionId: ActionId,
|
|
40
|
+
/** What the transaction's pend encodes to, by {@link jsonEncodedBytes}. */
|
|
41
|
+
readonly bytes: number,
|
|
42
|
+
/** The limit it exceeded: {@link MAX_TRANSACTION_BYTES}. */
|
|
43
|
+
readonly limit: number = MAX_TRANSACTION_BYTES,
|
|
44
|
+
) {
|
|
45
|
+
super(`transaction ${actionId} encodes to ${bytes} bytes, over the per-transaction limit MAX_TRANSACTION_BYTES (${limit} bytes); split it into smaller transactions`);
|
|
46
|
+
this.name = 'TransactionTooLargeError';
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* What `requests` — one transaction's pends that ride in one consensus record — encode to together,
|
|
52
|
+
* the measure {@link MAX_TRANSACTION_BYTES} is judged by.
|
|
53
|
+
*
|
|
54
|
+
* NOTE: encodes every request once more than sending it does, on the writer and again on the
|
|
55
|
+
* coordinator. Negligible for an ordinary write; if pend encoding ever shows in a profile of large
|
|
56
|
+
* writes, measure once on the writer and hand the size along instead.
|
|
57
|
+
*/
|
|
58
|
+
export function transactionBytes(requests: readonly PendRequest[]): number {
|
|
59
|
+
let bytes = 0;
|
|
60
|
+
for (const request of requests) bytes += jsonEncodedBytes(request);
|
|
61
|
+
return bytes;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Throws {@link TransactionTooLargeError} when `requests` together encode to more than
|
|
66
|
+
* {@link MAX_TRANSACTION_BYTES} ({@link transactionBytes}). The one rule both the writer (before
|
|
67
|
+
* sending) and the coordinator (on receipt) apply.
|
|
68
|
+
*/
|
|
69
|
+
export function assertTransactionFits(requests: readonly PendRequest[]): void {
|
|
70
|
+
if (requests.length === 0) return;
|
|
71
|
+
const bytes = transactionBytes(requests);
|
|
72
|
+
if (bytes > MAX_TRANSACTION_BYTES) throw new TransactionTooLargeError(requests[0]!.actionId, bytes);
|
|
73
|
+
}
|
package/src/reactivity/config.ts
CHANGED
|
@@ -43,9 +43,13 @@ export const SUBSCRIBER_TTL_CORE_MS = 90_000;
|
|
|
43
43
|
/** Subscriber registration TTL on an Edge node (ms), inherited from cohort-topic. */
|
|
44
44
|
export const SUBSCRIBER_TTL_EDGE_MS = 60_000;
|
|
45
45
|
/**
|
|
46
|
-
* Entries per log block —
|
|
47
|
-
* `EntriesPerBlock` entries (`chain/chain.ts`),
|
|
48
|
-
*
|
|
46
|
+
* Entries per log block — the most commits a tail block takes before the log rotates. The collection log
|
|
47
|
+
* starts a new tail block after `EntriesPerBlock` entries (`chain/chain.ts`), or sooner once the block
|
|
48
|
+
* reaches `BLOCK_TARGET_BYTES` (`network/block-size.ts`), so this is an upper bound, not a tunable of its
|
|
49
|
+
* own: a log of small entries rotates exactly here, while a log of large entries rotates earlier and its
|
|
50
|
+
* fill tracker's warm-up fires late or not at all. Nothing depends on the count being exact — the warm-up
|
|
51
|
+
* and filling signals are logged only, and live rotation is detected from observed tail changes
|
|
52
|
+
* (`ReactivityOriginationManager.observeTailCommit` in db-p2p).
|
|
49
53
|
*/
|
|
50
54
|
export const BLOCK_FILL_SIZE_DEFAULT = EntriesPerBlock;
|
|
51
55
|
/** Old-tail drain time after rotation (ms). Simulator-validated-pending. */
|
|
@@ -73,7 +77,7 @@ export interface ReactivityConfig {
|
|
|
73
77
|
readonly subscriberTtlCoreMs: number;
|
|
74
78
|
/** Subscriber TTL on an Edge node (ms). */
|
|
75
79
|
readonly subscriberTtlEdgeMs: number;
|
|
76
|
-
/**
|
|
80
|
+
/** Most entries per log block before the tail rotates — the log's `EntriesPerBlock`, an upper bound (see {@link BLOCK_FILL_SIZE_DEFAULT}). */
|
|
77
81
|
readonly blockFillSize: number;
|
|
78
82
|
/** Old-tail drain time after rotation (ms). Simulator-validated-pending. */
|
|
79
83
|
readonly tDrainMs: number;
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Reactivity — tail rotation lifecycle (`docs/reactivity.md` §Anchor, §Tail rotation).
|
|
3
3
|
*
|
|
4
4
|
* A collection's topic is stable (`reactivityCollectionTopicId`, `topic-anchor.ts`); what rotates is its
|
|
5
|
-
* **root**. When the tail block fills (`block_fill_size` entries, the log's `EntriesPerBlock
|
|
5
|
+
* **root**. When the tail block fills (at most `block_fill_size` entries, the log's `EntriesPerBlock`, or
|
|
6
|
+
* sooner by size — `BLOCK_TARGET_BYTES`) a new tail
|
|
6
7
|
* block is born and the root — the tail's storage group at `H(tailId)` — moves to the new tail's group. The
|
|
7
8
|
* tiers below the root sit at `coord_d(P, topicId)` and never move, so a rotation is "same topic, the root
|
|
8
9
|
* moved": a registration at the root re-registers under the new root key; one below the root keeps its place
|
|
@@ -13,7 +14,8 @@
|
|
|
13
14
|
* by {@link buildRotationHint} and detected subscriber-side by {@link detectRotation}.
|
|
14
15
|
* - **Block-fill tracking** — {@link BlockFillTracker} counts transactions in the current tail block and
|
|
15
16
|
* fires anticipatory **warm-up** at `block_fill_size − warm_threshold` and the **filling** signal at
|
|
16
|
-
* `block_fill_size` (the commit that carries the hint).
|
|
17
|
+
* `block_fill_size` (the commit that carries the hint). The count is an upper bound: a log of large
|
|
18
|
+
* entries rolls over by size first, so for it the signals come late or not at all.
|
|
17
19
|
* - **Drain** — {@link TailDrainGate} keeps the outgoing root serving renewals/replays for `T_drain` while
|
|
18
20
|
* bouncing *new* subscriptions with a `Promoted`-shaped {@link RotationRedirectV1} naming the new tail.
|
|
19
21
|
* - **Jittered re-registration** — {@link planReRegistration} / {@link planReRegistrationWave} schedule a
|
|
@@ -131,7 +133,7 @@ export type BlockFillSignal =
|
|
|
131
133
|
|
|
132
134
|
/** Construction inputs for a {@link BlockFillTracker}. */
|
|
133
135
|
export interface BlockFillTrackerInit {
|
|
134
|
-
/**
|
|
136
|
+
/** Most entries per log block before the tail rotates (default {@link BLOCK_FILL_SIZE_DEFAULT}, the log's `EntriesPerBlock`; a log of large entries rotates sooner, by size). */
|
|
135
137
|
readonly blockFillSize?: number;
|
|
136
138
|
/** Transactions remaining in the tail when anticipatory warm-up fires (default {@link WARM_THRESHOLD_DEFAULT}). */
|
|
137
139
|
readonly warmThreshold?: number;
|
|
@@ -141,6 +143,10 @@ export interface BlockFillTrackerInit {
|
|
|
141
143
|
* Counts transactions committed into the current tail block and emits the {@link BlockFillSignal} that
|
|
142
144
|
* drives anticipatory warm-up and the block-filling rotation trigger (`docs/reactivity.md` §Tail rotation,
|
|
143
145
|
* §Anticipatory warm-up). One tracker per collection on the tail-cohort primary.
|
|
146
|
+
*
|
|
147
|
+
* It counts commits only, so it assumes the block rotates at `blockFillSize`; a log that rolls over earlier
|
|
148
|
+
* by size leaves it out of step. That is harmless because nothing acts on its signals for correctness: a
|
|
149
|
+
* live node only logs them, and detects the rotation itself from the commit that names a new tail.
|
|
144
150
|
*/
|
|
145
151
|
export class BlockFillTracker {
|
|
146
152
|
readonly blockFillSize: number;
|
|
@@ -508,6 +508,18 @@ export abstract class DelegatingTransactor implements ITransactor {
|
|
|
508
508
|
get queryClusterNominees(): ((blockId: BlockId) => Promise<ClusterNomineesResult>) | undefined {
|
|
509
509
|
return this.inner.queryClusterNominees?.bind(this.inner);
|
|
510
510
|
}
|
|
511
|
+
|
|
512
|
+
// Getters for the same reason: the coordinator treats their absence as "pend and commit each
|
|
513
|
+
// collection on its own". `TestTransactor` offers neither, so a wrapped one keeps that path.
|
|
514
|
+
get pendMany(): ITransactor['pendMany'] {
|
|
515
|
+
const inner: ITransactor = this.inner;
|
|
516
|
+
return inner.pendMany?.bind(inner);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
get commitMany(): ITransactor['commitMany'] {
|
|
520
|
+
const inner: ITransactor = this.inner;
|
|
521
|
+
return inner.commitMany?.bind(inner);
|
|
522
|
+
}
|
|
511
523
|
}
|
|
512
524
|
|
|
513
525
|
/**
|