@optimystic/db-core 1.17.0 → 1.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/README.md +2 -1
  2. package/dist/src/blocks/helpers.d.ts +7 -0
  3. package/dist/src/blocks/helpers.d.ts.map +1 -1
  4. package/dist/src/blocks/helpers.js +7 -1
  5. package/dist/src/blocks/helpers.js.map +1 -1
  6. package/dist/src/btree/btree.d.ts +100 -4
  7. package/dist/src/btree/btree.d.ts.map +1 -1
  8. package/dist/src/btree/btree.js +252 -90
  9. package/dist/src/btree/btree.js.map +1 -1
  10. package/dist/src/btree/node-division.d.ts +37 -0
  11. package/dist/src/btree/node-division.d.ts.map +1 -0
  12. package/dist/src/btree/node-division.js +133 -0
  13. package/dist/src/btree/node-division.js.map +1 -0
  14. package/dist/src/chain/chain.d.ts +23 -0
  15. package/dist/src/chain/chain.d.ts.map +1 -1
  16. package/dist/src/chain/chain.js +43 -6
  17. package/dist/src/chain/chain.js.map +1 -1
  18. package/dist/src/collection/collection.d.ts +38 -5
  19. package/dist/src/collection/collection.d.ts.map +1 -1
  20. package/dist/src/collection/collection.js +126 -11
  21. package/dist/src/collection/collection.js.map +1 -1
  22. package/dist/src/collection/struct.d.ts +20 -0
  23. package/dist/src/collection/struct.d.ts.map +1 -1
  24. package/dist/src/collection/struct.js.map +1 -1
  25. package/dist/src/collections/tree/tree.d.ts.map +1 -1
  26. package/dist/src/collections/tree/tree.js +5 -3
  27. package/dist/src/collections/tree/tree.js.map +1 -1
  28. package/dist/src/log/log.d.ts +5 -1
  29. package/dist/src/log/log.d.ts.map +1 -1
  30. package/dist/src/log/log.js +7 -1
  31. package/dist/src/log/log.js.map +1 -1
  32. package/dist/src/network/block-size.d.ts +67 -0
  33. package/dist/src/network/block-size.d.ts.map +1 -0
  34. package/dist/src/network/block-size.js +82 -0
  35. package/dist/src/network/block-size.js.map +1 -0
  36. package/dist/src/network/index.d.ts +2 -0
  37. package/dist/src/network/index.d.ts.map +1 -1
  38. package/dist/src/network/index.js +2 -0
  39. package/dist/src/network/index.js.map +1 -1
  40. package/dist/src/network/struct.d.ts +13 -11
  41. package/dist/src/network/struct.d.ts.map +1 -1
  42. package/dist/src/network/struct.js.map +1 -1
  43. package/dist/src/network/transaction-size.d.ts +59 -0
  44. package/dist/src/network/transaction-size.d.ts.map +1 -0
  45. package/dist/src/network/transaction-size.js +74 -0
  46. package/dist/src/network/transaction-size.js.map +1 -0
  47. package/dist/src/reactivity/config.d.ts +8 -4
  48. package/dist/src/reactivity/config.d.ts.map +1 -1
  49. package/dist/src/reactivity/config.js +7 -3
  50. package/dist/src/reactivity/config.js.map +1 -1
  51. package/dist/src/reactivity/rotation.d.ts +9 -3
  52. package/dist/src/reactivity/rotation.d.ts.map +1 -1
  53. package/dist/src/reactivity/rotation.js +8 -2
  54. package/dist/src/reactivity/rotation.js.map +1 -1
  55. package/dist/src/testing/test-transactor.d.ts +2 -0
  56. package/dist/src/testing/test-transactor.d.ts.map +1 -1
  57. package/dist/src/testing/test-transactor.js +10 -0
  58. package/dist/src/testing/test-transactor.js.map +1 -1
  59. package/dist/src/transaction/coordinator.d.ts +72 -3
  60. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  61. package/dist/src/transaction/coordinator.js +320 -146
  62. package/dist/src/transaction/coordinator.js.map +1 -1
  63. package/dist/src/transactor/change-notifier.d.ts +19 -8
  64. package/dist/src/transactor/change-notifier.d.ts.map +1 -1
  65. package/dist/src/transactor/change-notifier.js.map +1 -1
  66. package/dist/src/transactor/network-transactor.d.ts +57 -8
  67. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  68. package/dist/src/transactor/network-transactor.js +185 -24
  69. package/dist/src/transactor/network-transactor.js.map +1 -1
  70. package/dist/src/transactor/transactor.d.ts +14 -0
  71. package/dist/src/transactor/transactor.d.ts.map +1 -1
  72. package/dist/src/transform/digest.d.ts +20 -0
  73. package/dist/src/transform/digest.d.ts.map +1 -1
  74. package/dist/src/transform/digest.js +23 -3
  75. package/dist/src/transform/digest.js.map +1 -1
  76. package/package.json +1 -1
  77. package/src/blocks/helpers.ts +8 -1
  78. package/src/btree/btree.ts +266 -88
  79. package/src/btree/node-division.ts +155 -0
  80. package/src/chain/chain.ts +55 -6
  81. package/src/collection/collection.ts +135 -13
  82. package/src/collection/struct.ts +18 -0
  83. package/src/collections/tree/tree.ts +4 -2
  84. package/src/log/log.ts +7 -1
  85. package/src/network/block-size.ts +81 -0
  86. package/src/network/index.ts +2 -0
  87. package/src/network/struct.ts +15 -13
  88. package/src/network/transaction-size.ts +73 -0
  89. package/src/reactivity/config.ts +8 -4
  90. package/src/reactivity/rotation.ts +9 -3
  91. package/src/testing/test-transactor.ts +12 -0
  92. package/src/transaction/coordinator.ts +383 -154
  93. package/src/transactor/change-notifier.ts +19 -8
  94. package/src/transactor/network-transactor.ts +192 -29
  95. package/src/transactor/transactor.ts +16 -0
  96. package/src/transform/digest.ts +36 -3
@@ -83,18 +83,20 @@ export type PendRequest = ActionTransforms & {
83
83
  */
84
84
  baseRevs?: BlockBaseRevs;
85
85
  /**
86
- * The writer's word that this pend is the action's only one — every block the action will commit
87
- * is in this one request — and that the action's commit goes to the coordinator this pend goes
88
- * to. Only then may a coordinator hold back the pend's final delivery to the other members and
89
- * send it inside the commit's first request (`ClusterCoordinator.mayDeferPendDelivery` in db-p2p):
90
- * a member that refuses the pend late refuses that one commit, before anything of the action is
91
- * durable. A send that pends several requests before committing any — several collections under
92
- * one action id, or one collection split across coordinators — must never carry it: one request's
93
- * late refusal would surface while another request's commit may already have landed, tearing the
94
- * action.
86
+ * The writer's word that this pend, together with the other pends sent with it in one request
87
+ * (`IRepo.pendMany`), is the action's whole reservation — every block the action will commit is in
88
+ * that one request — and that the action's commit goes as one request to the coordinator this pend
89
+ * goes to. Only then may a coordinator hold back the pend record's final delivery to the other
90
+ * members and send it inside the commit's first request (`ClusterCoordinator.mayDeferPendDelivery`
91
+ * in db-p2p): a member that refuses any of the record's pends late refuses that one commit record,
92
+ * before anything of the action is durable. A send that pends in several requests before committing
93
+ * any — several collections one request each under one action id, or one collection split across
94
+ * coordinators — must never carry it: one request's late refusal would surface while another
95
+ * request's commit may already have landed, tearing the action.
95
96
  *
96
- * Set by `TransactorSource.transact` and by `TransactionCoordinator` for a transaction of exactly
97
- * one collection; `NetworkTransactor.pend` keeps it only on a request carrying the whole pend.
97
+ * Set by `TransactorSource.transact`, and by `TransactionCoordinator` for a transaction of exactly
98
+ * one collection or for every request of a `pendMany` (whose commit goes through `commitMany`);
99
+ * `NetworkTransactor.pend` keeps it only on a request carrying the whole pend.
98
100
  * `Collection.completeOwnEntry` re-sends one participant of a half-landed multi-collection action
99
101
  * through `transact`, and carries it too: that re-send pends and commits its one collection on its
100
102
  * own, so its late refusal stops only its own commit, as an on-time refusal would.
@@ -310,8 +312,8 @@ export type CommitRequest = ActionBlocks & {
310
312
  // cost: `StorageRepo.persistProofIfContentMatches` retains no `BlockCommitProof` for a block that
311
313
  // declared no digest (`commit:proof-undeclared`), and a block with no proof is refused by any
312
314
  // receiver running the default `requirePushCertificate: true` (`push:reject-uncertified
313
- // reason=no-proof`). Such a block stays readable and pullable (`handlePull` is not certificate-gated)
314
- // and still repairs by corroboration while two or more holders remain, but it can never GAIN a
315
+ // reason=no-proof`). Such a block stays readable and still repairs by corroboration while two or
316
+ // more holders remain, but it can never GAIN a
315
317
  // holder by push — so spread-on-churn and cohort-growth healing silently stop maintaining its
316
318
  // replication factor. Kept optional anyway because: (a) required-but-nullable is the strongest form
317
319
  // available and still permits `undefined`, so it does not make the bad state unrepresentable, only
@@ -0,0 +1,73 @@
1
+ import type { ActionId } from "../collection/action.js";
2
+ import type { PendRequest } from "./struct.js";
3
+
4
+ /**
5
+ * The most bytes one transaction's pend may encode to: 7.5 MiB, measured by {@link jsonEncodedBytes}
6
+ * over the {@link PendRequest}s that travel together in one consensus record — transforms, validation
7
+ * payload and all. That is the sum of the batch when a transaction's collections travel together
8
+ * (`IRepo.pendMany`), and one collection's pend when they cannot and each is pended on its own.
9
+ *
10
+ * A pend's payload travels whole in one frame on every hop it takes: writer → coordinator on the repo
11
+ * protocol, then coordinator → each other cohort member on the cluster protocol, and back as the
12
+ * member's echoed answer. Both protocols decode under one frame cap (db-p2p's
13
+ * `MAX_BLOCK_MESSAGE_BYTES`), which is this limit plus a reserve for the envelope around the pend, so a
14
+ * transaction under this limit fits every frame it rides in. The limit is the same at every group
15
+ * size: a solo cohort takes no cluster hop, but is held to it all the same, so a write that commits
16
+ * on one machine does not start failing when a second joins.
17
+ *
18
+ * A writer with more to save than this splits it into several transactions.
19
+ */
20
+ export const MAX_TRANSACTION_BYTES = 7.5 * 1024 * 1024;
21
+
22
+ /**
23
+ * The bytes `value` encodes to on the wire: the UTF-8 length of its `JSON.stringify`. JSON is the
24
+ * encoding every request and response frame of the repo and cluster protocols carries, so this is the
25
+ * measure a frame cap is judged by.
26
+ */
27
+ export function jsonEncodedBytes(value: unknown): number {
28
+ return new TextEncoder().encode(JSON.stringify(value)).byteLength;
29
+ }
30
+
31
+ /**
32
+ * A transaction too large to send: its pend encodes to more than {@link MAX_TRANSACTION_BYTES}.
33
+ * Raised before anything is sent, so nothing was reserved and nothing needs cancelling. Not a
34
+ * `StaleFailure` and never retried — the same transaction is the same size on every attempt. The
35
+ * writer has to split the change into smaller transactions.
36
+ */
37
+ export class TransactionTooLargeError extends Error {
38
+ constructor(
39
+ readonly actionId: ActionId,
40
+ /** What the transaction's pend encodes to, by {@link jsonEncodedBytes}. */
41
+ readonly bytes: number,
42
+ /** The limit it exceeded: {@link MAX_TRANSACTION_BYTES}. */
43
+ readonly limit: number = MAX_TRANSACTION_BYTES,
44
+ ) {
45
+ super(`transaction ${actionId} encodes to ${bytes} bytes, over the per-transaction limit MAX_TRANSACTION_BYTES (${limit} bytes); split it into smaller transactions`);
46
+ this.name = 'TransactionTooLargeError';
47
+ }
48
+ }
49
+
50
+ /**
51
+ * What `requests` — one transaction's pends that ride in one consensus record — encode to together,
52
+ * the measure {@link MAX_TRANSACTION_BYTES} is judged by.
53
+ *
54
+ * NOTE: encodes every request once more than sending it does, on the writer and again on the
55
+ * coordinator. Negligible for an ordinary write; if pend encoding ever shows in a profile of large
56
+ * writes, measure once on the writer and hand the size along instead.
57
+ */
58
+ export function transactionBytes(requests: readonly PendRequest[]): number {
59
+ let bytes = 0;
60
+ for (const request of requests) bytes += jsonEncodedBytes(request);
61
+ return bytes;
62
+ }
63
+
64
+ /**
65
+ * Throws {@link TransactionTooLargeError} when `requests` together encode to more than
66
+ * {@link MAX_TRANSACTION_BYTES} ({@link transactionBytes}). The one rule both the writer (before
67
+ * sending) and the coordinator (on receipt) apply.
68
+ */
69
+ export function assertTransactionFits(requests: readonly PendRequest[]): void {
70
+ if (requests.length === 0) return;
71
+ const bytes = transactionBytes(requests);
72
+ if (bytes > MAX_TRANSACTION_BYTES) throw new TransactionTooLargeError(requests[0]!.actionId, bytes);
73
+ }
@@ -43,9 +43,13 @@ export const SUBSCRIBER_TTL_CORE_MS = 90_000;
43
43
  /** Subscriber registration TTL on an Edge node (ms), inherited from cohort-topic. */
44
44
  export const SUBSCRIBER_TTL_EDGE_MS = 60_000;
45
45
  /**
46
- * Entries per log block — drives tail rotation. The collection log starts a new tail block every
47
- * `EntriesPerBlock` entries (`chain/chain.ts`), so this is that number, not a tunable of its own: the fill
48
- * tracker's warm-up and filling signals fire at the commit counts the log actually rotates at.
46
+ * Entries per log block — the most commits a tail block takes before the log rotates. The collection log
47
+ * starts a new tail block after `EntriesPerBlock` entries (`chain/chain.ts`), or sooner once the block
48
+ * reaches `BLOCK_TARGET_BYTES` (`network/block-size.ts`), so this is an upper bound, not a tunable of its
49
+ * own: a log of small entries rotates exactly here, while a log of large entries rotates earlier and its
50
+ * fill tracker's warm-up fires late or not at all. Nothing depends on the count being exact — the warm-up
51
+ * and filling signals are logged only, and live rotation is detected from observed tail changes
52
+ * (`ReactivityOriginationManager.observeTailCommit` in db-p2p).
49
53
  */
50
54
  export const BLOCK_FILL_SIZE_DEFAULT = EntriesPerBlock;
51
55
  /** Old-tail drain time after rotation (ms). Simulator-validated-pending. */
@@ -73,7 +77,7 @@ export interface ReactivityConfig {
73
77
  readonly subscriberTtlCoreMs: number;
74
78
  /** Subscriber TTL on an Edge node (ms). */
75
79
  readonly subscriberTtlEdgeMs: number;
76
- /** Entries per log block — drives tail rotation; the log's `EntriesPerBlock`. */
80
+ /** Most entries per log block before the tail rotates — the log's `EntriesPerBlock`, an upper bound (see {@link BLOCK_FILL_SIZE_DEFAULT}). */
77
81
  readonly blockFillSize: number;
78
82
  /** Old-tail drain time after rotation (ms). Simulator-validated-pending. */
79
83
  readonly tDrainMs: number;
@@ -2,7 +2,8 @@
2
2
  * Reactivity — tail rotation lifecycle (`docs/reactivity.md` §Anchor, §Tail rotation).
3
3
  *
4
4
  * A collection's topic is stable (`reactivityCollectionTopicId`, `topic-anchor.ts`); what rotates is its
5
- * **root**. When the tail block fills (`block_fill_size` entries, the log's `EntriesPerBlock`) a new tail
5
+ * **root**. When the tail block fills (at most `block_fill_size` entries, the log's `EntriesPerBlock`, or
6
+ * sooner by size — `BLOCK_TARGET_BYTES`) a new tail
6
7
  * block is born and the root — the tail's storage group at `H(tailId)` — moves to the new tail's group. The
7
8
  * tiers below the root sit at `coord_d(P, topicId)` and never move, so a rotation is "same topic, the root
8
9
  * moved": a registration at the root re-registers under the new root key; one below the root keeps its place
@@ -13,7 +14,8 @@
13
14
  * by {@link buildRotationHint} and detected subscriber-side by {@link detectRotation}.
14
15
  * - **Block-fill tracking** — {@link BlockFillTracker} counts transactions in the current tail block and
15
16
  * fires anticipatory **warm-up** at `block_fill_size − warm_threshold` and the **filling** signal at
16
- * `block_fill_size` (the commit that carries the hint).
17
+ * `block_fill_size` (the commit that carries the hint). The count is an upper bound: a log of large
18
+ * entries rolls over by size first, so for it the signals come late or not at all.
17
19
  * - **Drain** — {@link TailDrainGate} keeps the outgoing root serving renewals/replays for `T_drain` while
18
20
  * bouncing *new* subscriptions with a `Promoted`-shaped {@link RotationRedirectV1} naming the new tail.
19
21
  * - **Jittered re-registration** — {@link planReRegistration} / {@link planReRegistrationWave} schedule a
@@ -131,7 +133,7 @@ export type BlockFillSignal =
131
133
 
132
134
  /** Construction inputs for a {@link BlockFillTracker}. */
133
135
  export interface BlockFillTrackerInit {
134
- /** Entries per log block before the tail rotates (default {@link BLOCK_FILL_SIZE_DEFAULT}, the log's `EntriesPerBlock`). */
136
+ /** Most entries per log block before the tail rotates (default {@link BLOCK_FILL_SIZE_DEFAULT}, the log's `EntriesPerBlock`; a log of large entries rotates sooner, by size). */
135
137
  readonly blockFillSize?: number;
136
138
  /** Transactions remaining in the tail when anticipatory warm-up fires (default {@link WARM_THRESHOLD_DEFAULT}). */
137
139
  readonly warmThreshold?: number;
@@ -141,6 +143,10 @@ export interface BlockFillTrackerInit {
141
143
  * Counts transactions committed into the current tail block and emits the {@link BlockFillSignal} that
142
144
  * drives anticipatory warm-up and the block-filling rotation trigger (`docs/reactivity.md` §Tail rotation,
143
145
  * §Anticipatory warm-up). One tracker per collection on the tail-cohort primary.
146
+ *
147
+ * It counts commits only, so it assumes the block rotates at `blockFillSize`; a log that rolls over earlier
148
+ * by size leaves it out of step. That is harmless because nothing acts on its signals for correctness: a
149
+ * live node only logs them, and detects the rotation itself from the commit that names a new tail.
144
150
  */
145
151
  export class BlockFillTracker {
146
152
  readonly blockFillSize: number;
@@ -508,6 +508,18 @@ export abstract class DelegatingTransactor implements ITransactor {
508
508
  get queryClusterNominees(): ((blockId: BlockId) => Promise<ClusterNomineesResult>) | undefined {
509
509
  return this.inner.queryClusterNominees?.bind(this.inner);
510
510
  }
511
+
512
+ // Getters for the same reason: the coordinator treats their absence as "pend and commit each
513
+ // collection on its own". `TestTransactor` offers neither, so a wrapped one keeps that path.
514
+ get pendMany(): ITransactor['pendMany'] {
515
+ const inner: ITransactor = this.inner;
516
+ return inner.pendMany?.bind(inner);
517
+ }
518
+
519
+ get commitMany(): ITransactor['commitMany'] {
520
+ const inner: ITransactor = this.inner;
521
+ return inner.commitMany?.bind(inner);
522
+ }
511
523
  }
512
524
 
513
525
  /**