@optimystic/db-core 1.12.1 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/src/cluster/i-cluster.d.ts +16 -1
  2. package/dist/src/cluster/i-cluster.d.ts.map +1 -1
  3. package/dist/src/cluster/structs.d.ts +60 -6
  4. package/dist/src/cluster/structs.d.ts.map +1 -1
  5. package/dist/src/cluster/structs.js +5 -0
  6. package/dist/src/cluster/structs.js.map +1 -1
  7. package/dist/src/collection/collection.d.ts.map +1 -1
  8. package/dist/src/collection/collection.js +5 -3
  9. package/dist/src/collection/collection.js.map +1 -1
  10. package/dist/src/index.d.ts +1 -0
  11. package/dist/src/index.d.ts.map +1 -1
  12. package/dist/src/index.js +1 -0
  13. package/dist/src/index.js.map +1 -1
  14. package/dist/src/network/durability.d.ts +17 -4
  15. package/dist/src/network/durability.d.ts.map +1 -1
  16. package/dist/src/network/durability.js +60 -15
  17. package/dist/src/network/durability.js.map +1 -1
  18. package/dist/src/network/i-key-network.d.ts +9 -0
  19. package/dist/src/network/i-key-network.d.ts.map +1 -1
  20. package/dist/src/network/struct.d.ts +45 -5
  21. package/dist/src/network/struct.d.ts.map +1 -1
  22. package/dist/src/network/struct.js.map +1 -1
  23. package/dist/src/transaction/coordinator.d.ts +23 -14
  24. package/dist/src/transaction/coordinator.d.ts.map +1 -1
  25. package/dist/src/transaction/coordinator.js +84 -48
  26. package/dist/src/transaction/coordinator.js.map +1 -1
  27. package/dist/src/transaction/durability.d.ts +31 -0
  28. package/dist/src/transaction/durability.d.ts.map +1 -0
  29. package/dist/src/transaction/durability.js +36 -0
  30. package/dist/src/transaction/durability.js.map +1 -0
  31. package/dist/src/transaction/errors.d.ts +14 -6
  32. package/dist/src/transaction/errors.d.ts.map +1 -1
  33. package/dist/src/transaction/errors.js +16 -5
  34. package/dist/src/transaction/errors.js.map +1 -1
  35. package/dist/src/transaction/index.d.ts +2 -0
  36. package/dist/src/transaction/index.d.ts.map +1 -1
  37. package/dist/src/transaction/index.js +1 -0
  38. package/dist/src/transaction/index.js.map +1 -1
  39. package/dist/src/transaction/session.d.ts +2 -0
  40. package/dist/src/transaction/session.d.ts.map +1 -1
  41. package/dist/src/transaction/session.js +4 -2
  42. package/dist/src/transaction/session.js.map +1 -1
  43. package/dist/src/transaction/transaction.d.ts +9 -0
  44. package/dist/src/transaction/transaction.d.ts.map +1 -1
  45. package/dist/src/transaction/transaction.js.map +1 -1
  46. package/dist/src/transactor/change-notifier.d.ts +20 -8
  47. package/dist/src/transactor/change-notifier.d.ts.map +1 -1
  48. package/dist/src/transactor/change-notifier.js.map +1 -1
  49. package/dist/src/transactor/network-transactor.d.ts +48 -5
  50. package/dist/src/transactor/network-transactor.d.ts.map +1 -1
  51. package/dist/src/transactor/network-transactor.js +98 -22
  52. package/dist/src/transactor/network-transactor.js.map +1 -1
  53. package/dist/src/transactor/owed-cancels.d.ts +96 -0
  54. package/dist/src/transactor/owed-cancels.d.ts.map +1 -0
  55. package/dist/src/transactor/owed-cancels.js +211 -0
  56. package/dist/src/transactor/owed-cancels.js.map +1 -0
  57. package/dist/src/transactor/transactor-source.d.ts +5 -4
  58. package/dist/src/transactor/transactor-source.d.ts.map +1 -1
  59. package/dist/src/transactor/transactor-source.js +11 -6
  60. package/dist/src/transactor/transactor-source.js.map +1 -1
  61. package/dist/src/utility/unref-timer.d.ts +9 -0
  62. package/dist/src/utility/unref-timer.d.ts.map +1 -0
  63. package/dist/src/utility/unref-timer.js +15 -0
  64. package/dist/src/utility/unref-timer.js.map +1 -0
  65. package/package.json +1 -1
  66. package/src/cluster/i-cluster.ts +13 -2
  67. package/src/cluster/structs.ts +62 -6
  68. package/src/collection/collection.ts +5 -3
  69. package/src/index.ts +1 -0
  70. package/src/network/durability.ts +63 -15
  71. package/src/network/i-key-network.ts +10 -0
  72. package/src/network/struct.ts +45 -5
  73. package/src/transaction/coordinator.ts +130 -79
  74. package/src/transaction/durability.ts +60 -0
  75. package/src/transaction/errors.ts +18 -3
  76. package/src/transaction/index.ts +2 -0
  77. package/src/transaction/session.ts +4 -2
  78. package/src/transaction/transaction.ts +9 -0
  79. package/src/transactor/change-notifier.ts +20 -8
  80. package/src/transactor/network-transactor.ts +120 -29
  81. package/src/transactor/owed-cancels.ts +252 -0
  82. package/src/transactor/transactor-source.ts +11 -6
  83. package/src/utility/unref-timer.ts +15 -0
@@ -25,7 +25,8 @@ export function durabilityRank(q: DurabilityQuorum): number {
25
25
  * Merge per-cohort reports into one action-level report: the weakest becomes the scalar answer and
26
26
  * the rest become `otherCohorts`, so a consumer that reads only the scalar fields is reading the
27
27
  * binding constraint. `torn` is the union across the inputs — an abandoned block is abandoned
28
- * whichever cohort it ran on. Reports that already carry `otherCohorts` are flattened first, so
28
+ * whichever cohort it ran on — and `blockRevs` the union at the higher revision per block, less
29
+ * every torn block. Reports that already carry `otherCohorts` are flattened first, so
29
30
  * merging a merge is the same as merging its inputs. Identical reports collapse to one: a commit
30
31
  * sends the tail and the sweep as separate batches, so ONE cohort routinely answers twice for one
31
32
  * action, and without this a healthy single-cohort commit would list its own cohort under
@@ -46,39 +47,62 @@ export function mergeDurability(reports: readonly WriteDurability[]): WriteDurab
46
47
  const ranked = [...flat].sort((a, b) => durabilityRank(a.quorum) - durabilityRank(b.quorum));
47
48
  const weakest = ranked[0]!;
48
49
  const others = ranked.slice(1);
49
- // Over the ORIGINAL reports: flattening strips `torn` (it is action-level), so the union must
50
- // be taken before that or a torn input merges to an untorn answer.
50
+ // Over the ORIGINAL reports: flattening strips `torn` and `blockRevs` (they are action-level),
51
+ // so both are taken before that or a torn input merges to an untorn answer.
51
52
  const torn = unionTorn(reports);
53
+ const blockRevs = withoutBlocks(highestBlockRevs(reports), torn);
52
54
  return {
53
55
  ...stripMergedFields(weakest),
54
56
  ...(others.length === 0 ? {} : { otherCohorts: others.map(stripMergedFields) }),
55
- ...(torn.length === 0 ? {} : { torn })
57
+ ...(torn.length === 0 ? {} : { torn }),
58
+ ...blockRevsField(blockRevs)
56
59
  };
57
60
  }
58
61
 
62
+ /**
63
+ * Name the blocks a write committed, at the revision it committed them at — set where the commit is
64
+ * confirmed, by the db-core layer that issued it. A block already named keeps the higher revision,
65
+ * and a block in `d.torn` is left out: it holds nothing.
66
+ */
67
+ export function withBlockRevs(d: WriteDurability, blockIds: readonly BlockId[], rev: number): WriteDurability {
68
+ const committed = Object.fromEntries(blockIds.map(blockId => [blockId, rev]));
69
+ const blockRevs = withoutBlocks(highestBlockRevs([d, { blockRevs: committed }]), d.torn ?? []);
70
+ return { ...d, ...blockRevsField(blockRevs) };
71
+ }
72
+
59
73
  /**
60
74
  * Clamp an action-level report below `full` when the action abandoned blocks. A cohort can hold
61
75
  * every block it was asked to commit and the action still be incomplete — the blocks the sweep
62
76
  * never reached are on nobody. Never raises the class; a report already below `full` keeps it.
63
- * Idempotent, and a no-op for an empty `torn`.
77
+ * A torn block leaves `blockRevs`, since it was not committed. Idempotent, and a no-op for an
78
+ * empty `torn`.
64
79
  */
65
80
  export function withTornBlocks(d: WriteDurability, torn: readonly BlockId[]): WriteDurability {
66
81
  if (torn.length === 0) return d;
67
82
  const allTorn = Array.from(new Set([...(d.torn ?? []), ...torn]));
83
+ const { blockRevs: _blockRevs, ...rest } = d;
68
84
  return {
69
- ...d,
85
+ ...rest,
70
86
  quorum: d.quorum === 'full' ? 'majority' : d.quorum,
71
- torn: allTorn
87
+ torn: allTorn,
88
+ ...blockRevsField(withoutBlocks(d.blockRevs ?? {}, allTorn))
72
89
  };
73
90
  }
74
91
 
75
- /** The single-node answer: this node holds it and knows nothing about a cohort beyond itself. */
76
- export function localDurability(selfPeerId?: string): WriteDurability {
92
+ /**
93
+ * The single-node answer: this node holds it and nobody else does yet. `absent` names the cohort
94
+ * members this node knows of that the write ran without — they are part of the cohort the write is
95
+ * owed to, so they count in `cohort` and are listed in `unconfirmed`. With none, it is a one-machine
96
+ * write that is complete. `absent` needs `selfPeerId`: without it the answer cannot name the cohort.
97
+ */
98
+ export function localDurability(selfPeerId?: string, absent: readonly string[] = []): WriteDurability {
99
+ if (selfPeerId === undefined) return { quorum: 'local', confirmed: 1, cohort: 1 };
77
100
  return {
78
101
  quorum: 'local',
79
102
  confirmed: 1,
80
- cohort: 1,
81
- ...(selfPeerId === undefined ? {} : { unconfirmed: [], cohortPeerIds: [selfPeerId] })
103
+ cohort: 1 + absent.length,
104
+ unconfirmed: [...absent],
105
+ cohortPeerIds: [selfPeerId, ...absent]
82
106
  };
83
107
  }
84
108
 
@@ -96,22 +120,25 @@ function flattenDurability(d: WriteDurability): WriteDurability[] {
96
120
  return [stripMergedFields(d), ...(d.otherCohorts ?? [])];
97
121
  }
98
122
 
99
- /** The per-cohort fields only — `otherCohorts` and `torn` are action-level and re-derived by the merge. */
123
+ /** The per-cohort fields only — `otherCohorts`, `torn` and `blockRevs` are action-level and
124
+ * re-derived by the merge. */
100
125
  function stripMergedFields(d: WriteDurability): WriteDurability {
101
- const { otherCohorts: _otherCohorts, torn: _torn, ...cohortReport } = d;
126
+ const { otherCohorts: _otherCohorts, torn: _torn, blockRevs: _blockRevs, ...cohortReport } = d;
102
127
  return cohortReport;
103
128
  }
104
129
 
105
130
  /**
106
131
  * First occurrence of each structurally identical per-cohort report, input order kept. Peer lists
107
132
  * compare as SETS: `findCluster` orders a cohort by routing distance to the block's key, so one
108
- * cohort answering for two blocks lists the same members in two orders.
133
+ * cohort answering for two blocks lists the same members in two orders. `recordedBy` is part of the
134
+ * identity: two otherwise identical answers whose shortfalls two different ledgers recorded are two
135
+ * facts, and a consumer waiting on full-replication events needs to see both.
109
136
  */
110
137
  function distinctReports(reports: readonly WriteDurability[]): WriteDurability[] {
111
138
  const seen = new Set<string>();
112
139
  const sorted = (ids: readonly string[] | undefined): string[] | null => ids === undefined ? null : [...ids].sort();
113
140
  return reports.filter(r => {
114
- const key = JSON.stringify([r.quorum, r.confirmed, r.cohort, sorted(r.unconfirmed), sorted(r.cohortPeerIds)]);
141
+ const key = JSON.stringify([r.quorum, r.confirmed, r.cohort, sorted(r.unconfirmed), sorted(r.cohortPeerIds), r.recordedBy ?? null]);
115
142
  if (seen.has(key)) return false;
116
143
  seen.add(key);
117
144
  return true;
@@ -121,3 +148,24 @@ function distinctReports(reports: readonly WriteDurability[]): WriteDurability[]
121
148
  function unionTorn(reports: readonly WriteDurability[]): BlockId[] {
122
149
  return Array.from(new Set(reports.flatMap(r => r.torn ?? [])));
123
150
  }
151
+
152
+ /** Every block any report names, at the highest revision any of them names it at. */
153
+ function highestBlockRevs(reports: readonly Pick<WriteDurability, 'blockRevs'>[]): Record<BlockId, number> {
154
+ // A Map, not assignment into `{}`: a header block's id is its collection id, which a caller
155
+ // names, and `__proto__` assigned into a plain object sets its prototype instead of a key.
156
+ const highest = new Map<BlockId, number>();
157
+ for (const [blockId, rev] of reports.flatMap(r => Object.entries(r.blockRevs ?? {}))) {
158
+ highest.set(blockId, Math.max(highest.get(blockId) ?? rev, rev));
159
+ }
160
+ return Object.fromEntries(highest);
161
+ }
162
+
163
+ function withoutBlocks(blockRevs: Readonly<Record<BlockId, number>>, excluded: readonly BlockId[]): Record<BlockId, number> {
164
+ const drop = new Set(excluded);
165
+ return Object.fromEntries(Object.entries(blockRevs).filter(([blockId]) => !drop.has(blockId)));
166
+ }
167
+
168
+ /** The field, or nothing: an empty map is left off, as an empty `torn` is. */
169
+ function blockRevsField(blockRevs: Readonly<Record<BlockId, number>>): Pick<WriteDurability, 'blockRevs'> {
170
+ return Object.keys(blockRevs).length === 0 ? {} : { blockRevs };
171
+ }
@@ -71,4 +71,14 @@ export type IKeyNetwork = {
71
71
  * such configuration omit this, which reads as "not awaiting".
72
72
  */
73
73
  awaitingBootstrapContact?(): Promise<boolean>;
74
+
75
+ /**
76
+ * Optionally name the peers this node knows serve this network that belong in `key`'s cohort but
77
+ * are missing from the cohort `findCluster` assembles now — typically members that are offline or
78
+ * unreachable. Peer-id strings, nearest the key first, never this node. A write made without them
79
+ * is owed to them, and naming them lets the writer's answer say so (`WriteDurability.unconfirmed`)
80
+ * and lets the copy be pushed to them as soon as they reconnect. Implementations that remember no
81
+ * such peers omit this, which reads as "none known".
82
+ */
83
+ absentCohortMembers?(key: RoutingKey): Promise<string[]>;
74
84
  }
@@ -82,6 +82,26 @@ export type PendRequest = ActionTransforms & {
82
82
  * pend named no base, not the primary check.
83
83
  */
84
84
  baseRevs?: BlockBaseRevs;
85
+ /**
86
+ * The writer's word that this pend is the action's only one — every block the action will commit
87
+ * is in this one request — and that the action's commit goes to the coordinator this pend goes
88
+ * to. Only then may a coordinator hold back the pend's final delivery to the other members and
89
+ * send it inside the commit's first request (`ClusterCoordinator.mayDeferPendDelivery` in db-p2p):
90
+ * a member that refuses the pend late refuses that one commit, before anything of the action is
91
+ * durable. A send that pends several requests before committing any — several collections under
92
+ * one action id, or one collection split across coordinators — must never carry it: one request's
93
+ * late refusal would surface while another request's commit may already have landed, tearing the
94
+ * action.
95
+ *
96
+ * Set by `TransactorSource.transact` and by `TransactionCoordinator` for a transaction of exactly
97
+ * one collection; `NetworkTransactor.pend` keeps it only on a request carrying the whole pend.
98
+ * `Collection.completeOwnEntry` re-sends one participant of a half-landed multi-collection action
99
+ * through `transact`, and carries it too: that re-send pends and commits its one collection on its
100
+ * own, so its late refusal stops only its own commit, as an on-time refusal would.
101
+ * Omitted rather than `false`, for the same preimage reason as {@link baseRevs}; a coordinator on
102
+ * an older build ignores it, and then delivers at once as it always did.
103
+ */
104
+ soleReservation?: true;
85
105
  };
86
106
 
87
107
  export type BlockActionStatus = ActionBlocks & {
@@ -97,12 +117,15 @@ export type DurabilityQuorum =
97
117
  * (`withTornBlocks` clamps `full` down to this; `torn` then names the blocks, and `unconfirmed`
98
118
  * may be empty). */
99
119
  | 'majority'
100
- /** The cohort resolved, and it is this node alone. Nobody else holds the revision yet. */
120
+ /** The cohort resolved, and it is this node alone. Nobody else holds the revision yet. When this
121
+ * node knows of other members of the cohort that it could not reach (`IKeyNetwork.absentCohortMembers`),
122
+ * they are named in `unconfirmed` and counted in `cohort`: the write is owed to them. With none
123
+ * named, it is a correct, complete one-machine write. */
101
124
  | 'local'
102
125
  /** The cohort did NOT resolve: the lookup threw, or named nobody, or named a single peer that
103
126
  * is not this node. The write is on this node's storage and the network was never consulted.
104
- * Deliberately NOT `local`: a genuine cohort of one is a correct, complete write, whereas this
105
- * is a write whose correct destination is unknown. */
127
+ * Deliberately NOT `local`: a write that lands where its cohort resolved to is in the right
128
+ * place, whereas this is a write whose correct destination is unknown. */
106
129
  | 'unrouted';
107
130
 
108
131
  /** What the answering layer knows about who holds a successful write. Each layer refines it:
@@ -120,15 +143,24 @@ export type WriteDurability = {
120
143
  * it means "accepted the pending record". The two are NOT comparable — a pend confers no
121
144
  * storage durability (see the note in `CoordinatorRepo.pendThroughCluster`). */
122
145
  readonly confirmed: number;
123
- /** Size of the cohort the write ran on. Zero exactly when `quorum` is 'unrouted'. */
146
+ /** Size of the cohort the write ran on. Zero exactly when `quorum` is 'unrouted'. For `local`
147
+ * it is one plus the known members the write ran without, so `local` does not imply 1. */
124
148
  readonly cohort: number;
125
149
  /** Cohort members that did not confirm, by peer-id string. Empty when `quorum` is 'full'.
126
150
  * ABSENT (not empty) when the answering layer could not name the cohort — an empty array
127
- * here means "named the cohort, nobody is missing". */
151
+ * here means "named the cohort, nobody is missing". On a `local` write these are the members
152
+ * this node knows of but could not reach when it wrote alone. */
128
153
  readonly unconfirmed?: readonly string[];
129
154
  /** The cohort the write ran on, by peer-id string. Absent where the answering layer has no
130
155
  * cohort view at all (a bare `StorageRepo` verdict). */
131
156
  readonly cohortPeerIds?: readonly string[];
157
+ /** Peer-id string of the node whose under-replication ledger recorded this write's missing
158
+ * copies — the one node that will fire `BlockDurabilityReachedEvent` for these blocks when they
159
+ * reach full replication. Set only below `full`, and only when every non-torn block's shortfall
160
+ * was recorded. Absent: no node is tracking the shortfall (nothing to push from the coordinator,
161
+ * a ledger fault, no ledger, a producer that predates the field), so no full-replication event
162
+ * should be expected. Per cohort answer: each `otherCohorts` entry carries its own. */
163
+ readonly recordedBy?: string;
132
164
  /** Present only on an action-level result assembled from MORE THAN ONE distinct coordinator
133
165
  * answer: the other answers. Identical answers are collapsed (`mergeDurability`), so a healthy
134
166
  * single-cohort commit has none; one cohort answering differently for two batches appears
@@ -140,6 +172,14 @@ export type WriteDurability = {
140
172
  * These blocks NEVER heal by replication: their transform is gone, and only the writer
141
173
  * re-driving the action puts them back. */
142
174
  readonly torn?: readonly BlockId[];
175
+ /** Action-level: every block this write committed, with the revision it committed at — what a
176
+ * host holds to match full-replication events (`BlockDurabilityReachedEvent`, keyed by block and
177
+ * revision) back to this write. Set by the db-core layer that issued the commit
178
+ * (`withBlockRevs`), never by a coordinator and never on the wire. Excludes `torn` blocks: they
179
+ * hold nothing. Merging two reports keeps, per block, the HIGHER revision — a block written at r
180
+ * and again at r+1 by one write is covered by an event at r+1 or later. A `Record` rather than a
181
+ * `Map` so the type stays plain JSON. */
182
+ readonly blockRevs?: Readonly<Record<BlockId, number>>;
143
183
  };
144
184
 
145
185
  export type PendSuccess = {