@optimystic/db-core 1.12.1 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/i-cluster.d.ts +16 -1
- package/dist/src/cluster/i-cluster.d.ts.map +1 -1
- package/dist/src/cluster/structs.d.ts +60 -6
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js +5 -0
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +5 -3
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/network/durability.d.ts +17 -4
- package/dist/src/network/durability.d.ts.map +1 -1
- package/dist/src/network/durability.js +60 -15
- package/dist/src/network/durability.js.map +1 -1
- package/dist/src/network/i-key-network.d.ts +9 -0
- package/dist/src/network/i-key-network.d.ts.map +1 -1
- package/dist/src/network/struct.d.ts +45 -5
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +23 -14
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +84 -48
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/durability.d.ts +31 -0
- package/dist/src/transaction/durability.d.ts.map +1 -0
- package/dist/src/transaction/durability.js +36 -0
- package/dist/src/transaction/durability.js.map +1 -0
- package/dist/src/transaction/errors.d.ts +14 -6
- package/dist/src/transaction/errors.d.ts.map +1 -1
- package/dist/src/transaction/errors.js +16 -5
- package/dist/src/transaction/errors.js.map +1 -1
- package/dist/src/transaction/index.d.ts +2 -0
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -0
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/session.d.ts +2 -0
- package/dist/src/transaction/session.d.ts.map +1 -1
- package/dist/src/transaction/session.js +4 -2
- package/dist/src/transaction/session.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +9 -0
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/transaction.js.map +1 -1
- package/dist/src/transactor/change-notifier.d.ts +20 -8
- package/dist/src/transactor/change-notifier.d.ts.map +1 -1
- package/dist/src/transactor/change-notifier.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +48 -5
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +98 -22
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/owed-cancels.d.ts +96 -0
- package/dist/src/transactor/owed-cancels.d.ts.map +1 -0
- package/dist/src/transactor/owed-cancels.js +211 -0
- package/dist/src/transactor/owed-cancels.js.map +1 -0
- package/dist/src/transactor/transactor-source.d.ts +5 -4
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +11 -6
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/utility/unref-timer.d.ts +9 -0
- package/dist/src/utility/unref-timer.d.ts.map +1 -0
- package/dist/src/utility/unref-timer.js +15 -0
- package/dist/src/utility/unref-timer.js.map +1 -0
- package/package.json +1 -1
- package/src/cluster/i-cluster.ts +13 -2
- package/src/cluster/structs.ts +62 -6
- package/src/collection/collection.ts +5 -3
- package/src/index.ts +1 -0
- package/src/network/durability.ts +63 -15
- package/src/network/i-key-network.ts +10 -0
- package/src/network/struct.ts +45 -5
- package/src/transaction/coordinator.ts +130 -79
- package/src/transaction/durability.ts +60 -0
- package/src/transaction/errors.ts +18 -3
- package/src/transaction/index.ts +2 -0
- package/src/transaction/session.ts +4 -2
- package/src/transaction/transaction.ts +9 -0
- package/src/transactor/change-notifier.ts +20 -8
- package/src/transactor/network-transactor.ts +120 -29
- package/src/transactor/owed-cancels.ts +252 -0
- package/src/transactor/transactor-source.ts +11 -6
- package/src/utility/unref-timer.ts +15 -0
|
@@ -25,7 +25,8 @@ export function durabilityRank(q: DurabilityQuorum): number {
|
|
|
25
25
|
* Merge per-cohort reports into one action-level report: the weakest becomes the scalar answer and
|
|
26
26
|
* the rest become `otherCohorts`, so a consumer that reads only the scalar fields is reading the
|
|
27
27
|
* binding constraint. `torn` is the union across the inputs — an abandoned block is abandoned
|
|
28
|
-
* whichever cohort it ran on
|
|
28
|
+
* whichever cohort it ran on — and `blockRevs` the union at the higher revision per block, less
|
|
29
|
+
* every torn block. Reports that already carry `otherCohorts` are flattened first, so
|
|
29
30
|
* merging a merge is the same as merging its inputs. Identical reports collapse to one: a commit
|
|
30
31
|
* sends the tail and the sweep as separate batches, so ONE cohort routinely answers twice for one
|
|
31
32
|
* action, and without this a healthy single-cohort commit would list its own cohort under
|
|
@@ -46,39 +47,62 @@ export function mergeDurability(reports: readonly WriteDurability[]): WriteDurab
|
|
|
46
47
|
const ranked = [...flat].sort((a, b) => durabilityRank(a.quorum) - durabilityRank(b.quorum));
|
|
47
48
|
const weakest = ranked[0]!;
|
|
48
49
|
const others = ranked.slice(1);
|
|
49
|
-
// Over the ORIGINAL reports: flattening strips `torn` (
|
|
50
|
-
//
|
|
50
|
+
// Over the ORIGINAL reports: flattening strips `torn` and `blockRevs` (they are action-level),
|
|
51
|
+
// so both are taken before that or a torn input merges to an untorn answer.
|
|
51
52
|
const torn = unionTorn(reports);
|
|
53
|
+
const blockRevs = withoutBlocks(highestBlockRevs(reports), torn);
|
|
52
54
|
return {
|
|
53
55
|
...stripMergedFields(weakest),
|
|
54
56
|
...(others.length === 0 ? {} : { otherCohorts: others.map(stripMergedFields) }),
|
|
55
|
-
...(torn.length === 0 ? {} : { torn })
|
|
57
|
+
...(torn.length === 0 ? {} : { torn }),
|
|
58
|
+
...blockRevsField(blockRevs)
|
|
56
59
|
};
|
|
57
60
|
}
|
|
58
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Name the blocks a write committed, at the revision it committed them at — set where the commit is
|
|
64
|
+
* confirmed, by the db-core layer that issued it. A block already named keeps the higher revision,
|
|
65
|
+
* and a block in `d.torn` is left out: it holds nothing.
|
|
66
|
+
*/
|
|
67
|
+
export function withBlockRevs(d: WriteDurability, blockIds: readonly BlockId[], rev: number): WriteDurability {
|
|
68
|
+
const committed = Object.fromEntries(blockIds.map(blockId => [blockId, rev]));
|
|
69
|
+
const blockRevs = withoutBlocks(highestBlockRevs([d, { blockRevs: committed }]), d.torn ?? []);
|
|
70
|
+
return { ...d, ...blockRevsField(blockRevs) };
|
|
71
|
+
}
|
|
72
|
+
|
|
59
73
|
/**
|
|
60
74
|
* Clamp an action-level report below `full` when the action abandoned blocks. A cohort can hold
|
|
61
75
|
* every block it was asked to commit and the action still be incomplete — the blocks the sweep
|
|
62
76
|
* never reached are on nobody. Never raises the class; a report already below `full` keeps it.
|
|
63
|
-
* Idempotent, and a no-op for an
|
|
77
|
+
* A torn block leaves `blockRevs`, since it was not committed. Idempotent, and a no-op for an
|
|
78
|
+
* empty `torn`.
|
|
64
79
|
*/
|
|
65
80
|
export function withTornBlocks(d: WriteDurability, torn: readonly BlockId[]): WriteDurability {
|
|
66
81
|
if (torn.length === 0) return d;
|
|
67
82
|
const allTorn = Array.from(new Set([...(d.torn ?? []), ...torn]));
|
|
83
|
+
const { blockRevs: _blockRevs, ...rest } = d;
|
|
68
84
|
return {
|
|
69
|
-
...
|
|
85
|
+
...rest,
|
|
70
86
|
quorum: d.quorum === 'full' ? 'majority' : d.quorum,
|
|
71
|
-
torn: allTorn
|
|
87
|
+
torn: allTorn,
|
|
88
|
+
...blockRevsField(withoutBlocks(d.blockRevs ?? {}, allTorn))
|
|
72
89
|
};
|
|
73
90
|
}
|
|
74
91
|
|
|
75
|
-
/**
|
|
76
|
-
|
|
92
|
+
/**
|
|
93
|
+
* The single-node answer: this node holds it and nobody else does yet. `absent` names the cohort
|
|
94
|
+
* members this node knows of that the write ran without — they are part of the cohort the write is
|
|
95
|
+
* owed to, so they count in `cohort` and are listed in `unconfirmed`. With none, it is a one-machine
|
|
96
|
+
* write that is complete. `absent` needs `selfPeerId`: without it the answer cannot name the cohort.
|
|
97
|
+
*/
|
|
98
|
+
export function localDurability(selfPeerId?: string, absent: readonly string[] = []): WriteDurability {
|
|
99
|
+
if (selfPeerId === undefined) return { quorum: 'local', confirmed: 1, cohort: 1 };
|
|
77
100
|
return {
|
|
78
101
|
quorum: 'local',
|
|
79
102
|
confirmed: 1,
|
|
80
|
-
cohort: 1,
|
|
81
|
-
|
|
103
|
+
cohort: 1 + absent.length,
|
|
104
|
+
unconfirmed: [...absent],
|
|
105
|
+
cohortPeerIds: [selfPeerId, ...absent]
|
|
82
106
|
};
|
|
83
107
|
}
|
|
84
108
|
|
|
@@ -96,22 +120,25 @@ function flattenDurability(d: WriteDurability): WriteDurability[] {
|
|
|
96
120
|
return [stripMergedFields(d), ...(d.otherCohorts ?? [])];
|
|
97
121
|
}
|
|
98
122
|
|
|
99
|
-
/** The per-cohort fields only — `otherCohorts` and `
|
|
123
|
+
/** The per-cohort fields only — `otherCohorts`, `torn` and `blockRevs` are action-level and
|
|
124
|
+
* re-derived by the merge. */
|
|
100
125
|
function stripMergedFields(d: WriteDurability): WriteDurability {
|
|
101
|
-
const { otherCohorts: _otherCohorts, torn: _torn, ...cohortReport } = d;
|
|
126
|
+
const { otherCohorts: _otherCohorts, torn: _torn, blockRevs: _blockRevs, ...cohortReport } = d;
|
|
102
127
|
return cohortReport;
|
|
103
128
|
}
|
|
104
129
|
|
|
105
130
|
/**
|
|
106
131
|
* First occurrence of each structurally identical per-cohort report, input order kept. Peer lists
|
|
107
132
|
* compare as SETS: `findCluster` orders a cohort by routing distance to the block's key, so one
|
|
108
|
-
* cohort answering for two blocks lists the same members in two orders.
|
|
133
|
+
* cohort answering for two blocks lists the same members in two orders. `recordedBy` is part of the
|
|
134
|
+
* identity: two otherwise identical answers whose shortfalls two different ledgers recorded are two
|
|
135
|
+
* facts, and a consumer waiting on full-replication events needs to see both.
|
|
109
136
|
*/
|
|
110
137
|
function distinctReports(reports: readonly WriteDurability[]): WriteDurability[] {
|
|
111
138
|
const seen = new Set<string>();
|
|
112
139
|
const sorted = (ids: readonly string[] | undefined): string[] | null => ids === undefined ? null : [...ids].sort();
|
|
113
140
|
return reports.filter(r => {
|
|
114
|
-
const key = JSON.stringify([r.quorum, r.confirmed, r.cohort, sorted(r.unconfirmed), sorted(r.cohortPeerIds)]);
|
|
141
|
+
const key = JSON.stringify([r.quorum, r.confirmed, r.cohort, sorted(r.unconfirmed), sorted(r.cohortPeerIds), r.recordedBy ?? null]);
|
|
115
142
|
if (seen.has(key)) return false;
|
|
116
143
|
seen.add(key);
|
|
117
144
|
return true;
|
|
@@ -121,3 +148,24 @@ function distinctReports(reports: readonly WriteDurability[]): WriteDurability[]
|
|
|
121
148
|
function unionTorn(reports: readonly WriteDurability[]): BlockId[] {
|
|
122
149
|
return Array.from(new Set(reports.flatMap(r => r.torn ?? [])));
|
|
123
150
|
}
|
|
151
|
+
|
|
152
|
+
/** Every block any report names, at the highest revision any of them names it at. */
|
|
153
|
+
function highestBlockRevs(reports: readonly Pick<WriteDurability, 'blockRevs'>[]): Record<BlockId, number> {
|
|
154
|
+
// A Map, not assignment into `{}`: a header block's id is its collection id, which a caller
|
|
155
|
+
// names, and `__proto__` assigned into a plain object sets its prototype instead of a key.
|
|
156
|
+
const highest = new Map<BlockId, number>();
|
|
157
|
+
for (const [blockId, rev] of reports.flatMap(r => Object.entries(r.blockRevs ?? {}))) {
|
|
158
|
+
highest.set(blockId, Math.max(highest.get(blockId) ?? rev, rev));
|
|
159
|
+
}
|
|
160
|
+
return Object.fromEntries(highest);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function withoutBlocks(blockRevs: Readonly<Record<BlockId, number>>, excluded: readonly BlockId[]): Record<BlockId, number> {
|
|
164
|
+
const drop = new Set(excluded);
|
|
165
|
+
return Object.fromEntries(Object.entries(blockRevs).filter(([blockId]) => !drop.has(blockId)));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** The field, or nothing: an empty map is left off, as an empty `torn` is. */
|
|
169
|
+
function blockRevsField(blockRevs: Readonly<Record<BlockId, number>>): Pick<WriteDurability, 'blockRevs'> {
|
|
170
|
+
return Object.keys(blockRevs).length === 0 ? {} : { blockRevs };
|
|
171
|
+
}
|
|
@@ -71,4 +71,14 @@ export type IKeyNetwork = {
|
|
|
71
71
|
* such configuration omit this, which reads as "not awaiting".
|
|
72
72
|
*/
|
|
73
73
|
awaitingBootstrapContact?(): Promise<boolean>;
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Optionally name the peers this node knows serve this network that belong in `key`'s cohort but
|
|
77
|
+
* are missing from the cohort `findCluster` assembles now — typically members that are offline or
|
|
78
|
+
* unreachable. Peer-id strings, nearest the key first, never this node. A write made without them
|
|
79
|
+
* is owed to them, and naming them lets the writer's answer say so (`WriteDurability.unconfirmed`)
|
|
80
|
+
* and lets the copy be pushed to them as soon as they reconnect. Implementations that remember no
|
|
81
|
+
* such peers omit this, which reads as "none known".
|
|
82
|
+
*/
|
|
83
|
+
absentCohortMembers?(key: RoutingKey): Promise<string[]>;
|
|
74
84
|
}
|
package/src/network/struct.ts
CHANGED
|
@@ -82,6 +82,26 @@ export type PendRequest = ActionTransforms & {
|
|
|
82
82
|
* pend named no base, not the primary check.
|
|
83
83
|
*/
|
|
84
84
|
baseRevs?: BlockBaseRevs;
|
|
85
|
+
/**
|
|
86
|
+
* The writer's word that this pend is the action's only one — every block the action will commit
|
|
87
|
+
* is in this one request — and that the action's commit goes to the coordinator this pend goes
|
|
88
|
+
* to. Only then may a coordinator hold back the pend's final delivery to the other members and
|
|
89
|
+
* send it inside the commit's first request (`ClusterCoordinator.mayDeferPendDelivery` in db-p2p):
|
|
90
|
+
* a member that refuses the pend late refuses that one commit, before anything of the action is
|
|
91
|
+
* durable. A send that pends several requests before committing any — several collections under
|
|
92
|
+
* one action id, or one collection split across coordinators — must never carry it: one request's
|
|
93
|
+
* late refusal would surface while another request's commit may already have landed, tearing the
|
|
94
|
+
* action.
|
|
95
|
+
*
|
|
96
|
+
* Set by `TransactorSource.transact` and by `TransactionCoordinator` for a transaction of exactly
|
|
97
|
+
* one collection; `NetworkTransactor.pend` keeps it only on a request carrying the whole pend.
|
|
98
|
+
* `Collection.completeOwnEntry` re-sends one participant of a half-landed multi-collection action
|
|
99
|
+
* through `transact`, and carries it too: that re-send pends and commits its one collection on its
|
|
100
|
+
* own, so its late refusal stops only its own commit, as an on-time refusal would.
|
|
101
|
+
* Omitted rather than `false`, for the same preimage reason as {@link baseRevs}; a coordinator on
|
|
102
|
+
* an older build ignores it, and then delivers at once as it always did.
|
|
103
|
+
*/
|
|
104
|
+
soleReservation?: true;
|
|
85
105
|
};
|
|
86
106
|
|
|
87
107
|
export type BlockActionStatus = ActionBlocks & {
|
|
@@ -97,12 +117,15 @@ export type DurabilityQuorum =
|
|
|
97
117
|
* (`withTornBlocks` clamps `full` down to this; `torn` then names the blocks, and `unconfirmed`
|
|
98
118
|
* may be empty). */
|
|
99
119
|
| 'majority'
|
|
100
|
-
/** The cohort resolved, and it is this node alone. Nobody else holds the revision yet.
|
|
120
|
+
/** The cohort resolved, and it is this node alone. Nobody else holds the revision yet. When this
|
|
121
|
+
* node knows of other members of the cohort that it could not reach (`IKeyNetwork.absentCohortMembers`),
|
|
122
|
+
* they are named in `unconfirmed` and counted in `cohort`: the write is owed to them. With none
|
|
123
|
+
* named, it is a correct, complete one-machine write. */
|
|
101
124
|
| 'local'
|
|
102
125
|
/** The cohort did NOT resolve: the lookup threw, or named nobody, or named a single peer that
|
|
103
126
|
* is not this node. The write is on this node's storage and the network was never consulted.
|
|
104
|
-
* Deliberately NOT `local`: a
|
|
105
|
-
* is a write whose correct destination is unknown. */
|
|
127
|
+
* Deliberately NOT `local`: a write that lands where its cohort resolved to is in the right
|
|
128
|
+
* place, whereas this is a write whose correct destination is unknown. */
|
|
106
129
|
| 'unrouted';
|
|
107
130
|
|
|
108
131
|
/** What the answering layer knows about who holds a successful write. Each layer refines it:
|
|
@@ -120,15 +143,24 @@ export type WriteDurability = {
|
|
|
120
143
|
* it means "accepted the pending record". The two are NOT comparable — a pend confers no
|
|
121
144
|
* storage durability (see the note in `CoordinatorRepo.pendThroughCluster`). */
|
|
122
145
|
readonly confirmed: number;
|
|
123
|
-
/** Size of the cohort the write ran on. Zero exactly when `quorum` is 'unrouted'.
|
|
146
|
+
/** Size of the cohort the write ran on. Zero exactly when `quorum` is 'unrouted'. For `local`
|
|
147
|
+
* it is one plus the known members the write ran without, so `local` does not imply 1. */
|
|
124
148
|
readonly cohort: number;
|
|
125
149
|
/** Cohort members that did not confirm, by peer-id string. Empty when `quorum` is 'full'.
|
|
126
150
|
* ABSENT (not empty) when the answering layer could not name the cohort — an empty array
|
|
127
|
-
* here means "named the cohort, nobody is missing".
|
|
151
|
+
* here means "named the cohort, nobody is missing". On a `local` write these are the members
|
|
152
|
+
* this node knows of but could not reach when it wrote alone. */
|
|
128
153
|
readonly unconfirmed?: readonly string[];
|
|
129
154
|
/** The cohort the write ran on, by peer-id string. Absent where the answering layer has no
|
|
130
155
|
* cohort view at all (a bare `StorageRepo` verdict). */
|
|
131
156
|
readonly cohortPeerIds?: readonly string[];
|
|
157
|
+
/** Peer-id string of the node whose under-replication ledger recorded this write's missing
|
|
158
|
+
* copies — the one node that will fire `BlockDurabilityReachedEvent` for these blocks when they
|
|
159
|
+
* reach full replication. Set only below `full`, and only when every non-torn block's shortfall
|
|
160
|
+
* was recorded. Absent: no node is tracking the shortfall (nothing to push from the coordinator,
|
|
161
|
+
* a ledger fault, no ledger, a producer that predates the field), so no full-replication event
|
|
162
|
+
* should be expected. Per cohort answer: each `otherCohorts` entry carries its own. */
|
|
163
|
+
readonly recordedBy?: string;
|
|
132
164
|
/** Present only on an action-level result assembled from MORE THAN ONE distinct coordinator
|
|
133
165
|
* answer: the other answers. Identical answers are collapsed (`mergeDurability`), so a healthy
|
|
134
166
|
* single-cohort commit has none; one cohort answering differently for two batches appears
|
|
@@ -140,6 +172,14 @@ export type WriteDurability = {
|
|
|
140
172
|
* These blocks NEVER heal by replication: their transform is gone, and only the writer
|
|
141
173
|
* re-driving the action puts them back. */
|
|
142
174
|
readonly torn?: readonly BlockId[];
|
|
175
|
+
/** Action-level: every block this write committed, with the revision it committed at — what a
|
|
176
|
+
* host holds to match full-replication events (`BlockDurabilityReachedEvent`, keyed by block and
|
|
177
|
+
* revision) back to this write. Set by the db-core layer that issued the commit
|
|
178
|
+
* (`withBlockRevs`), never by a coordinator and never on the wire. Excludes `torn` blocks: they
|
|
179
|
+
* hold nothing. Merging two reports keeps, per block, the HIGHER revision — a block written at r
|
|
180
|
+
* and again at r+1 by one write is covered by an event at r+1 or later. A `Record` rather than a
|
|
181
|
+
* `Map` so the type stays plain JSON. */
|
|
182
|
+
readonly blockRevs?: Readonly<Record<BlockId, number>>;
|
|
143
183
|
};
|
|
144
184
|
|
|
145
185
|
export type PendSuccess = {
|