@optimystic/db-core 1.25.0 → 1.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +10 -0
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cohort-topic/addressing.d.ts +26 -2
- package/dist/src/cohort-topic/addressing.d.ts.map +1 -1
- package/dist/src/cohort-topic/addressing.js +35 -4
- package/dist/src/cohort-topic/addressing.js.map +1 -1
- package/dist/src/cohort-topic/member-engine.d.ts +17 -10
- package/dist/src/cohort-topic/member-engine.d.ts.map +1 -1
- package/dist/src/cohort-topic/member-engine.js +20 -18
- package/dist/src/cohort-topic/member-engine.js.map +1 -1
- package/dist/src/cohort-topic/registration/renewal.d.ts +21 -14
- package/dist/src/cohort-topic/registration/renewal.d.ts.map +1 -1
- package/dist/src/cohort-topic/registration/renewal.js +38 -28
- package/dist/src/cohort-topic/registration/renewal.js.map +1 -1
- package/dist/src/collection/collection.d.ts +123 -34
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +231 -74
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/segments.d.ts +71 -14
- package/dist/src/collection/segments.d.ts.map +1 -1
- package/dist/src/collection/segments.js +94 -14
- package/dist/src/collection/segments.js.map +1 -1
- package/dist/src/network/i-key-network.d.ts +28 -0
- package/dist/src/network/i-key-network.d.ts.map +1 -1
- package/dist/src/network/i-repo.d.ts +16 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/offline/queue.d.ts +13 -0
- package/dist/src/offline/queue.d.ts.map +1 -1
- package/dist/src/offline/queue.js +22 -3
- package/dist/src/offline/queue.js.map +1 -1
- package/dist/src/reactivity/topic-anchor.d.ts.map +1 -1
- package/dist/src/reactivity/topic-anchor.js +20 -2
- package/dist/src/reactivity/topic-anchor.js.map +1 -1
- package/dist/src/transaction/coordinator.d.ts +1 -1
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +15 -16
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +16 -3
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +22 -5
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transform/atomic.d.ts +8 -2
- package/dist/src/transform/atomic.d.ts.map +1 -1
- package/dist/src/transform/atomic.js +10 -4
- package/dist/src/transform/atomic.js.map +1 -1
- package/dist/src/transform/helpers.d.ts +29 -3
- package/dist/src/transform/helpers.d.ts.map +1 -1
- package/dist/src/transform/helpers.js +62 -3
- package/dist/src/transform/helpers.js.map +1 -1
- package/dist/src/transform/tracker.d.ts +57 -5
- package/dist/src/transform/tracker.d.ts.map +1 -1
- package/dist/src/transform/tracker.js +111 -11
- package/dist/src/transform/tracker.js.map +1 -1
- package/dist/src/utility/expiring-lru-map.d.ts +16 -0
- package/dist/src/utility/expiring-lru-map.d.ts.map +1 -0
- package/dist/src/utility/expiring-lru-map.js +39 -0
- package/dist/src/utility/expiring-lru-map.js.map +1 -0
- package/package.json +1 -1
- package/src/cluster/structs.ts +10 -0
- package/src/cohort-topic/addressing.ts +42 -4
- package/src/cohort-topic/member-engine.ts +35 -28
- package/src/cohort-topic/registration/renewal.ts +62 -42
- package/src/collection/collection.ts +3462 -3272
- package/src/collection/segments.ts +140 -22
- package/src/network/i-key-network.ts +30 -0
- package/src/network/i-repo.ts +16 -0
- package/src/offline/queue.ts +23 -3
- package/src/reactivity/topic-anchor.ts +23 -3
- package/src/transaction/coordinator.ts +15 -16
- package/src/transactor/network-transactor.ts +24 -6
- package/src/transform/atomic.ts +12 -5
- package/src/transform/helpers.ts +61 -3
- package/src/transform/tracker.ts +110 -12
- package/src/utility/expiring-lru-map.ts +42 -0
|
@@ -1,11 +1,33 @@
|
|
|
1
|
-
import type { Action } from "../index.js";
|
|
1
|
+
import type { Action, Transforms } from "../index.js";
|
|
2
2
|
import type { AlreadyAppliedTally } from "./action.js";
|
|
3
|
+
import { addTallies, ZeroTally } from "./action.js";
|
|
3
4
|
import type { ActionRefusedError } from "./struct.js";
|
|
5
|
+
import { emptyTransforms, foldTransforms } from "../transform/helpers.js";
|
|
4
6
|
|
|
5
7
|
/** A queue entry's id, opaque to the collection: the offline-sync queue (docs/offline-sync.md)
|
|
6
8
|
* mints one per locally committed transaction and seals that transaction's actions under it. */
|
|
7
9
|
export type EntryId = string;
|
|
8
10
|
|
|
11
|
+
/** One action of the open segment with what staging it produced: the transforms its handler staged
|
|
12
|
+
* (`delta`) and what it accepted as already applied (`applied`), both as `Collection.stageInto`
|
|
13
|
+
* returned them. Either is absent when the action's staging is not on record — an action put back
|
|
14
|
+
* by a conflict filter's replacement or by an unseal, or one a replay has not reached — and a
|
|
15
|
+
* segment sealed over such a unit carries no delta, which every reader takes as "stage it again".
|
|
16
|
+
* An `act` call over several actions stages them as one unit: its delta and tally are recorded on
|
|
17
|
+
* the group's LAST action, the earlier ones recording an empty delta and the zero tally, so folding
|
|
18
|
+
* the units in order gives the group's effect at the point the group completed; nothing slices
|
|
19
|
+
* inside a group. Immutable once made, so a snapshot may hold the same unit. */
|
|
20
|
+
export type StagedUnit<TAction> = {
|
|
21
|
+
readonly action: Action<TAction>;
|
|
22
|
+
readonly delta?: Transforms;
|
|
23
|
+
readonly applied?: AlreadyAppliedTally;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/** Units for `actions` whose staging is not on record. */
|
|
27
|
+
export function unitsOf<TAction>(actions: readonly Action<TAction>[]): StagedUnit<TAction>[] {
|
|
28
|
+
return actions.map(action => ({ action }));
|
|
29
|
+
}
|
|
30
|
+
|
|
9
31
|
/** The actions one queue entry staged into a collection, closed off from later staging. */
|
|
10
32
|
export type SealedSegment<TAction> = {
|
|
11
33
|
readonly entryId: EntryId;
|
|
@@ -16,17 +38,47 @@ export type SealedSegment<TAction> = {
|
|
|
16
38
|
/** The refusal recorded for this segment, once a replay refused it. A refused segment is excluded
|
|
17
39
|
* from every later replay until it is dropped. */
|
|
18
40
|
refused?: ActionRefusedError;
|
|
41
|
+
/** What the actions staged into the LIVE tracker, the last time they were staged there: folded
|
|
42
|
+
* from the open units at the seal, or captured again by a replay (`Collection.replayActions`),
|
|
43
|
+
* so it names the ids the live tracker holds for the segment's blocks. A head commit builds its
|
|
44
|
+
* attempt from it and, once that attempt commits, takes the head out of the live tracker without
|
|
45
|
+
* running a handler (`Collection.headTrackerFor`, `Collection.subtractCommittedHead`). `undefined`
|
|
46
|
+
* when not on record — a segment put back by `Collection.restoreSealed` or an unseal before its
|
|
47
|
+
* next replay, or one sealed over a unit with no delta — and every reader then stages the actions
|
|
48
|
+
* again, which is slower and never wrong. Never changed in place: the attempt copies it, the
|
|
49
|
+
* subtraction copies what it re-folds, and a snapshot's read view may share its values.
|
|
50
|
+
*
|
|
51
|
+
* NOTE: memory — while the segment is queued the content it staged is held about twice, once
|
|
52
|
+
* here and once folded into the live tracker (a block that later segments also write is held once
|
|
53
|
+
* more per such segment's delta). Proportional to what is queued, and reclaimed with the segment. */
|
|
54
|
+
delta?: Transforms;
|
|
19
55
|
/** What the segment's actions accepted as already applied the last time they were staged over the
|
|
20
|
-
* committed state —
|
|
21
|
-
* (`Collection.
|
|
22
|
-
*
|
|
23
|
-
* over. Set directly, with no version bump:
|
|
24
|
-
*
|
|
25
|
-
* the segment
|
|
26
|
-
* since a refused segment is
|
|
56
|
+
* committed state — at the seal, from the open units' tallies, or by a replay
|
|
57
|
+
* (`Collection.replayActions`) or a head commit's rebuild (`Collection.buildHeadTracker`), all of
|
|
58
|
+
* which stage over the state the collection reads at, so when the head commit folds this is the
|
|
59
|
+
* tally of a staging over the state the write committed over. Set directly, with no version bump:
|
|
60
|
+
* it is not a change to the sealed list. Reported to the head commit's `onCommittedHead` hook
|
|
61
|
+
* when the segment is dropped as committed. Absent only before the segment's first staging — a
|
|
62
|
+
* refused staging leaves the previous value, which nothing reads, since a refused segment is
|
|
63
|
+
* never committed. */
|
|
27
64
|
applied?: AlreadyAppliedTally;
|
|
28
65
|
};
|
|
29
66
|
|
|
67
|
+
/** The open units' staging folded into one — the delta and tally a segment sealed over them
|
|
68
|
+
* carries — or neither, when any unit's staging is not on record. */
|
|
69
|
+
function foldUnits<TAction>(units: readonly StagedUnit<TAction>[]): { delta?: Transforms; applied?: AlreadyAppliedTally } {
|
|
70
|
+
const delta = emptyTransforms();
|
|
71
|
+
let applied = ZeroTally;
|
|
72
|
+
for (const unit of units) {
|
|
73
|
+
if (unit.delta === undefined || unit.applied === undefined) {
|
|
74
|
+
return {};
|
|
75
|
+
}
|
|
76
|
+
foldTransforms(delta, unit.delta);
|
|
77
|
+
applied = addTallies(applied, unit.applied);
|
|
78
|
+
}
|
|
79
|
+
return { delta, applied };
|
|
80
|
+
}
|
|
81
|
+
|
|
30
82
|
/**
|
|
31
83
|
* A collection's staged actions, in two parts: the sealed segments, one per queued entry that wrote
|
|
32
84
|
* the collection, oldest first, and the open segment the current transaction is staging into
|
|
@@ -38,26 +90,66 @@ export class SegmentList<TAction> {
|
|
|
38
90
|
* through a method that bumps `version`; read it through {@link replayable}, {@link find},
|
|
39
91
|
* {@link head} and {@link entryIds}. */
|
|
40
92
|
private readonly sealed: SealedSegment<TAction>[] = [];
|
|
41
|
-
/** The
|
|
42
|
-
|
|
93
|
+
/** The same segments by owner, so a lookup costs the same at any queue depth. Every method that
|
|
94
|
+
* changes {@link sealed} changes this too. */
|
|
95
|
+
private readonly byEntry = new Map<EntryId, SealedSegment<TAction>>();
|
|
96
|
+
/** The open segment: what the current transaction has staged, one unit per action
|
|
97
|
+
* ({@link StagedUnit}). Private so that it is replaced through {@link setOpen} alone and grown
|
|
98
|
+
* through {@link pushOpen} alone, each unit carrying what its staging produced. */
|
|
99
|
+
private units: StagedUnit<TAction>[] = [];
|
|
43
100
|
/** Bumped on every change to the sealed list — a seal, an unseal, an insert, a drop, a refusal
|
|
44
101
|
* recorded. Captured by a snapshot so a restore can tell whether the sealed segments it was taken
|
|
45
102
|
* over are still the ones in place (ticket `offline-sync-open-segment-snapshot`). Read and written
|
|
46
103
|
* only under the collection latch or from synchronous callers. */
|
|
47
104
|
version = 0;
|
|
48
105
|
|
|
49
|
-
/**
|
|
50
|
-
*
|
|
51
|
-
|
|
106
|
+
/** The open segment's units, in staging order: the array the list holds, never to be changed by
|
|
107
|
+
* a caller. */
|
|
108
|
+
openUnits(): readonly StagedUnit<TAction>[] {
|
|
109
|
+
return this.units;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** The open segment's actions, in staging order, as a new array. */
|
|
113
|
+
openActions(): Action<TAction>[] {
|
|
114
|
+
return this.units.map(unit => unit.action);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** How many actions the open segment holds. */
|
|
118
|
+
openCount(): number {
|
|
119
|
+
return this.units.length;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Replace the open segment: the one way it is set. A caller putting actions back whose staging
|
|
123
|
+
* it does not have records them through {@link unitsOf}, and the next replay records it. */
|
|
124
|
+
setOpen(units: StagedUnit<TAction>[]): void {
|
|
125
|
+
this.units = units;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Append the actions one `act` call staged as one unit, with the delta and tally that staging
|
|
129
|
+
* produced, recorded as {@link StagedUnit} describes for a group. */
|
|
130
|
+
pushOpen(actions: readonly Action<TAction>[], delta: Transforms, applied: AlreadyAppliedTally): void {
|
|
131
|
+
const last = actions.length - 1;
|
|
132
|
+
actions.forEach((action, index) => {
|
|
133
|
+
this.units.push(index === last ? { action, delta, applied } : { action, delta: emptyTransforms(), applied: ZeroTally });
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Close the open segment into a sealed segment owned by `entryId`, carrying the open units'
|
|
138
|
+
* deltas folded into one and their tallies summed ({@link SealedSegment.delta}, `applied`) — or
|
|
139
|
+
* neither, when a unit's staging is not on record. Returns `false` and changes nothing — the
|
|
140
|
+
* version included — when the open segment is empty: an entry that staged nothing here owns no
|
|
141
|
+
* segment here. Throws when `entryId` is already sealed. */
|
|
52
142
|
seal(entryId: EntryId): boolean {
|
|
53
143
|
if (this.find(entryId) !== undefined) {
|
|
54
144
|
throw new Error(`entry ${entryId} is already sealed in this collection`);
|
|
55
145
|
}
|
|
56
|
-
if (this.
|
|
146
|
+
if (this.units.length === 0) {
|
|
57
147
|
return false;
|
|
58
148
|
}
|
|
59
|
-
|
|
60
|
-
this.
|
|
149
|
+
const segment: SealedSegment<TAction> = { entryId, actions: this.units.map(unit => unit.action), ...foldUnits(this.units) };
|
|
150
|
+
this.sealed.push(segment);
|
|
151
|
+
this.byEntry.set(entryId, segment);
|
|
152
|
+
this.units = [];
|
|
61
153
|
this.version++;
|
|
62
154
|
return true;
|
|
63
155
|
}
|
|
@@ -68,17 +160,20 @@ export class SegmentList<TAction> {
|
|
|
68
160
|
* that one call site, which has just sealed and staged nothing since — and throws otherwise, since
|
|
69
161
|
* putting an older segment's actions behind a newer one's would reorder what was staged. Bumps
|
|
70
162
|
* `version`: the sealed list changed, even though the tracker did not (the transforms are already
|
|
71
|
-
* what they were).
|
|
163
|
+
* what they were). The units go back with no staging on record: the segment holds their deltas
|
|
164
|
+
* folded into one, and the enqueue that unseals is failing, so the rollback that follows it
|
|
165
|
+
* replays and records them again. */
|
|
72
166
|
unseal(entryId: EntryId): void {
|
|
73
167
|
const newest = this.sealed[this.sealed.length - 1];
|
|
74
168
|
if (newest === undefined || newest.entryId !== entryId) {
|
|
75
169
|
throw new Error(`entry ${entryId} is not the newest sealed segment (${newest?.entryId ?? 'nothing is sealed'}); only the newest can be unsealed`);
|
|
76
170
|
}
|
|
77
|
-
if (this.
|
|
171
|
+
if (this.units.length > 0) {
|
|
78
172
|
throw new Error(`entry ${entryId} cannot be unsealed over a non-empty open segment`);
|
|
79
173
|
}
|
|
80
174
|
this.sealed.pop();
|
|
81
|
-
this.
|
|
175
|
+
this.byEntry.delete(entryId);
|
|
176
|
+
this.units = unitsOf(newest.actions);
|
|
82
177
|
this.version++;
|
|
83
178
|
}
|
|
84
179
|
|
|
@@ -94,7 +189,9 @@ export class SegmentList<TAction> {
|
|
|
94
189
|
if (index < 0) {
|
|
95
190
|
throw new Error(`entry ${before} is not sealed in this collection; cannot insert ${entryId} before it`);
|
|
96
191
|
}
|
|
97
|
-
|
|
192
|
+
const segment: SealedSegment<TAction> = { entryId, actions: [...actions] };
|
|
193
|
+
this.sealed.splice(index, 0, segment);
|
|
194
|
+
this.byEntry.set(entryId, segment);
|
|
98
195
|
this.version++;
|
|
99
196
|
}
|
|
100
197
|
|
|
@@ -105,6 +202,7 @@ export class SegmentList<TAction> {
|
|
|
105
202
|
throw new Error(`entry ${entryId} is not sealed in this collection`);
|
|
106
203
|
}
|
|
107
204
|
const [dropped] = this.sealed.splice(index, 1);
|
|
205
|
+
this.byEntry.delete(entryId);
|
|
108
206
|
this.version++;
|
|
109
207
|
return dropped!;
|
|
110
208
|
}
|
|
@@ -120,7 +218,7 @@ export class SegmentList<TAction> {
|
|
|
120
218
|
}
|
|
121
219
|
|
|
122
220
|
find(entryId: EntryId): SealedSegment<TAction> | undefined {
|
|
123
|
-
return this.
|
|
221
|
+
return this.byEntry.get(entryId);
|
|
124
222
|
}
|
|
125
223
|
|
|
126
224
|
/** The oldest sealed segment, if any. */
|
|
@@ -144,6 +242,26 @@ export class SegmentList<TAction> {
|
|
|
144
242
|
|
|
145
243
|
/** How many actions a replay re-stages: the replayable sealed segments' plus the open segment's. */
|
|
146
244
|
replayableCount(): number {
|
|
147
|
-
return this.replayable().reduce((count, segment) => count + segment.actions.length, this.
|
|
245
|
+
return this.replayable().reduce((count, segment) => count + segment.actions.length, this.units.length);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** The deltas of everything a replay would re-stage, in staging order — each replayable sealed
|
|
249
|
+
* segment's, then each open unit's — or `undefined` when any is not on record. What a committed
|
|
250
|
+
* head is taken out of the live tracker against (`Collection.subtractCommittedHead`). */
|
|
251
|
+
stagedDeltas(): Transforms[] | undefined {
|
|
252
|
+
const deltas: Transforms[] = [];
|
|
253
|
+
for (const segment of this.replayable()) {
|
|
254
|
+
if (segment.delta === undefined) {
|
|
255
|
+
return undefined;
|
|
256
|
+
}
|
|
257
|
+
deltas.push(segment.delta);
|
|
258
|
+
}
|
|
259
|
+
for (const unit of this.units) {
|
|
260
|
+
if (unit.delta === undefined) {
|
|
261
|
+
return undefined;
|
|
262
|
+
}
|
|
263
|
+
deltas.push(unit.delta);
|
|
264
|
+
}
|
|
265
|
+
return deltas;
|
|
148
266
|
}
|
|
149
267
|
}
|
|
@@ -31,6 +31,22 @@ export type FindCoordinatorOptions = {
|
|
|
31
31
|
};
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* What {@link IKeyNetwork.connectedCohort} reports for a key: a cohort that is the whole serving
|
|
36
|
+
* network this node knows of, this node among it and every other member connected to it.
|
|
37
|
+
*
|
|
38
|
+
* A **connection run** is the span during which a remote peer has at least one open connection to
|
|
39
|
+
* this node; it ends when the peer's last connection closes, and the next connection starts a new
|
|
40
|
+
* run with a new id. So two reports with equal `members` and equal `runs` prove that no member's
|
|
41
|
+
* connection to this node broke between them.
|
|
42
|
+
*/
|
|
43
|
+
export type ConnectedCohort = {
|
|
44
|
+
/** The cohort `findCluster` assembles for the key now, as peer-id strings, this node included, sorted ascending. */
|
|
45
|
+
readonly members: readonly string[];
|
|
46
|
+
/** Per member other than this node: the id of its current connection run. Ids are never reused within a process. */
|
|
47
|
+
readonly runs: Readonly<Record<string, number>>;
|
|
48
|
+
};
|
|
49
|
+
|
|
34
50
|
/**
|
|
35
51
|
* Maps a block to the peers responsible for it.
|
|
36
52
|
*
|
|
@@ -81,4 +97,18 @@ export type IKeyNetwork = {
|
|
|
81
97
|
* such peers omit this, which reads as "none known".
|
|
82
98
|
*/
|
|
83
99
|
absentCohortMembers?(key: RoutingKey): Promise<string[]>;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Optionally report, for `key`, that the cohort `findCluster` would assemble now is every machine
|
|
103
|
+
* that could coordinate a commit to the key's block as far as this node can know — the whole
|
|
104
|
+
* serving network it knows of, this node included — and that every other member is connected to
|
|
105
|
+
* this node, with the connection run each is on (see {@link ConnectedCohort}). Answers from what
|
|
106
|
+
* the node already holds, without dialing or opening a stream.
|
|
107
|
+
*
|
|
108
|
+
* `undefined` means no such guarantee: the cohort is a selection from a wider network, a peer is
|
|
109
|
+
* not yet identified or is remembered but absent, a member is not connected, or the
|
|
110
|
+
* implementation cannot judge. Implementations that track no connections omit this, which reads
|
|
111
|
+
* the same, so a caller then behaves as if the method did not exist.
|
|
112
|
+
*/
|
|
113
|
+
connectedCohort?(key: RoutingKey): Promise<ConnectedCohort | undefined>;
|
|
84
114
|
}
|
package/src/network/i-repo.ts
CHANGED
|
@@ -36,6 +36,22 @@ export type MessageOptions = {
|
|
|
36
36
|
* peer reports what it holds itself, never what its cohort says. Never sent.
|
|
37
37
|
*/
|
|
38
38
|
skipClusterFetch?: boolean;
|
|
39
|
+
/**
|
|
40
|
+
* The writer of this pend runs in the same process as the storage it is sent to. Set by
|
|
41
|
+
* `NetworkTransactor.pend` and `pendMany`, on a pend sent to the repo of the transactor's own
|
|
42
|
+
* `localPeerId`, and by a transactor that wraps a `StorageRepo` directly in its own process (the
|
|
43
|
+
* Quereus plugin's `local` transactor), and never sent: the repo protocol copies `expiration` alone onto the wire, and
|
|
44
|
+
* the service that receives a remote pend builds its options from the message, so a remote writer
|
|
45
|
+
* cannot claim it. Storage stamps the pending records of such a pend — when it applies them alone,
|
|
46
|
+
* with no consensus — with the identity of the node's current run (`StorageRepo.runId`), and a
|
|
47
|
+
* later run judges a record so stamped abandoned: its writer was a transactor in a process of this
|
|
48
|
+
* node that no longer exists, so nothing can ever commit or cancel it (`isAbandonedClaim` in
|
|
49
|
+
* `packages/db-p2p/src/storage/pending-claim.ts`). The mark is positive on purpose, like
|
|
50
|
+
* {@link asker}: a path that forgets it leaves the record standing, as every record stood before
|
|
51
|
+
* the mark existed, whereas a forgotten negative mark would have a live remote writer's record
|
|
52
|
+
* judged dead.
|
|
53
|
+
*/
|
|
54
|
+
localWriter?: true;
|
|
39
55
|
}
|
|
40
56
|
|
|
41
57
|
export type RepoCommitRequest = {
|
package/src/offline/queue.ts
CHANGED
|
@@ -462,6 +462,26 @@ export class OfflineQueue {
|
|
|
462
462
|
return this.list.map(entry => snapshot(entry));
|
|
463
463
|
}
|
|
464
464
|
|
|
465
|
+
/** One queued entry as {@link entries} lists it, with how many are queued ahead of it and whether
|
|
466
|
+
* it is the newest; `undefined` for an id not queued. For a caller handling one entry's status,
|
|
467
|
+
* which {@link entries}' copy of the whole list would make grow with the queue: constant time for
|
|
468
|
+
* the newest entry, as every enqueued one is, and a scan only for one a {@link restore} inserted
|
|
469
|
+
* ahead of others. */
|
|
470
|
+
queuedEntry(entryId: EntryId): { readonly entry: QueuedEntry; readonly ahead: number; readonly newest: boolean } | undefined {
|
|
471
|
+
const entry = this.byId.get(entryId);
|
|
472
|
+
if (entry === undefined) return undefined;
|
|
473
|
+
const last = this.list.length - 1;
|
|
474
|
+
const newest = this.list[last] === entry;
|
|
475
|
+
return { entry: snapshot(entry), ahead: newest ? last : this.list.indexOf(entry), newest };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/** Whether a collection has recorded a refusal of a segment of the queued entry `entryId`: it is
|
|
479
|
+
* about to fail. `false` for an id not queued. */
|
|
480
|
+
segmentRefused(entryId: EntryId): boolean {
|
|
481
|
+
const entry = this.byId.get(entryId);
|
|
482
|
+
return entry !== undefined && this.recordedRefusal(entry) !== undefined;
|
|
483
|
+
}
|
|
484
|
+
|
|
465
485
|
/** The current status of a queued entry, or `undefined` for an id not queued — never queued, or
|
|
466
486
|
* already committed or failed (a settled entry leaves the queue with its terminal status event). */
|
|
467
487
|
status(entryId: EntryId): EntryStatus | undefined {
|
|
@@ -697,7 +717,7 @@ export class OfflineQueue {
|
|
|
697
717
|
/** The collections in which `entry` is still a sealed segment: every collection it wrote, until a
|
|
698
718
|
* partial commit consumes some. */
|
|
699
719
|
private remainingCollections(entry: Entry): Collection<any>[] {
|
|
700
|
-
return entry.collections.filter(collection => collection.
|
|
720
|
+
return entry.collections.filter(collection => collection.holdsSealed(entry.id));
|
|
701
721
|
}
|
|
702
722
|
|
|
703
723
|
/** The gate, asked with each of the head's remaining collections' header id and known log tail id.
|
|
@@ -1151,7 +1171,7 @@ export class OfflineQueue {
|
|
|
1151
1171
|
this.claimRemoval(entry);
|
|
1152
1172
|
this.pendingRefusals.delete(entry.id);
|
|
1153
1173
|
for (const collection of entry.collections) {
|
|
1154
|
-
if (outcomes.get(collection.id)?.kind !== 'staged' || !collection.
|
|
1174
|
+
if (outcomes.get(collection.id)?.kind !== 'staged' || !collection.holdsSealed(entry.id)) continue;
|
|
1155
1175
|
try {
|
|
1156
1176
|
await collection.dropSealed(entry.id);
|
|
1157
1177
|
} catch (err) {
|
|
@@ -1237,7 +1257,7 @@ export class OfflineQueue {
|
|
|
1237
1257
|
this.removing = entry;
|
|
1238
1258
|
try {
|
|
1239
1259
|
for (const collection of entry.collections) {
|
|
1240
|
-
if (!collection.
|
|
1260
|
+
if (!collection.holdsSealed(entry.id)) continue;
|
|
1241
1261
|
changedCollections.push(collection.id);
|
|
1242
1262
|
await collection.dropSealed(entry.id);
|
|
1243
1263
|
}
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
*/
|
|
34
34
|
|
|
35
35
|
import { createRingHash } from "../cohort-topic/ring-hash.js";
|
|
36
|
-
import { createTierAddressing } from "../cohort-topic/addressing.js";
|
|
36
|
+
import { createTierAddressing, type TierAddressing } from "../cohort-topic/addressing.js";
|
|
37
37
|
import type { IRingHash, RingCoord } from "../cohort-topic/ports.js";
|
|
38
38
|
|
|
39
39
|
/** Domain-separation suffix mixed into the reactivity topic. */
|
|
@@ -63,6 +63,26 @@ export function reactivityCollectionTopicId(collectionId: Uint8Array, hash: IRin
|
|
|
63
63
|
* subscriber walk's root step, the forwarder's gossip and recover reads — must start from this one value.
|
|
64
64
|
* It is the one coordinate of a collection's tree that changes when the tail rotates.
|
|
65
65
|
*/
|
|
66
|
-
export function reactivityRootCoord(tailId: Uint8Array, hash
|
|
67
|
-
return
|
|
66
|
+
export function reactivityRootCoord(tailId: Uint8Array, hash?: IRingHash): RingCoord {
|
|
67
|
+
return addressingFor(hash).rootCoord(tailId);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* One addressing per ring hash, so its root-coordinate cache survives across calls (GitHub #57). Keyed by
|
|
72
|
+
* the hash object, weakly, so a caller's hash and its cache go together; never shared across hashes, since a
|
|
73
|
+
* coordinate computed under one ring width is wrong under another.
|
|
74
|
+
*/
|
|
75
|
+
const addressings = new WeakMap<IRingHash, TierAddressing>();
|
|
76
|
+
let defaultAddressing: TierAddressing | undefined;
|
|
77
|
+
|
|
78
|
+
function addressingFor(hash: IRingHash | undefined): TierAddressing {
|
|
79
|
+
if (hash === undefined) {
|
|
80
|
+
return defaultAddressing ??= createTierAddressing(createRingHash());
|
|
81
|
+
}
|
|
82
|
+
let addressing = addressings.get(hash);
|
|
83
|
+
if (addressing === undefined) {
|
|
84
|
+
addressing = createTierAddressing(hash);
|
|
85
|
+
addressings.set(hash, addressing);
|
|
86
|
+
}
|
|
87
|
+
return addressing;
|
|
68
88
|
}
|
|
@@ -692,7 +692,7 @@ export class TransactionCoordinator {
|
|
|
692
692
|
* this call and the staging it guards.
|
|
693
693
|
*
|
|
694
694
|
* NOTE: this runs per applyActions call rather than once per transaction. Per-call cost for an
|
|
695
|
-
* already-captured collection is one Map.has; snapshotPending (
|
|
695
|
+
* already-captured collection is one Map.has; snapshotPending (one map entry per staged block)
|
|
696
696
|
* still runs at most once per collection per stamp. Unmeasured. If the registered-collection
|
|
697
697
|
* count ever grows large enough for the scan itself to matter, track a per-stamp "collections
|
|
698
698
|
* map size at last capture" rather than dropping the reconcile.
|
|
@@ -1322,13 +1322,12 @@ export class TransactionCoordinator {
|
|
|
1322
1322
|
// landing it is (reportSaved), and the next attempt commits only what is still
|
|
1323
1323
|
// staged (stagedCollections) — the durable half is never re-logged. The stamp stays
|
|
1324
1324
|
// open for the retry; reportSaved or the success path releases it.
|
|
1325
|
-
// The committed siblings keep their in-flight mark until the cycle ends,
|
|
1326
|
-
// blanket refresh between attempts
|
|
1327
|
-
//
|
|
1328
|
-
//
|
|
1329
|
-
//
|
|
1330
|
-
//
|
|
1331
|
-
// skips committed participants) — the refresh is a reader's, at a cost of one walk.
|
|
1325
|
+
// The committed siblings keep their in-flight mark until the cycle ends, but the
|
|
1326
|
+
// blanket refresh between attempts cannot find their own entry: recordCommitted
|
|
1327
|
+
// above advanced each one's held revision to the entry's own, and Log.getFrom
|
|
1328
|
+
// returns only entries ABOVE the held revision. Collection.consumeOwnEntry is
|
|
1329
|
+
// therefore unreachable for a sibling committed here, and this attempt was not
|
|
1330
|
+
// retained for it (the loop above skips committed participants) — the refresh is a reader's.
|
|
1332
1331
|
for (const [collectionId, durability] of committed) {
|
|
1333
1332
|
recordCollectionDurability(cycle.saved, collectionId, durability);
|
|
1334
1333
|
}
|
|
@@ -1569,8 +1568,8 @@ export class TransactionCoordinator {
|
|
|
1569
1568
|
|
|
1570
1569
|
// Restore each captured collection — transforms AND pending queue together, so the
|
|
1571
1570
|
// rolled-back stamp's actions leave the queue instead of being written into the next
|
|
1572
|
-
// transaction's durable log entry. `restorePending`
|
|
1573
|
-
// so no
|
|
1571
|
+
// transaction's durable log entry. `restorePending` takes its own maps over the captured
|
|
1572
|
+
// values, so no copy here. Restoring through the CAPTURED instance (rather than
|
|
1574
1573
|
// re-looking-up by id) is what keeps a mid-transaction instance swap under one id from
|
|
1575
1574
|
// pushing the old instance's staged state onto the new one.
|
|
1576
1575
|
// NOTE: this also discards actions staged OUTSIDE any tracked stamp (the Tree.stage /
|
|
@@ -1730,16 +1729,16 @@ export class TransactionCoordinator {
|
|
|
1730
1729
|
// between this loop and the applyActions() call below, or an interleaved stage would be
|
|
1731
1730
|
// captured inside the snapshot. Because the whole loop runs before ANY staging, an engine
|
|
1732
1731
|
// naming one collection in two batches yields the state before EITHER batch whichever
|
|
1733
|
-
// batch wins the key — so the dedupe is a COST guard (snapshotPending
|
|
1734
|
-
//
|
|
1732
|
+
// batch wins the key — so the dedupe is a COST guard (snapshotPending copies the transforms'
|
|
1733
|
+
// maps and structuredClones the action context), not a correctness one. An
|
|
1735
1734
|
// unregistered collection is skipped; applyActions throws on it moments later and the
|
|
1736
1735
|
// catch below converts that to a failure result, so the map is never read.
|
|
1737
1736
|
//
|
|
1738
1737
|
// NOTE: this runs unconditionally, so the overwhelmingly common all-succeed execute() pays
|
|
1739
|
-
// one
|
|
1740
|
-
//
|
|
1741
|
-
//
|
|
1742
|
-
//
|
|
1738
|
+
// one capture per participant to serve a rare branch. Unmeasured, and symmetric with
|
|
1739
|
+
// commitOnceLatched, which pays the same on every commit. If execute() ever shows up hot in a
|
|
1740
|
+
// profile, the remedy is the one the `NOTE:` at `Tracker.shareTransforms` names, not dropping
|
|
1741
|
+
// the restore.
|
|
1743
1742
|
const preStageSnapshots = new Map<CollectionId, CollectionSnapshot<any>>();
|
|
1744
1743
|
for (const { collectionId } of result.actions) {
|
|
1745
1744
|
if (preStageSnapshots.has(collectionId)) continue;
|
|
@@ -5,7 +5,7 @@ import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struc
|
|
|
5
5
|
import { mergeDurability, withTornBlocks } from "../network/durability.js";
|
|
6
6
|
import { judgeCohortLineage, type CohortLineage, type MemberLineage } from "../network/lineage.js";
|
|
7
7
|
import { assertTransactionFits, transactionBytes, MAX_TRANSACTION_BYTES } from "../network/transaction-size.js";
|
|
8
|
-
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, WriteFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability } from "../index.js";
|
|
8
|
+
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, WriteFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability, MessageOptions } from "../index.js";
|
|
9
9
|
import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
|
|
10
10
|
import type { HlcClock } from "../hlc/clock.js";
|
|
11
11
|
import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
|
|
@@ -793,7 +793,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
793
793
|
{
|
|
794
794
|
expiration,
|
|
795
795
|
dialTimeoutMs: this.dialTimeoutMs,
|
|
796
|
-
coordinatingBlockIds: batch.coordinatingBlockIds
|
|
796
|
+
coordinatingBlockIds: batch.coordinatingBlockIds,
|
|
797
|
+
...this.localWriterMark(batch.peerId)
|
|
797
798
|
}
|
|
798
799
|
),
|
|
799
800
|
batch => blockIdsForTransforms(batch.payload),
|
|
@@ -931,7 +932,8 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
931
932
|
answer = await repo.pendMany(requests, {
|
|
932
933
|
expiration: Date.now() + this.timeoutMs,
|
|
933
934
|
dialTimeoutMs: this.dialTimeoutMs,
|
|
934
|
-
coordinatingBlockIds: blockIds
|
|
935
|
+
coordinatingBlockIds: blockIds,
|
|
936
|
+
...this.localWriterMark(coordinator)
|
|
935
937
|
});
|
|
936
938
|
} catch (e) {
|
|
937
939
|
log('WARN: batched pend threw; pending each request on its own: actionId=%s peer=%s error=%s',
|
|
@@ -950,6 +952,16 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
950
952
|
return { success: true, pending: answer.pending, blockIds, durability: mergeDurability([answer.durability]) };
|
|
951
953
|
}
|
|
952
954
|
|
|
955
|
+
/**
|
|
956
|
+
* `{ localWriter: true }` when `peerId` is the node this transactor runs on (`localPeerId`), else
|
|
957
|
+
* nothing — the mark `MessageOptions.localWriter` describes, set on every pend of this writer that
|
|
958
|
+
* the node applies in process. Keyed on the peer each request actually goes to, so a batch re-homed
|
|
959
|
+
* by a retry onto another coordinator carries no mark.
|
|
960
|
+
*/
|
|
961
|
+
private localWriterMark(peerId: PeerId): Pick<MessageOptions, 'localWriter'> {
|
|
962
|
+
return this.localPeerId !== undefined && peerId.toString() === this.localPeerId ? { localWriter: true } : {};
|
|
963
|
+
}
|
|
964
|
+
|
|
953
965
|
/**
|
|
954
966
|
* The one coordinator {@link coordinatorCover} assigns every block in `blockIds` to, or `undefined`
|
|
955
967
|
* when it assigns more than one, or a lookup failed (logged).
|
|
@@ -1286,7 +1298,10 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1286
1298
|
*
|
|
1287
1299
|
* Residuals: the cancel is retried and checked ({@link dischargeCancel}), and one that outlasts its
|
|
1288
1300
|
* budget is kept and re-sent for this transactor's life ({@link owedCancels}), so only a writer that
|
|
1289
|
-
* exits first leaves the record
|
|
1301
|
+
* exits first leaves the record standing. A record the node applied alone for a writer in its own
|
|
1302
|
+
* process is stamped with that node run (`MessageOptions.localWriter`) and the node's next run drops
|
|
1303
|
+
* it on the next write to the block; on a larger cohort, or for a remote writer, it stays wedged
|
|
1304
|
+
* until a node-side backstop sweep exists (backlog:
|
|
1290
1305
|
* `debt-unpromotable-pending-records-need-a-sweep`); and this covers only the sweep's abandonment
|
|
1291
1306
|
* — `StorageRepo.commit`'s genuine-fault arm deliberately KEEPS a failed batch's pendings for a
|
|
1292
1307
|
* retry, so it is a second producer of the same durable state whenever that retry never comes.
|
|
@@ -1549,8 +1564,11 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1549
1564
|
* from this loop's own outstanding set, is what spares every caller from having to remember to.
|
|
1550
1565
|
*
|
|
1551
1566
|
* NOTE: a writer that exits (or stops this transactor, {@link stop}) while a cancel is still owed
|
|
1552
|
-
* strands the record
|
|
1553
|
-
* `
|
|
1567
|
+
* strands the record. The node reclaims it in one shape only: a record it applied alone, for a
|
|
1568
|
+
* writer in its own process (`MessageOptions.localWriter`), is stamped with the node run that
|
|
1569
|
+
* saved it, and the node's next run drops it on the next write to the block. A record on a larger
|
|
1570
|
+
* cohort, or a remote writer's, is reclaimed by nothing node-side; that residual is the backstop
|
|
1571
|
+
* tracked in backlog `debt-unpromotable-pending-records-need-a-sweep`.
|
|
1554
1572
|
*
|
|
1555
1573
|
* @param seedBatches Batches to use for round 0 only (see {@link cancelBatch}). Omitted → round
|
|
1556
1574
|
* 0 resolves coordinators live like every later round.
|
package/src/transform/atomic.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Tracker } from "./tracker.js";
|
|
2
|
-
import type { IBlock, BlockStore } from "../index.js";
|
|
2
|
+
import type { IBlock, BlockStore, Transforms } from "../index.js";
|
|
3
3
|
import { applyTransformToStore } from "./helpers.js";
|
|
4
4
|
|
|
5
5
|
export class Atomic<TBlock extends IBlock> extends Tracker<TBlock> {
|
|
@@ -7,7 +7,13 @@ export class Atomic<TBlock extends IBlock> extends Tracker<TBlock> {
|
|
|
7
7
|
super(store);
|
|
8
8
|
}
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
/** Flush what this atomic staged into its store and empty it.
|
|
11
|
+
*
|
|
12
|
+
* @returns the flushed transforms. This atomic no longer references them, and a parent
|
|
13
|
+
* `Tracker` copied what it kept ({@link Tracker.absorb}), so the caller may keep them as a record
|
|
14
|
+
* of what was staged; any other store may have taken the values by reference, so a caller over
|
|
15
|
+
* one must not change them. */
|
|
16
|
+
commit(): Transforms {
|
|
11
17
|
// A parent tracker takes the staged transform AND the bases pinned inside this atomic
|
|
12
18
|
// (Tracker.absorb): without the pins, a single act() carrying more actions than the read
|
|
13
19
|
// cache holds would lose digest coverage, the cache having evicted the early bases by the
|
|
@@ -15,10 +21,11 @@ export class Atomic<TBlock extends IBlock> extends Tracker<TBlock> {
|
|
|
15
21
|
// cache moved on from between the pin and the flush would be re-pinned at the newer revision.
|
|
16
22
|
// Any other store just receives the transform.
|
|
17
23
|
if (this.store instanceof Tracker) {
|
|
18
|
-
this.store.absorb(this);
|
|
19
|
-
} else {
|
|
20
|
-
applyTransformToStore(this.reset(), this.store);
|
|
24
|
+
return this.store.absorb(this);
|
|
21
25
|
}
|
|
26
|
+
const transforms = this.reset();
|
|
27
|
+
applyTransformToStore(transforms, this.store);
|
|
28
|
+
return transforms;
|
|
22
29
|
}
|
|
23
30
|
|
|
24
31
|
// rollback = reset
|