@optimystic/db-core 1.18.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/src/blocks/helpers.d.ts +7 -0
- package/dist/src/blocks/helpers.d.ts.map +1 -1
- package/dist/src/blocks/helpers.js +7 -1
- package/dist/src/blocks/helpers.js.map +1 -1
- package/dist/src/btree/btree.d.ts +100 -4
- package/dist/src/btree/btree.d.ts.map +1 -1
- package/dist/src/btree/btree.js +252 -90
- package/dist/src/btree/btree.js.map +1 -1
- package/dist/src/btree/node-division.d.ts +37 -0
- package/dist/src/btree/node-division.d.ts.map +1 -0
- package/dist/src/btree/node-division.js +133 -0
- package/dist/src/btree/node-division.js.map +1 -0
- package/dist/src/chain/chain.d.ts +23 -0
- package/dist/src/chain/chain.d.ts.map +1 -1
- package/dist/src/chain/chain.js +43 -6
- package/dist/src/chain/chain.js.map +1 -1
- package/dist/src/cluster/structs.d.ts +41 -8
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js +18 -7
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/collection/action.d.ts +29 -0
- package/dist/src/collection/action.d.ts.map +1 -1
- package/dist/src/collection/action.js +6 -0
- package/dist/src/collection/action.js.map +1 -1
- package/dist/src/collection/collection.d.ts +566 -120
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +1084 -495
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/collection/head-commit.d.ts +110 -0
- package/dist/src/collection/head-commit.d.ts.map +1 -0
- package/dist/src/collection/head-commit.js +65 -0
- package/dist/src/collection/head-commit.js.map +1 -0
- package/dist/src/collection/index.d.ts +2 -0
- package/dist/src/collection/index.d.ts.map +1 -1
- package/dist/src/collection/index.js +2 -0
- package/dist/src/collection/index.js.map +1 -1
- package/dist/src/collection/segments.d.ts +77 -0
- package/dist/src/collection/segments.d.ts.map +1 -0
- package/dist/src/collection/segments.js +110 -0
- package/dist/src/collection/segments.js.map +1 -0
- package/dist/src/collection/struct.d.ts +176 -1
- package/dist/src/collection/struct.d.ts.map +1 -1
- package/dist/src/collection/struct.js +70 -0
- package/dist/src/collection/struct.js.map +1 -1
- package/dist/src/collection/write-attempts.d.ts +245 -0
- package/dist/src/collection/write-attempts.d.ts.map +1 -0
- package/dist/src/collection/write-attempts.js +439 -0
- package/dist/src/collection/write-attempts.js.map +1 -0
- package/dist/src/collections/tree/struct.d.ts +100 -17
- package/dist/src/collections/tree/struct.d.ts.map +1 -1
- package/dist/src/collections/tree/struct.js +38 -7
- package/dist/src/collections/tree/struct.js.map +1 -1
- package/dist/src/collections/tree/tree.d.ts +17 -5
- package/dist/src/collections/tree/tree.d.ts.map +1 -1
- package/dist/src/collections/tree/tree.js +117 -29
- package/dist/src/collections/tree/tree.js.map +1 -1
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/log/log.d.ts +5 -1
- package/dist/src/log/log.d.ts.map +1 -1
- package/dist/src/log/log.js +7 -1
- package/dist/src/log/log.js.map +1 -1
- package/dist/src/network/block-size.d.ts +67 -0
- package/dist/src/network/block-size.d.ts.map +1 -0
- package/dist/src/network/block-size.js +82 -0
- package/dist/src/network/block-size.js.map +1 -0
- package/dist/src/network/i-repo.d.ts +3 -0
- package/dist/src/network/i-repo.d.ts.map +1 -1
- package/dist/src/network/index.d.ts +5 -0
- package/dist/src/network/index.d.ts.map +1 -1
- package/dist/src/network/index.js +5 -0
- package/dist/src/network/index.js.map +1 -1
- package/dist/src/network/promise-shortfall-error.d.ts +40 -0
- package/dist/src/network/promise-shortfall-error.d.ts.map +1 -0
- package/dist/src/network/promise-shortfall-error.js +67 -0
- package/dist/src/network/promise-shortfall-error.js.map +1 -0
- package/dist/src/network/refuse-solo.d.ts +8 -0
- package/dist/src/network/refuse-solo.d.ts.map +1 -0
- package/dist/src/network/refuse-solo.js +8 -0
- package/dist/src/network/refuse-solo.js.map +1 -0
- package/dist/src/network/stale-failure.d.ts +20 -3
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +24 -3
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +161 -13
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/network/transaction-size.d.ts +59 -0
- package/dist/src/network/transaction-size.d.ts.map +1 -0
- package/dist/src/network/transaction-size.js +74 -0
- package/dist/src/network/transaction-size.js.map +1 -0
- package/dist/src/network/validity-rejected-error.d.ts +28 -0
- package/dist/src/network/validity-rejected-error.d.ts.map +1 -0
- package/dist/src/network/validity-rejected-error.js +51 -0
- package/dist/src/network/validity-rejected-error.js.map +1 -0
- package/dist/src/offline/entry.d.ts +72 -0
- package/dist/src/offline/entry.d.ts.map +1 -0
- package/dist/src/offline/entry.js +73 -0
- package/dist/src/offline/entry.js.map +1 -0
- package/dist/src/offline/index.d.ts +5 -0
- package/dist/src/offline/index.d.ts.map +1 -0
- package/dist/src/offline/index.js +5 -0
- package/dist/src/offline/index.js.map +1 -0
- package/dist/src/offline/journal.d.ts +61 -0
- package/dist/src/offline/journal.d.ts.map +1 -0
- package/dist/src/offline/journal.js +2 -0
- package/dist/src/offline/journal.js.map +1 -0
- package/dist/src/offline/queue.d.ts +484 -0
- package/dist/src/offline/queue.d.ts.map +1 -0
- package/dist/src/offline/queue.js +1236 -0
- package/dist/src/offline/queue.js.map +1 -0
- package/dist/src/offline/reachability.d.ts +25 -0
- package/dist/src/offline/reachability.d.ts.map +1 -0
- package/dist/src/offline/reachability.js +13 -0
- package/dist/src/offline/reachability.js.map +1 -0
- package/dist/src/offline/status.d.ts +148 -0
- package/dist/src/offline/status.d.ts.map +1 -0
- package/dist/src/offline/status.js +63 -0
- package/dist/src/offline/status.js.map +1 -0
- package/dist/src/offline/wake.d.ts +30 -0
- package/dist/src/offline/wake.d.ts.map +1 -0
- package/dist/src/offline/wake.js +68 -0
- package/dist/src/offline/wake.js.map +1 -0
- package/dist/src/reactivity/config.d.ts +8 -4
- package/dist/src/reactivity/config.d.ts.map +1 -1
- package/dist/src/reactivity/config.js +7 -3
- package/dist/src/reactivity/config.js.map +1 -1
- package/dist/src/reactivity/rotation.d.ts +9 -3
- package/dist/src/reactivity/rotation.d.ts.map +1 -1
- package/dist/src/reactivity/rotation.js +8 -2
- package/dist/src/reactivity/rotation.js.map +1 -1
- package/dist/src/testing/test-transactor.d.ts +31 -0
- package/dist/src/testing/test-transactor.d.ts.map +1 -1
- package/dist/src/testing/test-transactor.js +60 -0
- package/dist/src/testing/test-transactor.js.map +1 -1
- package/dist/src/transaction/commit-failure-class.d.ts +103 -0
- package/dist/src/transaction/commit-failure-class.d.ts.map +1 -0
- package/dist/src/transaction/commit-failure-class.js +96 -0
- package/dist/src/transaction/commit-failure-class.js.map +1 -0
- package/dist/src/transaction/coordinator.d.ts +224 -31
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +841 -362
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transaction/entry-commit.d.ts +139 -0
- package/dist/src/transaction/entry-commit.d.ts.map +1 -0
- package/dist/src/transaction/entry-commit.js +199 -0
- package/dist/src/transaction/entry-commit.js.map +1 -0
- package/dist/src/transaction/index.d.ts +4 -1
- package/dist/src/transaction/index.d.ts.map +1 -1
- package/dist/src/transaction/index.js +1 -0
- package/dist/src/transaction/index.js.map +1 -1
- package/dist/src/transaction/transaction.d.ts +20 -0
- package/dist/src/transaction/transaction.d.ts.map +1 -1
- package/dist/src/transaction/validator.d.ts +7 -0
- package/dist/src/transaction/validator.d.ts.map +1 -1
- package/dist/src/transaction/validator.js +21 -12
- package/dist/src/transaction/validator.js.map +1 -1
- package/dist/src/transactor/change-notifier.d.ts +13 -4
- package/dist/src/transactor/change-notifier.d.ts.map +1 -1
- package/dist/src/transactor/change-notifier.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts +58 -8
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +244 -50
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/transactor/transactor-source.d.ts +22 -11
- package/dist/src/transactor/transactor-source.d.ts.map +1 -1
- package/dist/src/transactor/transactor-source.js +9 -13
- package/dist/src/transactor/transactor-source.js.map +1 -1
- package/dist/src/transactor/transactor.d.ts +14 -0
- package/dist/src/transactor/transactor.d.ts.map +1 -1
- package/dist/src/transform/digest.d.ts +20 -0
- package/dist/src/transform/digest.d.ts.map +1 -1
- package/dist/src/transform/digest.js +23 -3
- package/dist/src/transform/digest.js.map +1 -1
- package/dist/src/utility/visit-every.d.ts +14 -0
- package/dist/src/utility/visit-every.d.ts.map +1 -0
- package/dist/src/utility/visit-every.js +27 -0
- package/dist/src/utility/visit-every.js.map +1 -0
- package/package.json +1 -1
- package/src/blocks/helpers.ts +8 -1
- package/src/btree/btree.ts +266 -88
- package/src/btree/node-division.ts +155 -0
- package/src/chain/chain.ts +55 -6
- package/src/cluster/structs.ts +42 -7
- package/src/collection/action.ts +34 -0
- package/src/collection/collection.ts +1167 -540
- package/src/collection/head-commit.ts +148 -0
- package/src/collection/index.ts +2 -0
- package/src/collection/segments.ts +149 -0
- package/src/collection/struct.ts +201 -1
- package/src/collection/write-attempts.ts +506 -0
- package/src/collections/tree/struct.ts +97 -17
- package/src/collections/tree/tree.ts +129 -31
- package/src/index.ts +2 -0
- package/src/log/log.ts +7 -1
- package/src/network/block-size.ts +81 -0
- package/src/network/i-repo.ts +3 -0
- package/src/network/index.ts +5 -0
- package/src/network/promise-shortfall-error.ts +69 -0
- package/src/network/refuse-solo.ts +7 -0
- package/src/network/stale-failure.ts +27 -4
- package/src/network/struct.ts +169 -15
- package/src/network/transaction-size.ts +73 -0
- package/src/network/validity-rejected-error.ts +55 -0
- package/src/offline/entry.ts +127 -0
- package/src/offline/index.ts +4 -0
- package/src/offline/journal.ts +60 -0
- package/src/offline/queue.ts +1342 -0
- package/src/offline/reachability.ts +32 -0
- package/src/offline/status.ts +183 -0
- package/src/offline/wake.ts +67 -0
- package/src/reactivity/config.ts +8 -4
- package/src/reactivity/rotation.ts +9 -3
- package/src/testing/test-transactor.ts +65 -0
- package/src/transaction/commit-failure-class.ts +133 -0
- package/src/transaction/coordinator.ts +969 -393
- package/src/transaction/entry-commit.ts +285 -0
- package/src/transaction/index.ts +4 -0
- package/src/transaction/transaction.ts +21 -0
- package/src/transaction/validator.ts +22 -12
- package/src/transactor/change-notifier.ts +13 -4
- package/src/transactor/network-transactor.ts +252 -55
- package/src/transactor/transactor-source.ts +29 -13
- package/src/transactor/transactor.ts +16 -0
- package/src/transform/digest.ts +36 -3
- package/src/utility/visit-every.ts +25 -0
|
@@ -0,0 +1,1342 @@
|
|
|
1
|
+
import { randomBytes } from '@noble/hashes/utils.js';
|
|
2
|
+
import { toString as uint8ArrayToString } from 'uint8arrays/to-string';
|
|
3
|
+
import type { Collection } from '../collection/collection.js';
|
|
4
|
+
import type { EntryId } from '../collection/segments.js';
|
|
5
|
+
import { ActionRefusedError, SealedSegmentsError } from '../collection/struct.js';
|
|
6
|
+
import type { CollectionId, InFlightAttempt, SyncOptions } from '../collection/struct.js';
|
|
7
|
+
import type { AlreadyAppliedTally } from '../collection/action.js';
|
|
8
|
+
import type { ITransactor } from '../transactor/transactor.js';
|
|
9
|
+
import { TransactionCoordinator } from '../transaction/coordinator.js';
|
|
10
|
+
import { classifyCommitFailure, refusalReason } from '../transaction/commit-failure-class.js';
|
|
11
|
+
import { recordCollectionDurability, transactionDurability } from '../transaction/durability.js';
|
|
12
|
+
import type { TransactionDurability } from '../transaction/durability.js';
|
|
13
|
+
import type { CoordinatorPartialCommitError } from '../transaction/errors.js';
|
|
14
|
+
import type { RandFn } from '../utility/backoff.js';
|
|
15
|
+
import { createLogger } from '../logger.js';
|
|
16
|
+
import { alwaysReachable, gateBlockIds } from './reachability.js';
|
|
17
|
+
import type { ICohortReachability } from './reachability.js';
|
|
18
|
+
import {
|
|
19
|
+
EntryDiscardedError,
|
|
20
|
+
EntryExpiredError,
|
|
21
|
+
EntryNotDiscardableError,
|
|
22
|
+
QueueDisposedError,
|
|
23
|
+
UnknownEntryError,
|
|
24
|
+
} from './status.js';
|
|
25
|
+
import type { EntryStatus, EntryWaitingFor, OfflineQueueState, QueuedEntry, QueuedFailureReason } from './status.js';
|
|
26
|
+
import type { IOfflineJournal, JournalDriveState, JournalEntryRecord, JournalSettledStatus } from './journal.js';
|
|
27
|
+
import { committedField, currentStatus, driveState, effectOf, queuedStatus, queuedStatusChanged, snapshot } from './entry.js';
|
|
28
|
+
import type { Entry } from './entry.js';
|
|
29
|
+
import { DrainWaker } from './wake.js';
|
|
30
|
+
|
|
31
|
+
const log = createLogger('offline');
|
|
32
|
+
|
|
33
|
+
/** The re-check timer's defaults while the cohort is unreachable or after a transient failure. */
|
|
34
|
+
const DefaultRecheckBaseMs = 1_000;
|
|
35
|
+
const DefaultRecheckMaxMs = 60_000;
|
|
36
|
+
|
|
37
|
+
export type OfflineQueueOptions = {
|
|
38
|
+
/** The gate of docs/offline-sync.md §When to submit. Default: always reachable. */
|
|
39
|
+
reachability?: ICohortReachability;
|
|
40
|
+
/** An external wake signal — a gate's `ICohortReachability.onChange`, say: subscribe the queue's
|
|
41
|
+
* {@link OfflineQueue.wake} and return the unsubscribe, which {@link OfflineQueue.dispose} calls.
|
|
42
|
+
* The Quereus plugin does not use it: its controller calls `wake` itself, after retrying a deferred
|
|
43
|
+
* restore. */
|
|
44
|
+
wakeOn?: (wake: () => void) => () => void;
|
|
45
|
+
/** Fail an entry that stays unsettled this long, with reason `expired`. Default: no limit. An
|
|
46
|
+
* entry in doubt is never expired; it expires once its doubt resolves without committing. */
|
|
47
|
+
expireAfterMs?: number;
|
|
48
|
+
/** The re-check timer while the cohort is unreachable or after a transient failure: exponential
|
|
49
|
+
* from `baseMs` to `maxMs`, jittered (`jitteredBackoffMs`). Defaults 1 s / 60 s. */
|
|
50
|
+
recheck?: { baseMs?: number; maxMs?: number; rand?: RandFn };
|
|
51
|
+
/** Retry bounds handed to each submission (`Collection.commitHead` / `TransactionCoordinator.commitEntry`). */
|
|
52
|
+
submit?: Pick<SyncOptions, 'maxAttempts' | 'deadlineMs' | 'baseBackoffMs' | 'maxBackoffMs' | 'maxStalledAttempts'>;
|
|
53
|
+
/** Let an entry be committed by a lone node that knows it is not alone. Unless this is `true`,
|
|
54
|
+
* every submission sets `SyncOptions.refuseSoloCommit`, so a coordinator whose cohort is only
|
|
55
|
+
* itself refuses the entry while it knows of absent members, awaits a bootstrap peer or cannot
|
|
56
|
+
* route the write, and the entry waits for its cohort (`waitingFor: 'cohort'`) rather than settling
|
|
57
|
+
* `committed` on a write only this machine holds (docs/offline-sync.md §When to submit: the
|
|
58
|
+
* reachability gate). A node that is genuinely alone commits either way. Default `false`. */
|
|
59
|
+
allowSoloCommit?: boolean;
|
|
60
|
+
/** The durable journal (docs/offline-sync.md §Persistence), called at every moment the design's
|
|
61
|
+
* write ordering names: before `queued` is emitted, before each send, after a partial commit and
|
|
62
|
+
* an identity renewal, and before each terminal status. Absent, nothing is journaled and the queue
|
|
63
|
+
* is in memory only. */
|
|
64
|
+
journal?: IOfflineJournal;
|
|
65
|
+
/** Clock, for tests. Default `Date.now`. */
|
|
66
|
+
now?: () => number;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
/** What {@link OfflineQueue.restore} puts an entry back from: the journal's entry record, the
|
|
70
|
+
* collections the record's segments name (opened by the caller), and the drive record, when one was
|
|
71
|
+
* written. */
|
|
72
|
+
export type RestoreInit = {
|
|
73
|
+
readonly id: EntryId;
|
|
74
|
+
readonly seq: number;
|
|
75
|
+
readonly queuedAt: number;
|
|
76
|
+
readonly expiresAt?: number;
|
|
77
|
+
/** The collections the entry sealed a segment in, in the record's order — `stage` is called for
|
|
78
|
+
* each, in this order. */
|
|
79
|
+
readonly collections: readonly Collection<any>[];
|
|
80
|
+
/** From the drive record, when one was written: the entry is then in doubt and resumes these.
|
|
81
|
+
* Absent: nothing was ever sent; a fresh action id is minted. */
|
|
82
|
+
readonly drive?: JournalDriveState;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
/** Per collection, what {@link OfflineQueue.restore}'s `stage` found: the segment is in place
|
|
86
|
+
* (possibly refused — the refusal is reported here, since the collection cannot report it to a
|
|
87
|
+
* queue that does not yet hold the entry; see `Collection.restoreSealed`), or the collection already
|
|
88
|
+
* holds the entry's recorded attempt as committed (`Collection.holdsCommitted`), so nothing was
|
|
89
|
+
* staged. */
|
|
90
|
+
export type RestoredSegment = { kind: 'staged'; refused?: ActionRefusedError } | { kind: 'already-committed' };
|
|
91
|
+
|
|
92
|
+
/** A refusal `onSealedRefused` reported and the drain has not yet acted on. */
|
|
93
|
+
type PendingRefusal = { readonly error: ActionRefusedError; readonly causedBy: EntryId | undefined };
|
|
94
|
+
|
|
95
|
+
type Deferred<T> = { resolve: (value: T) => void; reject: (error: unknown) => void };
|
|
96
|
+
|
|
97
|
+
/** The drain hold a {@link OfflineQueue.restore} takes: the loop parks at the top of its next
|
|
98
|
+
* iteration (`parked` resolves there) until `release`, so a restore inserts its segments while no
|
|
99
|
+
* submission and no removal is under way. */
|
|
100
|
+
type DrainHold = {
|
|
101
|
+
readonly held: Promise<void>;
|
|
102
|
+
readonly release: () => void;
|
|
103
|
+
/** Resolved by the loop when it reaches the hold, by its exit, or at once by the taker when the
|
|
104
|
+
* loop was not running or was parked on an empty queue. */
|
|
105
|
+
readonly parked: Promise<void>;
|
|
106
|
+
readonly signalParked: () => void;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
/** The journal's write-ahead write threw: nothing was sent, so the entry is kept queued and re-driven
|
|
110
|
+
* after the backoff. Private to the queue — the journal's own error travels as `cause`. */
|
|
111
|
+
class RecordAttemptFailed extends Error {
|
|
112
|
+
constructor(readonly cause: unknown) {
|
|
113
|
+
super('the journal refused the attempt before it was sent');
|
|
114
|
+
this.name = 'RecordAttemptFailed';
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** A fresh entry or action id: 16 random bytes, base64url — the shape a sync's action id has. */
|
|
119
|
+
function mintId(): string {
|
|
120
|
+
return uint8ArrayToString(randomBytes(16), 'base64url');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function describe(error: unknown): string {
|
|
124
|
+
return error instanceof Error ? error.message : String(error);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** What `entry` wrote across every submission: the collections earlier partial commits committed,
|
|
128
|
+
* beside what the last submission answered (`undefined` when it answered nothing new). */
|
|
129
|
+
function entryDurability(entry: Entry, written: TransactionDurability | undefined): TransactionDurability {
|
|
130
|
+
const collections = new Map(entry.committed);
|
|
131
|
+
for (const [collectionId, durability] of written?.collections ?? []) {
|
|
132
|
+
recordCollectionDurability(collections, collectionId, durability);
|
|
133
|
+
}
|
|
134
|
+
return transactionDurability(collections);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The one transactor `remaining` share, which the per-entry coordinator commits through. An entry
|
|
139
|
+
* whose collections sit on different transactors cannot be one network transaction (each transactor
|
|
140
|
+
* is its own storage); the queue's callers never seal such a set together, so this is a defect that
|
|
141
|
+
* stops the drain rather than a failure of the entry.
|
|
142
|
+
*/
|
|
143
|
+
function sharedTransactor(entry: Entry, remaining: readonly Collection<any>[]): ITransactor {
|
|
144
|
+
const transactor = remaining[0]!.transactor;
|
|
145
|
+
const stranger = remaining.find(collection => collection.transactor !== transactor);
|
|
146
|
+
if (stranger !== undefined) {
|
|
147
|
+
throw new Error(`offline queue: entry ${entry.id} spans collections on different transactors (${remaining[0]!.id} and ${stranger.id}); an entry is one network transaction on one transactor`);
|
|
148
|
+
}
|
|
149
|
+
return transactor;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** The journal's entry record for `entry`, built from each sealed collection's segment. */
|
|
153
|
+
function entryRecord(entry: Entry): JournalEntryRecord {
|
|
154
|
+
return {
|
|
155
|
+
id: entry.id,
|
|
156
|
+
seq: entry.seq,
|
|
157
|
+
queuedAt: entry.queuedAt,
|
|
158
|
+
...(entry.expiresAt === undefined ? {} : { expiresAt: entry.expiresAt }),
|
|
159
|
+
segments: entry.collections.map(collection => {
|
|
160
|
+
const actions = collection.sealedActions(entry.id);
|
|
161
|
+
if (actions === undefined) {
|
|
162
|
+
throw new Error(`offline queue: entry ${entry.id} is not sealed in ${collection.id}, which sealPending just answered true for`);
|
|
163
|
+
}
|
|
164
|
+
return { collectionId: collection.id, actions };
|
|
165
|
+
}),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* The offline-sync queue (docs/offline-sync.md §Draining): an in-memory list of locally committed
|
|
171
|
+
* entries, each the sealed segments one local commit left in the collections it wrote, drained one
|
|
172
|
+
* at a time in queue order once the cohort can be reached.
|
|
173
|
+
*
|
|
174
|
+
* **One drain loop per queue, serialised.** At most one submission is on the network at a time, and
|
|
175
|
+
* the loop is the only thing that removes entries' segments from the collections apart from
|
|
176
|
+
* {@link discard}, whose removal runs on the same removal chain. Each iteration: park while a
|
|
177
|
+
* {@link restore} holds the drain; apply the refusals {@link onSealedRefused} has collected; expire
|
|
178
|
+
* what has outlived `expireAfterMs`; wait for a wake when the queue is empty; fail the head when a
|
|
179
|
+
* collection already records its refusal; ask the reachability gate; submit the head —
|
|
180
|
+
* `Collection.commitHead` for an entry over one collection, `TransactionCoordinator.commitEntry`
|
|
181
|
+
* over several — and apply the outcome by its class (`classifyCommitFailure`). The gate and the
|
|
182
|
+
* transient failures share one re-check timer, whose step a wake or a committed entry resets.
|
|
183
|
+
*
|
|
184
|
+
* **A partly committed entry is finished, not rolled back** (docs/offline-sync.md §Partial commits):
|
|
185
|
+
* the committed collections' segments are consumed, the entry stays at the head in doubt, and each
|
|
186
|
+
* later submission drives only the collections still holding its segment, under the same action id.
|
|
187
|
+
*
|
|
188
|
+
* **Removing an entry cascades** (docs/offline-sync.md §Removing an entry): a failed entry's
|
|
189
|
+
* segments are dropped from its collections, each drop replays what remains, a later entry the
|
|
190
|
+
* replay refuses is reported through {@link onSealedRefused} and failed in turn with `causedBy`
|
|
191
|
+
* set, until nothing is pending. The callback runs under the collection latch, so it only records
|
|
192
|
+
* and wakes; the loop acts.
|
|
193
|
+
*
|
|
194
|
+
* **The journal is written through at every step of an entry's life** (docs/offline-sync.md
|
|
195
|
+
* §Write ordering), when one is given: {@link enqueue} writes the entry record before it admits the
|
|
196
|
+
* entry, every send is preceded by the drive record (`IOfflineJournal.driving`, the write-ahead that
|
|
197
|
+
* makes the identity rule survive a crash), a partial commit and an identity renewal rewrite it, and
|
|
198
|
+
* each terminal status is written before it is emitted. {@link restore} is the inverse, for a journal
|
|
199
|
+
* putting its records back after a restart.
|
|
200
|
+
*
|
|
201
|
+
* Everything an application sees goes through {@link onStatus} and {@link onQueueState}; a listener
|
|
202
|
+
* that throws is logged and never breaks the drain.
|
|
203
|
+
*/
|
|
204
|
+
export class OfflineQueue {
|
|
205
|
+
private readonly list: Entry[] = [];
|
|
206
|
+
private readonly byId = new Map<EntryId, Entry>();
|
|
207
|
+
private nextSeq = 1;
|
|
208
|
+
private readonly reachability: ICohortReachability;
|
|
209
|
+
private readonly waker: DrainWaker;
|
|
210
|
+
private readonly submitOptions: OfflineQueueOptions['submit'];
|
|
211
|
+
private readonly refuseSoloCommit: boolean;
|
|
212
|
+
private readonly expireAfterMs: number | undefined;
|
|
213
|
+
private readonly journal: IOfflineJournal | undefined;
|
|
214
|
+
private readonly now: () => number;
|
|
215
|
+
private readonly unsubscribeWake: (() => void) | undefined;
|
|
216
|
+
|
|
217
|
+
private readonly statusListeners = new Set<(status: EntryStatus) => void>();
|
|
218
|
+
private readonly stateListeners = new Set<(state: OfflineQueueState) => void>();
|
|
219
|
+
private lastState: OfflineQueueState | undefined;
|
|
220
|
+
private unclassifiedFailures = 0;
|
|
221
|
+
|
|
222
|
+
private readonly pendingRefusals = new Map<EntryId, PendingRefusal>();
|
|
223
|
+
/** The entry whose segments are being dropped, while they are: the attribution of the refusals
|
|
224
|
+
* those drops report (see {@link causeOf}). */
|
|
225
|
+
private removing: Entry | undefined;
|
|
226
|
+
/** Every removal — the loop's and {@link discard}'s — runs on this chain, one at a time, so two
|
|
227
|
+
* never interleave their drops and {@link causeOf} always names one removal. Never rejects: a
|
|
228
|
+
* removal's throw reaches whoever awaited that removal, or {@link noteDefect} for a discard. */
|
|
229
|
+
private removals: Promise<void> = Promise.resolve();
|
|
230
|
+
/** Removals chained and not yet finished — a `discard`'s runs off the loop's path, and
|
|
231
|
+
* `whenDrained` answers at once only when none is. */
|
|
232
|
+
private pendingRemovals = 0;
|
|
233
|
+
|
|
234
|
+
/** Every {@link restore} runs on this chain, one at a time, so two never hold the drain at once.
|
|
235
|
+
* Never rejects: a restore's throw reaches its own caller. */
|
|
236
|
+
private restores: Promise<void> = Promise.resolve();
|
|
237
|
+
/** The hold the restore in progress has on the drain, while it has one (see {@link restore}). */
|
|
238
|
+
private hold: DrainHold | undefined;
|
|
239
|
+
/** The entry a {@link restore} has registered and not yet admitted: listed, so the refusals its
|
|
240
|
+
* insertion reports about other entries are attributed and its `stage` can read its position, but
|
|
241
|
+
* not yet discardable.
|
|
242
|
+
* NOTE: a `discard` of ANOTHER entry chained while `stage` runs is not held back; its drops run
|
|
243
|
+
* beside the insertion, serialised per collection only by the collection latch, and a refusal of
|
|
244
|
+
* the restoring entry's already-inserted segment that such a drop's replay reports is attributed
|
|
245
|
+
* by {@link causeOf} to the entry being discarded. Fine while restores run at start-up, before an
|
|
246
|
+
* application discards anything; if that attribution ever misleads, make `discard` wait out the
|
|
247
|
+
* hold too. */
|
|
248
|
+
private restoring: Entry | undefined;
|
|
249
|
+
|
|
250
|
+
private inFlight: { readonly entry: Entry; readonly controller: AbortController } | undefined;
|
|
251
|
+
private loop: Promise<void> | undefined;
|
|
252
|
+
/** The loop ran and has exited (disposal, or a defect): nothing parks any more. */
|
|
253
|
+
private loopStopped = false;
|
|
254
|
+
/** The loop is parked on an empty queue (so `whenDrained` can answer at once). */
|
|
255
|
+
private waitingForWork = false;
|
|
256
|
+
private readonly drainedWaiters: Deferred<void>[] = [];
|
|
257
|
+
private disposed = false;
|
|
258
|
+
private disposal: Promise<void> | undefined;
|
|
259
|
+
/** The error that stopped the drain (`SealedSegmentsError` out of a submission, a removal that
|
|
260
|
+
* threw): `whenDrained` rejects with it and `enqueue` refuses. */
|
|
261
|
+
private defect: unknown;
|
|
262
|
+
|
|
263
|
+
constructor(options: OfflineQueueOptions = {}) {
|
|
264
|
+
this.reachability = options.reachability ?? alwaysReachable;
|
|
265
|
+
this.waker = new DrainWaker(
|
|
266
|
+
{ baseMs: options.recheck?.baseMs ?? DefaultRecheckBaseMs, capMs: options.recheck?.maxMs ?? DefaultRecheckMaxMs },
|
|
267
|
+
options.recheck?.rand);
|
|
268
|
+
this.submitOptions = options.submit;
|
|
269
|
+
this.refuseSoloCommit = options.allowSoloCommit !== true;
|
|
270
|
+
this.expireAfterMs = options.expireAfterMs;
|
|
271
|
+
this.journal = options.journal;
|
|
272
|
+
this.now = options.now ?? Date.now;
|
|
273
|
+
this.unsubscribeWake = options.wakeOn?.(() => this.wake());
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Seal each collection's open segment under a new entry, write the journal's entry record, and
|
|
278
|
+
* queue it — in that order, so an entry is never admitted that the journal does not hold
|
|
279
|
+
* (docs/offline-sync.md §Write ordering, rule 1). `undefined` when no collection had anything
|
|
280
|
+
* staged — nothing to queue, nothing sealed, nothing written (`Collection.sealPending` answers
|
|
281
|
+
* `false` and changes nothing for an empty open segment, so a sibling with nothing staged is
|
|
282
|
+
* simply not part of the entry). Emits `queued` before resolving, so a caller can attribute it to
|
|
283
|
+
* the commit it is in. Starts the drain on the first call.
|
|
284
|
+
*
|
|
285
|
+
* **A journal write that throws fails the enqueue**: every collection sealed is unsealed again, in
|
|
286
|
+
* reverse sealing order (`Collection.unseal`, which puts each segment's actions back into its open
|
|
287
|
+
* segment), nothing is registered, no status is emitted, and the journal's error is rethrown. The
|
|
288
|
+
* caller then rolls its transaction back as for any failed commit. Without a journal the write is
|
|
289
|
+
* skipped and the enqueue cannot fail this way; it is one path either way.
|
|
290
|
+
*
|
|
291
|
+
* Latch-free like `sealPending`: the caller brackets it (no act or commit of these collections is
|
|
292
|
+
* in flight, and no second enqueue of them runs until this one has resolved — the unseal on
|
|
293
|
+
* failure requires each segment to still be the newest sealed one).
|
|
294
|
+
*
|
|
295
|
+
* @throws QueueDisposedError after {@link dispose}; the drain's defect, as `cause`, once the
|
|
296
|
+
* drain has stopped on one (nothing queued then would ever drain); whatever the journal threw. */
|
|
297
|
+
async enqueue(collections: readonly Collection<any>[], init?: { id?: EntryId; timestamp?: number }): Promise<QueuedEntry | undefined> {
|
|
298
|
+
this.assertAccepting();
|
|
299
|
+
const id = init?.id ?? mintId();
|
|
300
|
+
if (this.byId.has(id)) throw new Error(`entry ${id} is already queued`);
|
|
301
|
+
const sealed = collections.filter(collection => collection.sealPending(id));
|
|
302
|
+
if (sealed.length === 0) return undefined;
|
|
303
|
+
const queuedAt = init?.timestamp ?? this.now();
|
|
304
|
+
const entry: Entry = {
|
|
305
|
+
id,
|
|
306
|
+
seq: this.nextSeq,
|
|
307
|
+
queuedAt,
|
|
308
|
+
expiresAt: this.expireAfterMs === undefined ? undefined : queuedAt + this.expireAfterMs,
|
|
309
|
+
collections: sealed,
|
|
310
|
+
actionId: mintId(),
|
|
311
|
+
attempts: 0,
|
|
312
|
+
inDoubt: false,
|
|
313
|
+
retained: new Map(),
|
|
314
|
+
committed: new Map(),
|
|
315
|
+
applied: new Map(),
|
|
316
|
+
phase: 'queued',
|
|
317
|
+
waitingFor: this.list.length > 0 ? 'earlier-entry' : undefined,
|
|
318
|
+
lastError: undefined,
|
|
319
|
+
lastQueued: undefined,
|
|
320
|
+
expiryDeferredLogged: false,
|
|
321
|
+
};
|
|
322
|
+
// The sequence number is reserved for the write, so a restore that runs meanwhile takes a later
|
|
323
|
+
// one, and handed back when the write is refused and nothing took a later one since, so a
|
|
324
|
+
// refused enqueue leaves no gap in the order the journal lists.
|
|
325
|
+
this.nextSeq = entry.seq + 1;
|
|
326
|
+
try {
|
|
327
|
+
await this.journalQueued(entry);
|
|
328
|
+
} catch (err) {
|
|
329
|
+
if (this.nextSeq === entry.seq + 1) this.nextSeq = entry.seq;
|
|
330
|
+
throw err;
|
|
331
|
+
}
|
|
332
|
+
this.list.push(entry);
|
|
333
|
+
this.byId.set(id, entry);
|
|
334
|
+
this.emitQueued(entry);
|
|
335
|
+
this.publishState();
|
|
336
|
+
this.startDrain();
|
|
337
|
+
return snapshot(entry);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Put an entry back from a journal's records after a restart (docs/offline-sync.md §Restore after
|
|
342
|
+
* a restart): the only way to register an entry whose segments `enqueue` did not seal. The entry
|
|
343
|
+
* is registered first — inserted by `seq`, so a record restored late sits ahead of entries queued
|
|
344
|
+
* since — and `stage` is then called for each of its collections, in `init.collections` order, to
|
|
345
|
+
* put the segment in place (`Collection.restoreSealed`, at the position the caller finds by
|
|
346
|
+
* reading {@link entries}) or to answer that the collection already holds the entry's recorded
|
|
347
|
+
* attempt as committed (`Collection.holdsCommitted`). What `stage` answers decides what follows:
|
|
348
|
+
*
|
|
349
|
+
* - every collection `already-committed`: the entry is settled `committed` at once, with a
|
|
350
|
+
* durability whose every collection has an `undefined` class (the write is saved; who holds it
|
|
351
|
+
* is unknown), the journal's `settled` written as for any commit, and no `queued` status;
|
|
352
|
+
* - some `already-committed`: those are recorded as committed with `undefined` durability and the
|
|
353
|
+
* entry stays queued in doubt, as after a partial commit, the next submission driving the rest;
|
|
354
|
+
* - a `staged` segment `refused`: the refusal is queued for the loop's first step, which fails the
|
|
355
|
+
* entry through the ordinary cascade (with no `causedBy`);
|
|
356
|
+
* - otherwise it is simply queued. In every case but the first, `queued` is emitted and the drain
|
|
357
|
+
* woken, and the result is the entry as {@link entries} lists it.
|
|
358
|
+
*
|
|
359
|
+
* **The drain is held for the whole call.** The call waits until the loop is parked between
|
|
360
|
+
* iterations — not mid-submission, not mid-removal — holds it there, registers the entry, runs
|
|
361
|
+
* `stage`, and releases. A segment inserted ahead of the current head while that head's commit
|
|
362
|
+
* is on the network would fail the commit's own "head still first" assertion when it retakes the
|
|
363
|
+
* latch, and a `commitHead` that finds another segment ahead of the one it was asked for throws
|
|
364
|
+
* `SealedSegmentsError` (`not-head`), which the loop reads as a defect. A loop that is not
|
|
365
|
+
* running, or is parked on an empty queue, counts as parked at once; several restores serialise.
|
|
366
|
+
* `discard` during a restore is refused for the entry being restored (`UnknownEntryError`: it is
|
|
367
|
+
* registered but not yet admitted) and allowed for others, whose removals run on the removal
|
|
368
|
+
* chain the hold waits out before `stage` runs. A removal chained DURING `stage` runs beside it,
|
|
369
|
+
* serialised per collection by the collection latch only.
|
|
370
|
+
*
|
|
371
|
+
* **Identity.** With `init.drive`: the action id, the attempts to resume and the committed
|
|
372
|
+
* collections are taken from it, and the entry is in doubt when either map is non-empty. Without:
|
|
373
|
+
* a fresh action id, nothing retained, not in doubt.
|
|
374
|
+
*
|
|
375
|
+
* **A `stage` that throws** rolls the registration back: the entry leaves the queue, every segment
|
|
376
|
+
* `stage` already put in place is dropped (`Collection.dropSealed`), nothing is emitted, and the
|
|
377
|
+
* error is rethrown.
|
|
378
|
+
*
|
|
379
|
+
* **`dispose` during a restore.** A restore that has not started `stage` when the queue is
|
|
380
|
+
* disposed rejects with `QueueDisposedError` and releases the hold; one that has finishes its
|
|
381
|
+
* registration (the loop is stopping anyway, and an entry registered after disposal is left
|
|
382
|
+
* sealed for the application exactly like one that was queued before it).
|
|
383
|
+
*
|
|
384
|
+
* @throws QueueDisposedError after {@link dispose}; the drain's defect, as `cause`, once the
|
|
385
|
+
* drain has stopped on one; an `Error` for an id or a `seq` already queued; whatever `stage`
|
|
386
|
+
* threw. */
|
|
387
|
+
restore(init: RestoreInit, stage: (collection: Collection<any>) => Promise<RestoredSegment>): Promise<QueuedEntry | undefined> {
|
|
388
|
+
this.assertAccepting();
|
|
389
|
+
if (this.byId.has(init.id)) throw new Error(`entry ${init.id} is already queued`);
|
|
390
|
+
const run = this.restores.then(() => this.restoreHeld(init, stage));
|
|
391
|
+
const settled = (): void => {};
|
|
392
|
+
this.restores = run.then(settled, settled);
|
|
393
|
+
return run;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* The collection callback: pass as `CollectionInitOptions.onSealedRefused` of every collection
|
|
398
|
+
* whose entries this queue drains. Runs synchronously under the collection latch, so it calls no
|
|
399
|
+
* latched method (`dropSealed`, `update`, `commitHead`): it records the refusal and wakes the
|
|
400
|
+
* loop, which acts on it (step one of each iteration). A refusal for an entry this queue does not
|
|
401
|
+
* hold — another queue's, or one already removed — is ignored; for the entry being submitted it
|
|
402
|
+
* is noted and left to that submission, whose cycle re-checks the recorded refusal and throws it.
|
|
403
|
+
*/
|
|
404
|
+
readonly onSealedRefused = (entryId: EntryId, error: ActionRefusedError): void => {
|
|
405
|
+
const entry = this.byId.get(entryId);
|
|
406
|
+
if (entry === undefined) {
|
|
407
|
+
// NOTE: this also covers an entry `enqueue` has sealed but not yet registered (its journal
|
|
408
|
+
// write is outstanding) whose segment a head commit's replay refused meanwhile. The refusal
|
|
409
|
+
// stays recorded on the segment, and the loop's head check (`recordedRefusal`) fails the entry
|
|
410
|
+
// once it reaches the head — later than a pending refusal would, never lost. If that delay
|
|
411
|
+
// ever matters, re-check `sealedRefusal` for each collection when `enqueue` registers.
|
|
412
|
+
log('offline:refusal-for-unknown-entry entry=%s error=%s', entryId, error.message);
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
// The first refusal reported for an entry is the one it fails with; a second collection's
|
|
416
|
+
// refusal of the same entry adds nothing the application can act on separately.
|
|
417
|
+
if (this.pendingRefusals.has(entryId)) return;
|
|
418
|
+
this.pendingRefusals.set(entryId, { error, causedBy: this.causeOf(entry) });
|
|
419
|
+
this.wake();
|
|
420
|
+
};
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Reserve every sequence number up to and including `seq`: the next {@link enqueue} takes a higher
|
|
424
|
+
* one. For a journal that has read records it cannot put back yet (a table it cannot reach), so a
|
|
425
|
+
* transaction queued meanwhile never takes a journaled record's number — {@link restore} refuses a
|
|
426
|
+
* `seq` another queued entry holds. Never lowers the counter.
|
|
427
|
+
*/
|
|
428
|
+
reserveSeqThrough(seq: number): void {
|
|
429
|
+
this.nextSeq = Math.max(this.nextSeq, seq + 1);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/** The queued entries, in queue order. */
|
|
433
|
+
entries(): readonly QueuedEntry[] {
|
|
434
|
+
return this.list.map(entry => snapshot(entry));
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** The current status of a queued entry, or `undefined` for an id not queued — never queued, or
|
|
438
|
+
* already committed or failed (a settled entry leaves the queue with its terminal status event). */
|
|
439
|
+
status(entryId: EntryId): EntryStatus | undefined {
|
|
440
|
+
const entry = this.byId.get(entryId);
|
|
441
|
+
return entry === undefined ? undefined : currentStatus(entry);
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Withdraw a never-sent entry: it fails with reason `discarded` and cascades like any failure
|
|
446
|
+
* (docs/offline-sync.md §Removing an entry). Removed from the queue at once, so the drain cannot
|
|
447
|
+
* pick it up; its segments are dropped on the removal chain and `failed` is emitted when the drops
|
|
448
|
+
* are done.
|
|
449
|
+
*
|
|
450
|
+
* @throws EntryNotDiscardableError for an entry that is in doubt (sent at least once — the
|
|
451
|
+
* application cannot withdraw a change it cannot rule out having saved) or being submitted;
|
|
452
|
+
* UnknownEntryError for an id not queued, or one a {@link restore} has registered but not yet
|
|
453
|
+
* admitted. */
|
|
454
|
+
discard(entryId: EntryId): void {
|
|
455
|
+
const entry = this.byId.get(entryId);
|
|
456
|
+
if (entry === undefined || this.restoring === entry) throw new UnknownEntryError(entryId);
|
|
457
|
+
if (this.inFlight?.entry === entry) throw new EntryNotDiscardableError(entryId, 'submitting');
|
|
458
|
+
if (entry.inDoubt) throw new EntryNotDiscardableError(entryId, 'in-doubt');
|
|
459
|
+
this.claimRemoval(entry);
|
|
460
|
+
this.chainRemoval(entry, 'discarded', new EntryDiscardedError(entryId), undefined)
|
|
461
|
+
.catch(err => this.noteDefect(err));
|
|
462
|
+
this.wake();
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
onStatus(listener: (status: EntryStatus) => void): () => void {
|
|
466
|
+
this.statusListeners.add(listener);
|
|
467
|
+
return () => { this.statusListeners.delete(listener); };
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
onQueueState(listener: (state: OfflineQueueState) => void): () => void {
|
|
471
|
+
this.stateListeners.add(listener);
|
|
472
|
+
return () => { this.stateListeners.delete(listener); };
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
queueState(): OfflineQueueState {
|
|
476
|
+
const head = this.list[0];
|
|
477
|
+
const waitingFor = head?.waitingFor;
|
|
478
|
+
return {
|
|
479
|
+
depth: this.list.length,
|
|
480
|
+
oldestQueuedAt: head?.queuedAt,
|
|
481
|
+
draining: this.inFlight !== undefined,
|
|
482
|
+
waitingFor: waitingFor === 'cohort' || waitingFor === 'backoff' ? waitingFor : undefined,
|
|
483
|
+
unclassifiedFailures: this.unclassifiedFailures,
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/** Kick the drain: a new local commit ({@link enqueue} calls it), a connection event, a test. A
|
|
488
|
+
* wake that finds the loop busy is kept for its next wait, and a wake resets the re-check backoff. */
|
|
489
|
+
wake(): void {
|
|
490
|
+
this.waker.wake();
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** Resolves when the queue is empty and every removal has finished; rejects with
|
|
494
|
+
* {@link QueueDisposedError} on {@link dispose}, and with the drain's defect once it has stopped
|
|
495
|
+
* on one.
|
|
496
|
+
* NOTE: a {@link restore} in progress is not waited for: on an empty queue it answers at once
|
|
497
|
+
* while the restore is about to register an entry. Fine while restores run at start-up, before
|
|
498
|
+
* anything awaits a drain; if a host ever awaits both, make this wait out the restore chain. */
|
|
499
|
+
whenDrained(): Promise<void> {
|
|
500
|
+
if (this.disposed) return Promise.reject(new QueueDisposedError());
|
|
501
|
+
if (this.defect !== undefined) return Promise.reject(this.defect);
|
|
502
|
+
const idle = this.loop === undefined || (this.waitingForWork && this.pendingRemovals === 0);
|
|
503
|
+
if (this.list.length === 0 && idle) return Promise.resolve();
|
|
504
|
+
return new Promise<void>((resolve, reject) => { this.drainedWaiters.push({ resolve, reject }); });
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Stop draining. Unsubscribes the wake signal, clears the timer, aborts the submission in flight
|
|
509
|
+
* (its cycle stops after its current network `await`) and resolves once the loop has applied that
|
|
510
|
+
* submission's outcome locally — a committed outcome is still emitted, a failure is recorded on the
|
|
511
|
+
* entry as usual, and no further entry is submitted. Entries left queued stay sealed in their
|
|
512
|
+
* collections (they are the application's; nothing is dropped). `whenDrained` rejects with
|
|
513
|
+
* {@link QueueDisposedError}; a later `enqueue` throws it. Idempotent.
|
|
514
|
+
*/
|
|
515
|
+
dispose(): Promise<void> {
|
|
516
|
+
this.disposal ??= this.disposeOnce();
|
|
517
|
+
return this.disposal;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
private async disposeOnce(): Promise<void> {
|
|
521
|
+
this.disposed = true;
|
|
522
|
+
this.unsubscribeWake?.();
|
|
523
|
+
this.inFlight?.controller.abort(new QueueDisposedError());
|
|
524
|
+
this.wake();
|
|
525
|
+
await this.loop;
|
|
526
|
+
await this.settledRemovals();
|
|
527
|
+
const error = new QueueDisposedError();
|
|
528
|
+
for (const waiter of this.drainedWaiters.splice(0)) waiter.reject(error);
|
|
529
|
+
this.publishState();
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/** The refusal every entry point makes after {@link dispose} and once the drain has stopped on a
|
|
533
|
+
* defect: nothing queued then would ever drain. */
|
|
534
|
+
private assertAccepting(): void {
|
|
535
|
+
if (this.disposed) throw new QueueDisposedError();
|
|
536
|
+
if (this.defect !== undefined) throw new Error('the offline queue stopped on a defect; see cause', { cause: this.defect });
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Start the drain on the first entry; wake it for every later one. */
|
|
540
|
+
private startDrain(): void {
|
|
541
|
+
if (this.loop === undefined) {
|
|
542
|
+
this.loop = this.drain();
|
|
543
|
+
} else {
|
|
544
|
+
this.wake();
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// ── the drain ──────────────────────────────────────────────────────────────────────────────
|
|
549
|
+
|
|
550
|
+
private async drain(): Promise<void> {
|
|
551
|
+
try {
|
|
552
|
+
while (!this.disposed) {
|
|
553
|
+
if (this.defect !== undefined) throw this.defect;
|
|
554
|
+
await this.parkForHold();
|
|
555
|
+
if (this.disposed) break;
|
|
556
|
+
await this.processPendingRefusals();
|
|
557
|
+
if (this.disposed) break;
|
|
558
|
+
await this.expireDue();
|
|
559
|
+
if (this.disposed) break;
|
|
560
|
+
const head = this.list[0];
|
|
561
|
+
if (head === undefined) {
|
|
562
|
+
this.settleDrained();
|
|
563
|
+
this.publishState();
|
|
564
|
+
this.waitingForWork = true;
|
|
565
|
+
try {
|
|
566
|
+
await this.waker.untilWoken();
|
|
567
|
+
} finally {
|
|
568
|
+
this.waitingForWork = false;
|
|
569
|
+
}
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
const refusal = this.recordedRefusal(head);
|
|
573
|
+
if (refusal !== undefined) {
|
|
574
|
+
// A refresh found it before the drain did: reported at once, nothing sent.
|
|
575
|
+
await this.failAndCascade(head, refusalReason(refusal), refusal, this.pendingRefusals.get(head.id)?.causedBy);
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
if (!await this.gateOpen(head)) {
|
|
579
|
+
this.setQueued(head, 'cohort');
|
|
580
|
+
await this.waker.untilWokenOrTimer();
|
|
581
|
+
continue;
|
|
582
|
+
}
|
|
583
|
+
// `discard` may have taken the head, or `dispose` may have run, while the gate was asked;
|
|
584
|
+
// a new iteration re-reads the one and ends on the other.
|
|
585
|
+
if (this.disposed || this.list[0] !== head) continue;
|
|
586
|
+
const retryLater = await this.submit(head);
|
|
587
|
+
if (retryLater) {
|
|
588
|
+
await this.waker.untilWokenOrTimer();
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
} catch (err) {
|
|
592
|
+
this.stopOnDefect(err);
|
|
593
|
+
} finally {
|
|
594
|
+
// Nothing parks any more: a restore waiting for the loop to park proceeds, since the loop
|
|
595
|
+
// touches no entry from here on.
|
|
596
|
+
this.loopStopped = true;
|
|
597
|
+
this.hold?.signalParked();
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/** Step zero of every iteration: while a {@link restore} holds the drain, say so and wait for its
|
|
602
|
+
* release — between iterations, so no submission and no removal of the loop's is under way. */
|
|
603
|
+
private async parkForHold(): Promise<void> {
|
|
604
|
+
const hold = this.hold;
|
|
605
|
+
if (hold === undefined) return;
|
|
606
|
+
hold.signalParked();
|
|
607
|
+
await hold.held;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/** Step one of every iteration: fail each entry a replay has refused since the last one, in
|
|
611
|
+
* queue order, each failure through the removal chain and so able to report more. */
|
|
612
|
+
private async processPendingRefusals(): Promise<void> {
|
|
613
|
+
for (;;) {
|
|
614
|
+
await this.settledRemovals();
|
|
615
|
+
const next = this.nextPendingRefusal();
|
|
616
|
+
if (next === undefined) return;
|
|
617
|
+
const { entry, refusal } = next;
|
|
618
|
+
await this.fail(entry, refusalReason(refusal.error), refusal.error, refusal.causedBy);
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/** The pending refusal whose entry is earliest in queue order, dropping any for an entry no
|
|
623
|
+
* longer queued. The submitting entry's is never met here: the loop runs this step only between
|
|
624
|
+
* submissions, and its submission throws the refusal itself. */
|
|
625
|
+
private nextPendingRefusal(): { entry: Entry; refusal: PendingRefusal } | undefined {
|
|
626
|
+
for (const entryId of [...this.pendingRefusals.keys()]) {
|
|
627
|
+
if (!this.byId.has(entryId)) this.pendingRefusals.delete(entryId);
|
|
628
|
+
}
|
|
629
|
+
for (const entry of this.list) {
|
|
630
|
+
const refusal = this.pendingRefusals.get(entry.id);
|
|
631
|
+
if (refusal !== undefined) return { entry, refusal };
|
|
632
|
+
}
|
|
633
|
+
return undefined;
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/** Step two: expire what has outlived its `expiresAt`, in queue order, head or not. An entry in
|
|
637
|
+
* doubt is kept and logged (`offline:expiry-deferred`): a possibly-landed write is never dropped
|
|
638
|
+
* on a clock; it expires once its doubt resolves without committing. Stops when the queue is
|
|
639
|
+
* disposed mid-step, since disposal leaves every entry as it is.
|
|
640
|
+
* NOTE: checked once per iteration, and a non-empty queue always has a timer or a submission
|
|
641
|
+
* pending, so an expiry is applied at most one re-check delay (`recheck.maxMs`, 60 s by default)
|
|
642
|
+
* late; if an application ever needs it on the minute, arm a timer at the earliest `expiresAt`. */
|
|
643
|
+
private async expireDue(): Promise<void> {
|
|
644
|
+
const now = this.now();
|
|
645
|
+
for (const entry of [...this.list]) {
|
|
646
|
+
if (this.disposed) return;
|
|
647
|
+
if (entry.expiresAt === undefined || entry.expiresAt > now || !this.byId.has(entry.id)) continue;
|
|
648
|
+
if (entry.inDoubt) {
|
|
649
|
+
if (!entry.expiryDeferredLogged) {
|
|
650
|
+
entry.expiryDeferredLogged = true;
|
|
651
|
+
log('offline:expiry-deferred entry=%s action=%s expiresAt=%d: in doubt, kept until its outcome is known',
|
|
652
|
+
entry.id, entry.actionId, entry.expiresAt);
|
|
653
|
+
}
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
await this.failAndCascade(entry, 'expired', new EntryExpiredError(entry.id, entry.queuedAt, entry.expiresAt));
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/** The refusal a collection has recorded for `entry`'s segment, if any has. */
|
|
661
|
+
private recordedRefusal(entry: Entry): ActionRefusedError | undefined {
|
|
662
|
+
for (const collection of entry.collections) {
|
|
663
|
+
const refusal = collection.sealedRefusal(entry.id);
|
|
664
|
+
if (refusal !== undefined) return refusal;
|
|
665
|
+
}
|
|
666
|
+
return undefined;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/** The collections in which `entry` is still a sealed segment: every collection it wrote, until a
|
|
670
|
+
* partial commit consumes some. */
|
|
671
|
+
private remainingCollections(entry: Entry): Collection<any>[] {
|
|
672
|
+
return entry.collections.filter(collection => collection.sealedEntries().includes(entry.id));
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/** The gate, asked with each of the head's remaining collections' header id and known log tail id.
|
|
676
|
+
* A gate that throws is logged and read as closed: the next re-check asks again. */
|
|
677
|
+
private async gateOpen(entry: Entry): Promise<boolean> {
|
|
678
|
+
const blockIds = this.remainingCollections(entry).flatMap(gateBlockIds);
|
|
679
|
+
try {
|
|
680
|
+
return await this.reachability.reachable(blockIds);
|
|
681
|
+
} catch (err) {
|
|
682
|
+
log('offline:reachability-threw entry=%s error=%s', entry.id, describe(err));
|
|
683
|
+
return false;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// ── submitting ─────────────────────────────────────────────────────────────────────────────
|
|
688
|
+
|
|
689
|
+
/** Submit the head once and apply the outcome. Answers whether the loop should wait out the
|
|
690
|
+
* re-check timer before its next iteration (a transient failure) rather than go straight on (the
|
|
691
|
+
* entry committed or failed). */
|
|
692
|
+
private async submit(entry: Entry): Promise<boolean> {
|
|
693
|
+
const remaining = this.remainingCollections(entry);
|
|
694
|
+
if (remaining.length === 0) {
|
|
695
|
+
throw new Error(`offline queue: entry ${entry.id} holds no sealed segment in any collection it wrote`);
|
|
696
|
+
}
|
|
697
|
+
entry.attempts++;
|
|
698
|
+
entry.phase = 'submitting';
|
|
699
|
+
const controller = new AbortController();
|
|
700
|
+
this.inFlight = { entry, controller };
|
|
701
|
+
this.emit({ kind: 'submitting', entryId: entry.id, attempt: entry.attempts });
|
|
702
|
+
this.publishState();
|
|
703
|
+
let outcome: { durability: TransactionDurability } | { error: unknown };
|
|
704
|
+
try {
|
|
705
|
+
outcome = { durability: await this.send(entry, remaining, controller.signal) };
|
|
706
|
+
} catch (error) {
|
|
707
|
+
outcome = { error };
|
|
708
|
+
} finally {
|
|
709
|
+
this.inFlight = undefined;
|
|
710
|
+
}
|
|
711
|
+
if ('durability' in outcome) {
|
|
712
|
+
await this.committed(entry, outcome.durability);
|
|
713
|
+
return false;
|
|
714
|
+
}
|
|
715
|
+
return this.applyFailure(entry, outcome.error);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/** One submission: `commitHead` for an entry over one collection, the coordinator's per-entry
|
|
719
|
+
* slice over several — built on the transactor the remaining collections share, since a queue
|
|
720
|
+
* holds collections of whatever transactor each was opened on — each under the entry's action id
|
|
721
|
+
* and local commit time, resuming from the attempts retained under that id, and — unless the queue
|
|
722
|
+
* allows it — refusing to be committed by a lone node that knows it is not alone. Each committed
|
|
723
|
+
* head segment's tally reaches {@link recordApplied} through the `onCommittedHead` hook. */
|
|
724
|
+
private async send(entry: Entry, remaining: readonly Collection<any>[], signal: AbortSignal): Promise<TransactionDurability> {
|
|
725
|
+
const submit: SyncOptions = { ...this.submitOptions, signal, ...(this.refuseSoloCommit ? { refuseSoloCommit: true } : {}) };
|
|
726
|
+
const recordAttempt = (attempts: ReadonlyMap<CollectionId, InFlightAttempt>) => this.recordAttempts(entry, attempts);
|
|
727
|
+
const onCommittedHead = (collectionId: CollectionId, applied: AlreadyAppliedTally) => this.recordApplied(entry, collectionId, applied);
|
|
728
|
+
if (remaining.length === 1) {
|
|
729
|
+
const collection = remaining[0]!;
|
|
730
|
+
const resume = entry.retained.get(collection.id);
|
|
731
|
+
const written = await collection.commitHead(entry.id, {
|
|
732
|
+
...submit,
|
|
733
|
+
actionId: entry.actionId,
|
|
734
|
+
timestamp: entry.queuedAt,
|
|
735
|
+
recordAttempt: attempt => recordAttempt(new Map([[collection.id, attempt]])),
|
|
736
|
+
onCommittedHead: applied => onCommittedHead(collection.id, applied),
|
|
737
|
+
...(resume === undefined ? {} : { resume }),
|
|
738
|
+
});
|
|
739
|
+
return transactionDurability(new Map([[collection.id, written]]));
|
|
740
|
+
}
|
|
741
|
+
const coordinator = new TransactionCoordinator(sharedTransactor(entry, remaining),
|
|
742
|
+
new Map(remaining.map(collection => [collection.id, collection])), { pendValidation: 'none' });
|
|
743
|
+
const resume = new Map([...entry.retained].filter(([collectionId]) => remaining.some(collection => collection.id === collectionId)));
|
|
744
|
+
return coordinator.commitEntry({
|
|
745
|
+
entryId: entry.id,
|
|
746
|
+
actionId: entry.actionId,
|
|
747
|
+
timestamp: entry.queuedAt,
|
|
748
|
+
collectionIds: remaining.map(collection => collection.id),
|
|
749
|
+
recordAttempt,
|
|
750
|
+
onCommittedHead,
|
|
751
|
+
...(resume.size === 0 ? {} : { resume }),
|
|
752
|
+
}, submit);
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
/** The `onCommittedHead` hook's closure: record what `collectionId`'s head segment accepted as
|
|
756
|
+
* already applied, for the entry's `effect` (`effectOf`). Runs under the collection latch, so it
|
|
757
|
+
* records and nothing else. One segment per collection per entry is dropped as committed exactly
|
|
758
|
+
* once, so this is a set, not a sum. */
|
|
759
|
+
private recordApplied(entry: Entry, collectionId: CollectionId, applied: AlreadyAppliedTally): void {
|
|
760
|
+
entry.applied.set(collectionId, applied);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
/** The queue's write-ahead wrapper: the one moment before a send the queue is told about. The
|
|
764
|
+
* journal's drive record goes first, with the attempts about to go beside the ones already
|
|
765
|
+
* retained (docs/offline-sync.md §Write ordering, rule 2); a throw out of it aborts the cycle with
|
|
766
|
+
* nothing sent (both drivers promise that), so an attempt it refused is neither retained nor a
|
|
767
|
+
* reason for doubt — the entry stays discardable, and `resume` on the next drive names only
|
|
768
|
+
* attempts that may have landed. */
|
|
769
|
+
private async recordAttempts(entry: Entry, attempts: ReadonlyMap<CollectionId, InFlightAttempt>): Promise<void> {
|
|
770
|
+
try {
|
|
771
|
+
await this.journal?.driving(entry.id, driveState(entry, attempts));
|
|
772
|
+
} catch (err) {
|
|
773
|
+
log('offline:journal-write-failed entry=%s action=%s step=send error=%s', entry.id, entry.actionId, describe(err));
|
|
774
|
+
throw new RecordAttemptFailed(err);
|
|
775
|
+
}
|
|
776
|
+
// NOTE: in doubt is STICKY — set here, once the first attempt of the entry is about to be sent,
|
|
777
|
+
// and cleared only by a terminal outcome or by `torn-not-saved`. A cycle's own refreshes
|
|
778
|
+
// resolve its earlier attempts (each refresh finds a landed own entry, and an attempt at an
|
|
779
|
+
// unchanged revision finishes an earlier one as the same action), so the last cycle's
|
|
780
|
+
// classification speaks for the cycle; but a coordinator lagging its cohort can still hide a
|
|
781
|
+
// minority-held tail, and the sticky rule is what keeps that from ever discarding a
|
|
782
|
+
// possibly-landed entry. While in doubt the entry is neither discardable nor expirable, and is
|
|
783
|
+
// re-driven only under its action id with its retained attempts as `resume`.
|
|
784
|
+
entry.inDoubt = true;
|
|
785
|
+
for (const [collectionId, attempt] of attempts) {
|
|
786
|
+
entry.retained.set(collectionId, attempt);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
/** Settle `entry` committed, reporting every collection it wrote: `written`, the last submission's
|
|
791
|
+
* answer, beside what earlier partial commits committed. The journal hears the status first
|
|
792
|
+
* (docs/offline-sync.md §Write ordering, rule 3). */
|
|
793
|
+
private async committed(entry: Entry, written?: TransactionDurability): Promise<void> {
|
|
794
|
+
this.claimRemoval(entry);
|
|
795
|
+
this.pendingRefusals.delete(entry.id);
|
|
796
|
+
entry.retained.clear();
|
|
797
|
+
entry.inDoubt = false;
|
|
798
|
+
const status: JournalSettledStatus = {
|
|
799
|
+
kind: 'committed',
|
|
800
|
+
entryId: entry.id,
|
|
801
|
+
durability: entryDurability(entry, written),
|
|
802
|
+
effect: effectOf(entry),
|
|
803
|
+
};
|
|
804
|
+
await this.journalSettled(status);
|
|
805
|
+
this.emit(status);
|
|
806
|
+
this.waker.resetBackoff();
|
|
807
|
+
this.publishState();
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* Apply a failed submission by its class (docs/offline-sync.md §Classifying an attempt's outcome):
|
|
812
|
+
*
|
|
813
|
+
* | class | transition |
|
|
814
|
+
* |---|---|
|
|
815
|
+
* | the journal's write-ahead threw | `queued`, `backoff` — nothing was sent |
|
|
816
|
+
* | `ActionRefusedError` thrown directly (the head's recorded refusal) | `failed`, `refusalReason` |
|
|
817
|
+
* | `cohort-unreachable` | `queued`, `cohort`; the timer re-checks the gate |
|
|
818
|
+
* | `contention`, `expired`, `in-doubt`, `unknown` | `queued`, `backoff`; `unknown` is also logged and counted |
|
|
819
|
+
* | `guard-refused`, `rejected`, `too-large` | `failed`, with the class's reason |
|
|
820
|
+
* | `torn-not-saved` | `queued`, `backoff`; doubt cleared (unless part of the entry is committed), a fresh action id, retained attempts cleared, the drive record rewritten |
|
|
821
|
+
* | `partial` | `partially-committed`, then `queued`, `backoff`, in doubt, under the same id unless its cause is `torn-not-saved` ({@link partiallyCommitted}) |
|
|
822
|
+
*
|
|
823
|
+
* `SealedSegmentsError` out of a submission means the head alignment invariant broke — a queue
|
|
824
|
+
* defect — and is thrown out of the loop, which stops the drain.
|
|
825
|
+
*
|
|
826
|
+
* **One action id for the entry's whole life, with one exception.** Storage treats a re-send of
|
|
827
|
+
* the same action at the same revision as finishing it and at a new revision as a new write, so
|
|
828
|
+
* re-driving under the same id is always safe and keeps the member-side slot hold the write earned
|
|
829
|
+
* by losing. The exception is `torn-not-saved`: a re-drive under that id would find its own
|
|
830
|
+
* stranded log entry on every refresh, re-send it, and be told again that a rival built past it —
|
|
831
|
+
* a loop. A fresh id lets the next refresh treat the stranded entry as a stranger's and move on,
|
|
832
|
+
* which is exactly the proof `final: true` gives. A refused pend does NOT mint a fresh id here: it
|
|
833
|
+
* proves nothing about the cycle's earlier attempts under the same id.
|
|
834
|
+
*
|
|
835
|
+
* @returns whether the entry stays queued (wait out the timer) rather than having settled. */
|
|
836
|
+
private async applyFailure(entry: Entry, error: unknown): Promise<boolean> {
|
|
837
|
+
if (error instanceof SealedSegmentsError) throw error;
|
|
838
|
+
if (error instanceof RecordAttemptFailed) {
|
|
839
|
+
this.setQueued(entry, 'backoff', error.cause);
|
|
840
|
+
return true;
|
|
841
|
+
}
|
|
842
|
+
if (error instanceof QueueDisposedError && this.disposed) {
|
|
843
|
+
// The queue's own abort of the submission in flight: not a failure of the write, so not
|
|
844
|
+
// its `lastError` either.
|
|
845
|
+
this.setQueued(entry, undefined);
|
|
846
|
+
return true;
|
|
847
|
+
}
|
|
848
|
+
if (error instanceof ActionRefusedError) {
|
|
849
|
+
await this.failAndCascade(entry, refusalReason(error), error, this.pendingRefusals.get(entry.id)?.causedBy);
|
|
850
|
+
return false;
|
|
851
|
+
}
|
|
852
|
+
const outcome = classifyCommitFailure(error);
|
|
853
|
+
switch (outcome.kind) {
|
|
854
|
+
case 'cohort-unreachable': {
|
|
855
|
+
this.setQueued(entry, 'cohort', error);
|
|
856
|
+
return true;
|
|
857
|
+
}
|
|
858
|
+
case 'contention':
|
|
859
|
+
case 'expired':
|
|
860
|
+
case 'in-doubt': {
|
|
861
|
+
this.setQueued(entry, 'backoff', error);
|
|
862
|
+
return true;
|
|
863
|
+
}
|
|
864
|
+
case 'torn-not-saved': {
|
|
865
|
+
// The torn attempt is provably not saved, but a partial commit's half is: an entry with a
|
|
866
|
+
// committed part stays in doubt, so it is never discarded or expired.
|
|
867
|
+
entry.inDoubt = entry.committed.size > 0;
|
|
868
|
+
this.renewIdentity(entry);
|
|
869
|
+
await this.journalDriving(entry, 'renewal');
|
|
870
|
+
this.setQueued(entry, 'backoff', error);
|
|
871
|
+
return true;
|
|
872
|
+
}
|
|
873
|
+
case 'partial': {
|
|
874
|
+
return this.partiallyCommitted(entry, outcome.error);
|
|
875
|
+
}
|
|
876
|
+
case 'unknown': {
|
|
877
|
+
this.unclassifiedFailures++;
|
|
878
|
+
log('offline:unclassified-failure entry=%s action=%s error=%s', entry.id, entry.actionId, describe(error));
|
|
879
|
+
this.setQueued(entry, 'backoff', error);
|
|
880
|
+
return true;
|
|
881
|
+
}
|
|
882
|
+
case 'guard-refused': {
|
|
883
|
+
await this.failAndCascade(entry, outcome.refusal, outcome.error, this.pendingRefusals.get(entry.id)?.causedBy);
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
886
|
+
case 'rejected': {
|
|
887
|
+
await this.failAndCascade(entry, 'rejected', outcome.error);
|
|
888
|
+
return false;
|
|
889
|
+
}
|
|
890
|
+
case 'too-large': {
|
|
891
|
+
await this.failAndCascade(entry, 'too-large', outcome.error);
|
|
892
|
+
return false;
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
/**
|
|
898
|
+
* A submission committed part of the entry (docs/offline-sync.md §Partial commits). The committed
|
|
899
|
+
* collections' segments are already consumed and their attempts resolved, so they are recorded on
|
|
900
|
+
* the entry and leave `retained`; the rest are still sealed and first, their attempts retained, and
|
|
901
|
+
* the next submission drives exactly them, under the same action id (a fresh one when the rest
|
|
902
|
+
* failed `torn-not-saved`). The entry is in doubt from here to its terminal outcome, so it is
|
|
903
|
+
* neither discarded nor expired. The journal's drive record is rewritten once the entry's state is
|
|
904
|
+
* settled — after a renewal too, so one write carries both — and `partially-committed` is emitted
|
|
905
|
+
* in place of the `queued` status this transition would otherwise report.
|
|
906
|
+
*
|
|
907
|
+
* @returns whether the entry stays queued: `false` only when no collection still holds its segment
|
|
908
|
+
* (the failure came after every collection was saved), which settles it committed. */
|
|
909
|
+
private async partiallyCommitted(entry: Entry, error: CoordinatorPartialCommitError): Promise<boolean> {
|
|
910
|
+
for (const [collectionId, durability] of error.durability.collections) {
|
|
911
|
+
recordCollectionDurability(entry.committed, collectionId, durability);
|
|
912
|
+
entry.retained.delete(collectionId);
|
|
913
|
+
}
|
|
914
|
+
entry.inDoubt = true;
|
|
915
|
+
const pending = this.remainingCollections(entry).map(collection => collection.id);
|
|
916
|
+
if (pending.length === 0) {
|
|
917
|
+
await this.committed(entry);
|
|
918
|
+
return false;
|
|
919
|
+
}
|
|
920
|
+
// The coordinator reports any failure after a saved participant as a partial commit, so the
|
|
921
|
+
// rest may have failed `torn-not-saved`; re-driving it under the same id would only meet that
|
|
922
|
+
// stranded entry again.
|
|
923
|
+
if (classifyCommitFailure(error.reason).kind === 'torn-not-saved') this.renewIdentity(entry);
|
|
924
|
+
await this.journalDriving(entry, 'partial-commit');
|
|
925
|
+
const committedCollections = [...entry.committed.keys()];
|
|
926
|
+
log('offline:partially-committed entry=%s action=%s committed=%o pending=%o reason=%s',
|
|
927
|
+
entry.id, entry.actionId, committedCollections, pending, describe(error.reason));
|
|
928
|
+
entry.phase = 'queued';
|
|
929
|
+
entry.waitingFor = 'backoff';
|
|
930
|
+
entry.lastError = error;
|
|
931
|
+
entry.lastQueued = queuedStatus(entry);
|
|
932
|
+
this.emit({
|
|
933
|
+
kind: 'partially-committed',
|
|
934
|
+
entryId: entry.id,
|
|
935
|
+
committedCollections,
|
|
936
|
+
pendingCollections: pending,
|
|
937
|
+
cause: error.reason instanceof Error ? error.reason : error,
|
|
938
|
+
});
|
|
939
|
+
this.publishState();
|
|
940
|
+
return true;
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
/** After a `torn-not-saved` outcome: nothing sent under the entry's action id is saved or will
|
|
944
|
+
* land, so the next submission goes under a fresh one with nothing to resume. A partly committed
|
|
945
|
+
* entry's committed half keeps the old id; only the remainder commits under the new one. The
|
|
946
|
+
* caller writes the journal's drive record afterwards ({@link journalDriving}). */
|
|
947
|
+
private renewIdentity(entry: Entry): void {
|
|
948
|
+
entry.actionId = mintId();
|
|
949
|
+
entry.retained.clear();
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// ── the journal ────────────────────────────────────────────────────────────────────────────
|
|
953
|
+
|
|
954
|
+
/** Rule 1 of docs/offline-sync.md §Write ordering: the entry record, before the entry is admitted.
|
|
955
|
+
* On a throw every collection sealed is unsealed, in reverse sealing order, and the error
|
|
956
|
+
* rethrown — nothing is queued. */
|
|
957
|
+
private async journalQueued(entry: Entry): Promise<void> {
|
|
958
|
+
if (this.journal === undefined) return;
|
|
959
|
+
try {
|
|
960
|
+
await this.journal.queued(entryRecord(entry));
|
|
961
|
+
} catch (err) {
|
|
962
|
+
for (const collection of [...entry.collections].reverse()) {
|
|
963
|
+
collection.unseal(entry.id);
|
|
964
|
+
}
|
|
965
|
+
log('offline:journal-write-failed entry=%s step=queued error=%s: the enqueue fails and nothing is queued', entry.id, describe(err));
|
|
966
|
+
throw err;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
/** The drive record after a partial commit or an identity renewal (rules 2 and 4 of
|
|
971
|
+
* docs/offline-sync.md §Write ordering). A throw is logged and the drain goes on: the entry stays
|
|
972
|
+
* at the head in doubt either way, and the next send's write-ahead repeats the state. The write
|
|
973
|
+
* before a send is {@link recordAttempts}', whose throw aborts the send. */
|
|
974
|
+
private async journalDriving(entry: Entry, step: 'partial-commit' | 'renewal'): Promise<void> {
|
|
975
|
+
try {
|
|
976
|
+
await this.journal?.driving(entry.id, driveState(entry));
|
|
977
|
+
} catch (err) {
|
|
978
|
+
log('offline:journal-write-failed entry=%s action=%s step=%s error=%s: the drain goes on; the next send rewrites the record',
|
|
979
|
+
entry.id, entry.actionId, step, describe(err));
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/** Rule 3 of docs/offline-sync.md §Write ordering: the terminal status, before it is emitted. A
|
|
984
|
+
* throw is logged and swallowed here, never where the outcome was decided: the network has the
|
|
985
|
+
* write, or the segments are gone, and a stale record is what the restore after a restart
|
|
986
|
+
* resolves. */
|
|
987
|
+
private async journalSettled(status: JournalSettledStatus): Promise<void> {
|
|
988
|
+
try {
|
|
989
|
+
await this.journal?.settled(status);
|
|
990
|
+
} catch (err) {
|
|
991
|
+
log('offline:journal-write-failed entry=%s step=settled error=%s: the outcome stands; a restart resolves the stale record',
|
|
992
|
+
status.entryId, describe(err));
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
// ── restoring ──────────────────────────────────────────────────────────────────────────────
|
|
997
|
+
// NOTE: this file is near 1300 lines (`wc -l`). The restore machinery below (the hold, the
|
|
998
|
+
// registration by seq, the staging and its rollback) reads and writes the queue's private state
|
|
999
|
+
// directly, which is why it stays here; if the file grows again, it is the section to extract
|
|
1000
|
+
// behind a small interface over `list`, `byId` and the hold.
|
|
1001
|
+
|
|
1002
|
+
/** One {@link restore}, run on the restore chain: take the hold, wait for the loop to park and the
|
|
1003
|
+
* removal chain to drain, register, stage, admit, release. */
|
|
1004
|
+
private async restoreHeld(init: RestoreInit, stage: (collection: Collection<any>) => Promise<RestoredSegment>): Promise<QueuedEntry | undefined> {
|
|
1005
|
+
const hold = this.takeHold();
|
|
1006
|
+
try {
|
|
1007
|
+
await hold.parked;
|
|
1008
|
+
await this.settledRemovals();
|
|
1009
|
+
this.assertAccepting();
|
|
1010
|
+
const entry = this.registerRestored(init);
|
|
1011
|
+
this.restoring = entry;
|
|
1012
|
+
let outcomes: Map<CollectionId, RestoredSegment>;
|
|
1013
|
+
try {
|
|
1014
|
+
outcomes = await this.stageRestored(entry, stage);
|
|
1015
|
+
} finally {
|
|
1016
|
+
this.restoring = undefined;
|
|
1017
|
+
}
|
|
1018
|
+
return await this.admitRestored(entry, outcomes);
|
|
1019
|
+
} finally {
|
|
1020
|
+
this.releaseHold(hold);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
/** Hold the drain: the loop parks at the top of its next iteration. A loop that is not running
|
|
1025
|
+
* (never started, or stopped) or is parked on an empty queue is parked already; one waiting out
|
|
1026
|
+
* the re-check timer is woken so it reaches the hold without waiting the timer out. */
|
|
1027
|
+
private takeHold(): DrainHold {
|
|
1028
|
+
let release!: () => void;
|
|
1029
|
+
let signalParked!: () => void;
|
|
1030
|
+
const held = new Promise<void>(resolve => { release = resolve; });
|
|
1031
|
+
const parked = new Promise<void>(resolve => { signalParked = resolve; });
|
|
1032
|
+
const hold: DrainHold = { held, release, parked, signalParked };
|
|
1033
|
+
this.hold = hold;
|
|
1034
|
+
if (this.loop === undefined || this.loopStopped || this.waitingForWork) {
|
|
1035
|
+
signalParked();
|
|
1036
|
+
} else {
|
|
1037
|
+
this.wake();
|
|
1038
|
+
}
|
|
1039
|
+
return hold;
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
/** Release the hold and wake the loop, which goes on from the top of the iteration it parked in. */
|
|
1043
|
+
private releaseHold(hold: DrainHold): void {
|
|
1044
|
+
if (this.hold === hold) this.hold = undefined;
|
|
1045
|
+
hold.release();
|
|
1046
|
+
if (this.loop !== undefined) this.wake();
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
/** Register a restored entry: built from the record and the drive state, inserted by `seq`, with
|
|
1050
|
+
* `waitingFor` set by position and `nextSeq` raised past it. */
|
|
1051
|
+
private registerRestored(init: RestoreInit): Entry {
|
|
1052
|
+
if (this.byId.has(init.id)) throw new Error(`entry ${init.id} is already queued`);
|
|
1053
|
+
if (this.list.some(queued => queued.seq === init.seq)) {
|
|
1054
|
+
throw new Error(`entry ${init.id} cannot be restored at seq ${init.seq}, which another queued entry holds`);
|
|
1055
|
+
}
|
|
1056
|
+
const drive = init.drive;
|
|
1057
|
+
const entry: Entry = {
|
|
1058
|
+
id: init.id,
|
|
1059
|
+
seq: init.seq,
|
|
1060
|
+
queuedAt: init.queuedAt,
|
|
1061
|
+
expiresAt: init.expiresAt,
|
|
1062
|
+
collections: init.collections,
|
|
1063
|
+
actionId: drive?.actionId ?? mintId(),
|
|
1064
|
+
// The count of submissions is not journaled. A drive record is written only ahead of a send,
|
|
1065
|
+
// so one proves at least one submission was made; it is reported as exactly one, the least
|
|
1066
|
+
// that is true, rather than invented.
|
|
1067
|
+
attempts: drive === undefined ? 0 : 1,
|
|
1068
|
+
inDoubt: drive !== undefined && (drive.attempts.size > 0 || drive.committed.size > 0),
|
|
1069
|
+
retained: new Map(drive?.attempts),
|
|
1070
|
+
committed: new Map(drive?.committed),
|
|
1071
|
+
applied: new Map(),
|
|
1072
|
+
phase: 'queued',
|
|
1073
|
+
waitingFor: undefined,
|
|
1074
|
+
lastError: undefined,
|
|
1075
|
+
lastQueued: undefined,
|
|
1076
|
+
expiryDeferredLogged: false,
|
|
1077
|
+
};
|
|
1078
|
+
this.insertBySeq(entry);
|
|
1079
|
+
this.byId.set(entry.id, entry);
|
|
1080
|
+
this.nextSeq = Math.max(this.nextSeq, entry.seq + 1);
|
|
1081
|
+
return entry;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
/** Insert `entry` into the list by `seq` and re-derive every entry's "waiting for an earlier
|
|
1085
|
+
* entry" flag from its position: the head waits for no earlier entry, every other entry does. */
|
|
1086
|
+
private insertBySeq(entry: Entry): void {
|
|
1087
|
+
const index = this.list.findIndex(queued => queued.seq > entry.seq);
|
|
1088
|
+
this.list.splice(index < 0 ? this.list.length : index, 0, entry);
|
|
1089
|
+
this.list.forEach((queued, position) => {
|
|
1090
|
+
if (position === 0) {
|
|
1091
|
+
if (queued.waitingFor === 'earlier-entry') queued.waitingFor = undefined;
|
|
1092
|
+
} else {
|
|
1093
|
+
queued.waitingFor = 'earlier-entry';
|
|
1094
|
+
}
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
/** Run `stage` for each of the entry's collections, in order; on a throw, roll the registration
|
|
1099
|
+
* back — the entry leaves the queue and every segment already staged is dropped — and rethrow. */
|
|
1100
|
+
private async stageRestored(entry: Entry, stage: (collection: Collection<any>) => Promise<RestoredSegment>): Promise<Map<CollectionId, RestoredSegment>> {
|
|
1101
|
+
const outcomes = new Map<CollectionId, RestoredSegment>();
|
|
1102
|
+
try {
|
|
1103
|
+
for (const collection of entry.collections) {
|
|
1104
|
+
outcomes.set(collection.id, await stage(collection));
|
|
1105
|
+
}
|
|
1106
|
+
} catch (err) {
|
|
1107
|
+
await this.unregisterRestored(entry, outcomes);
|
|
1108
|
+
throw err;
|
|
1109
|
+
}
|
|
1110
|
+
return outcomes;
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
/** The rollback of a restore whose `stage` threw: the entry leaves the queue, and each segment
|
|
1114
|
+
* `stage` put in place is dropped — every one, a drop that throws being logged, since the stage's
|
|
1115
|
+
* own error is the one the caller hears. */
|
|
1116
|
+
private async unregisterRestored(entry: Entry, outcomes: ReadonlyMap<CollectionId, RestoredSegment>): Promise<void> {
|
|
1117
|
+
this.claimRemoval(entry);
|
|
1118
|
+
this.pendingRefusals.delete(entry.id);
|
|
1119
|
+
for (const collection of entry.collections) {
|
|
1120
|
+
if (outcomes.get(collection.id)?.kind !== 'staged' || !collection.sealedEntries().includes(entry.id)) continue;
|
|
1121
|
+
try {
|
|
1122
|
+
await collection.dropSealed(entry.id);
|
|
1123
|
+
} catch (err) {
|
|
1124
|
+
log('offline:restore-rollback-drop-failed entry=%s collection=%s error=%s', entry.id, collection.id, describe(err));
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
/** Admit a registered entry by what `stage` answered (see {@link restore}). */
|
|
1130
|
+
private async admitRestored(entry: Entry, outcomes: ReadonlyMap<CollectionId, RestoredSegment>): Promise<QueuedEntry | undefined> {
|
|
1131
|
+
let staged = 0;
|
|
1132
|
+
for (const [collectionId, outcome] of outcomes) {
|
|
1133
|
+
if (outcome.kind === 'already-committed') {
|
|
1134
|
+
// A class the drive record knew is kept; one it did not is unknown, not fabricated.
|
|
1135
|
+
if (!entry.committed.has(collectionId)) entry.committed.set(collectionId, undefined);
|
|
1136
|
+
entry.inDoubt = true;
|
|
1137
|
+
continue;
|
|
1138
|
+
}
|
|
1139
|
+
staged++;
|
|
1140
|
+
if (outcome.refused !== undefined && !this.pendingRefusals.has(entry.id)) {
|
|
1141
|
+
this.pendingRefusals.set(entry.id, { error: outcome.refused, causedBy: undefined });
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
if (staged === 0) {
|
|
1145
|
+
log('offline:restored-already-committed entry=%s action=%s collections=%o', entry.id, entry.actionId, [...outcomes.keys()]);
|
|
1146
|
+
// Every collection is in `entry.committed` by now, each with the class the drive record knew
|
|
1147
|
+
// or none; nothing new was written, so nothing is merged over them.
|
|
1148
|
+
await this.committed(entry);
|
|
1149
|
+
return undefined;
|
|
1150
|
+
}
|
|
1151
|
+
this.emitQueued(entry);
|
|
1152
|
+
this.publishState();
|
|
1153
|
+
this.startDrain();
|
|
1154
|
+
return snapshot(entry);
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
// ── removing ───────────────────────────────────────────────────────────────────────────────
|
|
1158
|
+
|
|
1159
|
+
/** Fail `entry` and then act on every refusal its removal reported, in queue order, until none is
|
|
1160
|
+
* pending (docs/offline-sync.md §Removing an entry). */
|
|
1161
|
+
private async failAndCascade(entry: Entry, reason: QueuedFailureReason, error: Error, causedBy?: EntryId): Promise<void> {
|
|
1162
|
+
await this.fail(entry, reason, error, causedBy);
|
|
1163
|
+
await this.processPendingRefusals();
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
/** Fail one entry: removed from the queue at once, its segments dropped on the removal chain.
|
|
1167
|
+
* Nothing happens for an entry a `discard` has already claimed. */
|
|
1168
|
+
private async fail(entry: Entry, reason: QueuedFailureReason, error: Error, causedBy: EntryId | undefined): Promise<void> {
|
|
1169
|
+
if (!this.claimRemoval(entry)) return;
|
|
1170
|
+
await this.chainRemoval(entry, reason, error, causedBy);
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
/** Take `entry` out of the queue (the synchronous half of every removal, a commit's included), so
|
|
1174
|
+
* the drain cannot pick it again. Answers `false` when it was already taken. */
|
|
1175
|
+
private claimRemoval(entry: Entry): boolean {
|
|
1176
|
+
if (!this.byId.delete(entry.id)) return false;
|
|
1177
|
+
const index = this.list.indexOf(entry);
|
|
1178
|
+
if (index >= 0) this.list.splice(index, 1);
|
|
1179
|
+
const head = this.list[0];
|
|
1180
|
+
if (head !== undefined && head.waitingFor === 'earlier-entry') head.waitingFor = undefined;
|
|
1181
|
+
return true;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
/** Queue `entry`'s drops behind every removal already chained; the returned promise is that
|
|
1185
|
+
* removal's own, so its throw reaches the caller while the chain itself never rejects. */
|
|
1186
|
+
private chainRemoval(entry: Entry, reason: QueuedFailureReason, error: Error, causedBy: EntryId | undefined): Promise<void> {
|
|
1187
|
+
this.pendingRemovals++;
|
|
1188
|
+
const removal = this.removals.then(() => this.remove(entry, reason, error, causedBy));
|
|
1189
|
+
const settled = (): void => { this.pendingRemovals--; };
|
|
1190
|
+
this.removals = removal.then(settled, settled);
|
|
1191
|
+
return removal;
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
/** The asynchronous half of a failure: drop the entry's segment from every collection that still
|
|
1195
|
+
* holds one (a collection the entry had no segment in, or whose segment a partial commit consumed,
|
|
1196
|
+
* is skipped), then emit `failed` naming those collections, and the ones a partial commit
|
|
1197
|
+
* committed whatever triggered the failure — the journal hearing the status first. Each drop
|
|
1198
|
+
* replays and may report refusals of later entries through {@link onSealedRefused}, which
|
|
1199
|
+
* {@link causeOf} attributes while `removing` names this entry. */
|
|
1200
|
+
private async remove(entry: Entry, reason: QueuedFailureReason, error: Error, causedBy: EntryId | undefined): Promise<void> {
|
|
1201
|
+
this.pendingRefusals.delete(entry.id);
|
|
1202
|
+
const changedCollections: CollectionId[] = [];
|
|
1203
|
+
this.removing = entry;
|
|
1204
|
+
try {
|
|
1205
|
+
for (const collection of entry.collections) {
|
|
1206
|
+
if (!collection.sealedEntries().includes(entry.id)) continue;
|
|
1207
|
+
changedCollections.push(collection.id);
|
|
1208
|
+
await collection.dropSealed(entry.id);
|
|
1209
|
+
}
|
|
1210
|
+
} finally {
|
|
1211
|
+
this.removing = undefined;
|
|
1212
|
+
}
|
|
1213
|
+
if (entry.committed.size > 0) {
|
|
1214
|
+
log('offline:partially-committed-entry-failed entry=%s action=%s committed=%o dropped=%o reason=%s: the network holds part of the transaction',
|
|
1215
|
+
entry.id, entry.actionId, [...entry.committed.keys()], changedCollections, reason);
|
|
1216
|
+
}
|
|
1217
|
+
const status: JournalSettledStatus = {
|
|
1218
|
+
kind: 'failed',
|
|
1219
|
+
entryId: entry.id,
|
|
1220
|
+
reason,
|
|
1221
|
+
error,
|
|
1222
|
+
changedCollections,
|
|
1223
|
+
...(causedBy === undefined ? {} : { causedBy }),
|
|
1224
|
+
...committedField(entry),
|
|
1225
|
+
};
|
|
1226
|
+
await this.journalSettled(status);
|
|
1227
|
+
this.emit(status);
|
|
1228
|
+
this.publishState();
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
/**
|
|
1232
|
+
* Which earlier entry a refusal of `entry` is attributed to: the nearest one, in queue order, that
|
|
1233
|
+
* is being removed right now or already has a refusal pending; `undefined` when there is none (a
|
|
1234
|
+
* rival's commit refused the entry directly, found by a reader's refresh — or a restore's
|
|
1235
|
+
* insertion did, which removes nothing).
|
|
1236
|
+
*
|
|
1237
|
+
* Nearest rather than the entry being dropped, because one replay refuses every later segment
|
|
1238
|
+
* that no longer applies at once: dropping E1 re-stages E2 (refused) and then E3 over state
|
|
1239
|
+
* without E2, so E3's refusal is reported during E1's drop even though it is E2's image E3 built on.
|
|
1240
|
+
* Naming the nearest failing entry gives the chain E3 ← E2 ← E1 an application can follow, where
|
|
1241
|
+
* the literal "the entry whose drop ran the replay" would flatten it to E3 ← E1, E2 ← E1.
|
|
1242
|
+
*/
|
|
1243
|
+
private causeOf(entry: Entry): EntryId | undefined {
|
|
1244
|
+
let cause: Entry | undefined;
|
|
1245
|
+
for (const earlier of this.list) {
|
|
1246
|
+
if (earlier.seq >= entry.seq) break;
|
|
1247
|
+
if (this.pendingRefusals.has(earlier.id)) cause = earlier;
|
|
1248
|
+
}
|
|
1249
|
+
const removing = this.removing;
|
|
1250
|
+
if (removing !== undefined && removing.seq < entry.seq && (cause === undefined || removing.seq > cause.seq)) {
|
|
1251
|
+
cause = removing;
|
|
1252
|
+
}
|
|
1253
|
+
return cause?.id;
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/** The removal chain, once every removal chained so far — including ones chained while waiting —
|
|
1257
|
+
* has finished. */
|
|
1258
|
+
private async settledRemovals(): Promise<void> {
|
|
1259
|
+
let chain: Promise<void>;
|
|
1260
|
+
do {
|
|
1261
|
+
chain = this.removals;
|
|
1262
|
+
await chain;
|
|
1263
|
+
} while (chain !== this.removals);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
// ── stopping ───────────────────────────────────────────────────────────────────────────────
|
|
1267
|
+
|
|
1268
|
+
private settleDrained(): void {
|
|
1269
|
+
for (const waiter of this.drainedWaiters.splice(0)) waiter.resolve();
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
/** The drain has met a defect: stop it, reject every `whenDrained`, and leave every entry as it
|
|
1273
|
+
* is. Segments stay sealed in their collections, so nothing the application committed is lost.
|
|
1274
|
+
* NOTE: a removal that threw is read as a defect too, though `dropSealed`'s replay reads blocks
|
|
1275
|
+
* through the collection's source and so can throw for a read that could not be served (a cache
|
|
1276
|
+
* miss while the cohort is unreachable). Fine while every replay runs over blocks the staging
|
|
1277
|
+
* read into the cache; if `offline:drain-stopped` is ever seen with such an error, a removal
|
|
1278
|
+
* needs to be retried on the re-check timer rather than stop the drain. */
|
|
1279
|
+
private stopOnDefect(error: unknown): void {
|
|
1280
|
+
this.defect = error;
|
|
1281
|
+
log('offline:drain-stopped error=%s', describe(error));
|
|
1282
|
+
for (const waiter of this.drainedWaiters.splice(0)) waiter.reject(error);
|
|
1283
|
+
this.publishState();
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
/** A removal off the loop's path (a `discard`) threw: the loop meets it on its next iteration. */
|
|
1287
|
+
private noteDefect(error: unknown): void {
|
|
1288
|
+
this.defect ??= error;
|
|
1289
|
+
this.wake();
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
// ── reporting ──────────────────────────────────────────────────────────────────────────────
|
|
1293
|
+
|
|
1294
|
+
/** Back to `queued` after a gate check or a failed submission; emitted only when something an
|
|
1295
|
+
* application can see changed since the last `queued` status. */
|
|
1296
|
+
private setQueued(entry: Entry, waitingFor: EntryWaitingFor | undefined, lastError?: unknown): void {
|
|
1297
|
+
entry.phase = 'queued';
|
|
1298
|
+
entry.waitingFor = waitingFor;
|
|
1299
|
+
if (lastError !== undefined) entry.lastError = lastError;
|
|
1300
|
+
if (!queuedStatusChanged(entry.lastQueued, queuedStatus(entry))) return;
|
|
1301
|
+
this.emitQueued(entry);
|
|
1302
|
+
this.publishState();
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
private emitQueued(entry: Entry): void {
|
|
1306
|
+
const status = queuedStatus(entry);
|
|
1307
|
+
entry.lastQueued = status;
|
|
1308
|
+
this.emit(status);
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
private emit(status: EntryStatus): void {
|
|
1312
|
+
for (const listener of this.statusListeners) {
|
|
1313
|
+
try {
|
|
1314
|
+
listener(status);
|
|
1315
|
+
} catch (err) {
|
|
1316
|
+
log('offline:listener-threw listener=status entry=%s kind=%s error=%s', status.entryId, status.kind, describe(err));
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
/** Publish the queue-wide state when it differs from the last one published. */
|
|
1322
|
+
private publishState(): void {
|
|
1323
|
+
const state = this.queueState();
|
|
1324
|
+
const last = this.lastState;
|
|
1325
|
+
if (last !== undefined
|
|
1326
|
+
&& last.depth === state.depth
|
|
1327
|
+
&& last.oldestQueuedAt === state.oldestQueuedAt
|
|
1328
|
+
&& last.draining === state.draining
|
|
1329
|
+
&& last.waitingFor === state.waitingFor
|
|
1330
|
+
&& last.unclassifiedFailures === state.unclassifiedFailures) {
|
|
1331
|
+
return;
|
|
1332
|
+
}
|
|
1333
|
+
this.lastState = state;
|
|
1334
|
+
for (const listener of this.stateListeners) {
|
|
1335
|
+
try {
|
|
1336
|
+
listener(state);
|
|
1337
|
+
} catch (err) {
|
|
1338
|
+
log('offline:listener-threw listener=state error=%s', describe(err));
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
}
|