@941design/marmot 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +709 -0
- package/LICENSE +21 -0
- package/README.md +33 -2
- package/dist/account/admin.d.ts +72 -0
- package/dist/account/admin.js +209 -0
- package/dist/account/device-deps.d.ts +379 -0
- package/dist/account/device-deps.js +36 -0
- package/dist/account/device-lineage.d.ts +199 -0
- package/dist/account/device-lineage.js +390 -0
- package/dist/account/device-registry.d.ts +93 -0
- package/dist/account/device-registry.js +1290 -0
- package/dist/account/device-rotation.d.ts +61 -0
- package/dist/account/device-rotation.js +827 -0
- package/dist/account/enrolment.d.ts +339 -0
- package/dist/account/enrolment.js +1247 -0
- package/dist/account/identity-proof.d.ts +79 -0
- package/dist/account/identity-proof.js +264 -0
- package/dist/account/index.d.ts +17 -0
- package/dist/account/index.js +16 -0
- package/dist/account/restore-rotation.d.ts +21 -0
- package/dist/account/restore-rotation.js +22 -0
- package/dist/account/same-account-authorization.d.ts +86 -0
- package/dist/account/same-account-authorization.js +180 -0
- package/dist/account/teardown.d.ts +127 -0
- package/dist/account/teardown.js +856 -0
- package/dist/client/before-create-group.d.ts +18 -0
- package/dist/client/before-create-group.js +65 -0
- package/dist/client/before-publish-hooks.d.ts +121 -0
- package/dist/client/before-publish-hooks.js +180 -0
- package/dist/client/client-config.d.ts +42 -0
- package/dist/client/client-config.js +13 -0
- package/dist/client/commit-redrive.d.ts +68 -0
- package/dist/client/commit-redrive.js +221 -0
- package/dist/client/component-update.d.ts +43 -0
- package/dist/client/component-update.js +53 -0
- package/dist/client/confirm-hooks.d.ts +50 -0
- package/dist/client/confirm-hooks.js +59 -0
- package/dist/client/create-client.d.ts +1022 -0
- package/dist/client/create-client.js +1125 -0
- package/dist/client/group-disband.d.ts +28 -0
- package/dist/client/group-disband.js +64 -0
- package/dist/client/group-history.d.ts +435 -0
- package/dist/client/group-history.js +1295 -0
- package/dist/client/group-profile.d.ts +31 -0
- package/dist/client/group-profile.js +62 -0
- package/dist/client/group-records.d.ts +485 -0
- package/dist/client/group-records.js +1034 -0
- package/dist/client/group-removal.d.ts +36 -0
- package/dist/client/group-removal.js +127 -0
- package/dist/client/group-remove-member.d.ts +49 -0
- package/dist/client/group-remove-member.js +163 -0
- package/dist/client/group-runtime.d.ts +388 -0
- package/dist/client/group-runtime.js +2701 -0
- package/dist/client/group-snapshot.d.ts +98 -0
- package/dist/client/group-snapshot.js +453 -0
- package/dist/client/history-pages.d.ts +107 -0
- package/dist/client/history-pages.js +194 -0
- package/dist/client/inbox-send.d.ts +19 -0
- package/dist/client/inbox-send.js +71 -0
- package/dist/client/index.d.ts +21 -0
- package/dist/client/index.js +13 -0
- package/dist/client/ingest.d.ts +82 -0
- package/dist/client/ingest.js +1550 -0
- package/dist/client/lifecycle-events.d.ts +63 -0
- package/dist/client/lifecycle-events.js +35 -0
- package/dist/client/opaque-candidates.d.ts +81 -0
- package/dist/client/opaque-candidates.js +169 -0
- package/dist/client/own-change-outbox.d.ts +66 -0
- package/dist/client/own-change-outbox.js +156 -0
- package/dist/client/pending-leaf.d.ts +116 -0
- package/dist/client/pending-leaf.js +378 -0
- package/dist/client/permissions.d.ts +9 -0
- package/dist/client/permissions.js +12 -0
- package/dist/client/pipeline.d.ts +554 -0
- package/dist/client/pipeline.js +3545 -0
- package/dist/client/reentrancy-guard.d.ts +96 -0
- package/dist/client/reentrancy-guard.js +153 -0
- package/dist/client/rekey.d.ts +58 -0
- package/dist/client/rekey.js +149 -0
- package/dist/client/transport-policy.d.ts +181 -0
- package/dist/client/transport-policy.js +214 -0
- package/dist/history/equivocation.d.ts +55 -0
- package/dist/history/equivocation.js +113 -0
- package/dist/history/fencing.d.ts +25 -0
- package/dist/history/fencing.js +61 -0
- package/dist/history/hex.d.ts +3 -0
- package/dist/history/hex.js +10 -0
- package/dist/history/index.d.ts +73 -0
- package/dist/history/index.js +31 -0
- package/dist/history/notifications.d.ts +270 -0
- package/dist/history/notifications.js +264 -0
- package/dist/history/ordering-key.d.ts +24 -0
- package/dist/history/ordering-key.js +101 -0
- package/dist/history/own-change-fork.d.ts +112 -0
- package/dist/history/own-change-fork.js +196 -0
- package/dist/history/own-change.d.ts +106 -0
- package/dist/history/own-change.js +146 -0
- package/dist/history/policy.d.ts +48 -0
- package/dist/history/policy.js +52 -0
- package/dist/history/retained-window.d.ts +215 -0
- package/dist/history/retained-window.js +565 -0
- package/dist/inbox/demux.d.ts +90 -0
- package/dist/inbox/demux.js +78 -0
- package/dist/inbox/handler-registry.d.ts +42 -0
- package/dist/inbox/handler-registry.js +37 -0
- package/dist/inbox/index.d.ts +11 -0
- package/dist/inbox/index.js +10 -0
- package/dist/inbox/seen-set.d.ts +82 -0
- package/dist/inbox/seen-set.js +202 -0
- package/dist/inbox/subscription.d.ts +236 -0
- package/dist/inbox/subscription.js +644 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +9 -0
- package/dist/invitations/competing-welcome.d.ts +223 -0
- package/dist/invitations/competing-welcome.js +128 -0
- package/dist/invitations/decline.d.ts +61 -0
- package/dist/invitations/decline.js +143 -0
- package/dist/invitations/expect-welcome.d.ts +66 -0
- package/dist/invitations/expect-welcome.js +138 -0
- package/dist/invitations/index.d.ts +16 -0
- package/dist/invitations/index.js +16 -0
- package/dist/invitations/invite.d.ts +303 -0
- package/dist/invitations/invite.js +966 -0
- package/dist/invitations/offers.d.ts +79 -0
- package/dist/invitations/offers.js +115 -0
- package/dist/invitations/on-proposal.d.ts +47 -0
- package/dist/invitations/on-proposal.js +40 -0
- package/dist/invitations/pending-proposal-policy.d.ts +52 -0
- package/dist/invitations/pending-proposal-policy.js +139 -0
- package/dist/invitations/proposal-deadline.d.ts +133 -0
- package/dist/invitations/proposal-deadline.js +164 -0
- package/dist/invitations/relay-planes.d.ts +43 -0
- package/dist/invitations/relay-planes.js +70 -0
- package/dist/invitations/welcome-processor.d.ts +253 -0
- package/dist/invitations/welcome-processor.js +1180 -0
- package/dist/key-packages/device-key.d.ts +131 -0
- package/dist/key-packages/device-key.js +272 -0
- package/dist/key-packages/device-rotation-records.d.ts +83 -0
- package/dist/key-packages/device-rotation-records.js +173 -0
- package/dist/key-packages/group-refs.d.ts +40 -0
- package/dist/key-packages/group-refs.js +79 -0
- package/dist/key-packages/index.d.ts +18 -0
- package/dist/key-packages/index.js +18 -0
- package/dist/key-packages/key-package-ref.d.ts +16 -0
- package/dist/key-packages/key-package-ref.js +21 -0
- package/dist/key-packages/observation-stream.d.ts +4 -0
- package/dist/key-packages/observation-stream.js +120 -0
- package/dist/key-packages/offer-rumor.d.ts +26 -0
- package/dist/key-packages/offer-rumor.js +69 -0
- package/dist/key-packages/publisher.d.ts +174 -0
- package/dist/key-packages/publisher.js +1247 -0
- package/dist/key-packages/select-key-package.d.ts +24 -0
- package/dist/key-packages/select-key-package.js +90 -0
- package/dist/key-packages/slot-manager.d.ts +167 -0
- package/dist/key-packages/slot-manager.js +327 -0
- package/dist/key-packages/validate-key-package-event.d.ts +3 -0
- package/dist/key-packages/validate-key-package-event.js +49 -0
- package/dist/media/cache.d.ts +14 -0
- package/dist/media/cache.js +10 -0
- package/dist/media/index.d.ts +52 -0
- package/dist/media/index.js +71 -0
- package/dist/mls.d.ts +15 -0
- package/dist/mls.js +15 -0
- package/dist/shared/account-pubkey.d.ts +4 -0
- package/dist/shared/account-pubkey.js +10 -0
- package/dist/shared/errors.d.ts +21 -0
- package/dist/shared/errors.js +85 -0
- package/dist/shared/hex.d.ts +4 -0
- package/dist/shared/hex.js +13 -0
- package/dist/shared/index.d.ts +14 -0
- package/dist/shared/index.js +14 -0
- package/dist/shared/kinds.d.ts +8 -0
- package/dist/shared/kinds.js +28 -0
- package/dist/shared/publish-classification.d.ts +26 -0
- package/dist/shared/publish-classification.js +31 -0
- package/dist/storage/envelope.d.ts +39 -0
- package/dist/storage/envelope.js +201 -0
- package/dist/storage/index.d.ts +227 -0
- package/dist/storage/index.js +459 -0
- package/dist/storage/indexeddb/idb-types.d.ts +68 -0
- package/dist/storage/indexeddb/idb-types.js +1 -0
- package/dist/storage/indexeddb/index.d.ts +17 -0
- package/dist/storage/indexeddb/index.js +141 -0
- package/dist/storage/lease.d.ts +41 -0
- package/dist/storage/lease.js +117 -0
- package/dist/storage/memory/deterministic-signer.d.ts +27 -0
- package/dist/storage/memory/deterministic-signer.js +65 -0
- package/dist/storage/memory/index.d.ts +32 -0
- package/dist/storage/memory/index.js +87 -0
- package/dist/storage/memory/lock-manager.d.ts +8 -0
- package/dist/storage/memory/lock-manager.js +71 -0
- package/package.json +66 -3
|
@@ -0,0 +1,1550 @@
|
|
|
1
|
+
import { deserializeApplicationData, deserializeClientState, getMarmotGroupView, getNostrGroupIdHex, serializeClientState, } from '@941design/marmot-engine/core';
|
|
2
|
+
import { enforceStagedProposals, enforceStateTransition } from '../account/index.js';
|
|
3
|
+
import { canonicalPath, captureWindow, equivocationNotification, fencingErrorIn, orderingKeyOf, restrictWindow, REWIND_HORIZON, sameWindow, tagOf, writeEquivocation, writeFence, writeWindow, EQUIVOCATION_EXTENSION, FENCE_EXTENSION, } from '../history/index.js';
|
|
4
|
+
import { announceNotifications, deliverTransition, fenceGroup, noteEquivocations, planTransition, recordEquivocations, recordTransition, retainedLedgerNodes, WITHDRAWN_DISPOSITION } from './group-history.js';
|
|
5
|
+
import { recordSelectedDisband } from './group-disband.js';
|
|
6
|
+
import { listPendingIntentRecords, markRatchetSkipDelivered, OWN_COMMIT_EXTENSION, pruneOwnCommitMarkers, pruneTerminalLedger, RATCHET_SKIP_EXTENSION, readIngestCursor, readTerminalLedgerEntry, writeGroupState, writeIngestCursor, writeTerminalLedgerEntry, } from './group-records.js';
|
|
7
|
+
import { clearRemoval, nextHeldRemoval, REMOVAL_EXTENSION, settlesAt, writeRemoval } from './group-removal.js';
|
|
8
|
+
import { clearOpaque, isEmptyOpaque, nextOpaque, OPAQUE_EXTENSION, writeOpaque } from './opaque-candidates.js';
|
|
9
|
+
import { clearPendingCommit, dropPendingLeaf, ledgerOwnCopies, PENDING_COMMIT_EXTENSION, pendingWatch, reinstatable, reinstatePendingLeaf } from './pending-leaf.js';
|
|
10
|
+
import { EXCLUDED_PROPOSALS_EXTENSION, exclusionsAt, readExclusions } from '../invitations/index.js';
|
|
11
|
+
import { MarmotError } from '../shared/index.js';
|
|
12
|
+
import { assertNotDisposed, backoffMs, electionOf, expireDeferredProposals, failAwaitingIntents, MAX_TIMER_DELAY_MS, releaseAwaitingSync, releaseQueueCommitHold, releaseSelfRemoveElectionWait, reportStrandedSelfUpdateReenqueues, resumePending, scheduleDeferredReenqueueRetry, scheduleSelfRemoveCommit, scheduleWithdrawnRedrives, takeStrandedSelfUpdateReenqueues, } from './pipeline.js';
|
|
13
|
+
import { HISTORY_LIVE_AFTER_WALK_PASSES, HISTORY_MAX_PAGES, HISTORY_REWALK_SLACK_PAGES, fetchHistoryPages } from './history-pages.js';
|
|
14
|
+
import { CURSOR_FUTURE_SKEW_S, CURSOR_REDELIVERY_MARGIN_S, GROUP_EVENT_KIND, LEDGER_RETENTION_S, absentPins, groupEventFilter, groupRelayPlane, nextCursor, } from './transport-policy.js';
|
|
15
|
+
/**
|
|
16
|
+
* The tree-fed convergence pass over the engine's current history tree, as an ordinary batch of no events (proof gate,
|
|
17
|
+
* persistence, notifications). The engine runs it on every ingest; this runs it on its own, after a rebuild changed
|
|
18
|
+
* the tree without any envelope to trigger one (the pending-restore resolution, client/pending-leaf.ts). Inside a
|
|
19
|
+
* group operation.
|
|
20
|
+
*/
|
|
21
|
+
export async function treeFedPass(rt) {
|
|
22
|
+
await ingestEvents(rt, [], { treeFed: true });
|
|
23
|
+
}
|
|
24
|
+
/** The engine's deferral reason for a commit naming a proposal not staged here (`deferredReasons.missingParent`). */
|
|
25
|
+
const MISSING_PARENT = 'missing_parent';
|
|
26
|
+
function hTagOf(event) {
|
|
27
|
+
for (const tag of event.tags)
|
|
28
|
+
if (tag[0] === 'h')
|
|
29
|
+
return tag[1];
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Ledger dispositions of authenticated, terminally processed events — a
|
|
34
|
+
* ledger hit on one may advance the cursor. `past-epoch` is deliberately
|
|
35
|
+
* absent: the engine skips such an envelope without MLS-authenticating it.
|
|
36
|
+
*/
|
|
37
|
+
const AUTHENTICATED_TERMINAL = new Set(['applied', 'application', 'duplicate', 'own']);
|
|
38
|
+
/**
|
|
39
|
+
* Maps one engine-yielded result to a client `IngestDisposition`, or
|
|
40
|
+
* `undefined` for a result kind that carries no envelope and so has no
|
|
41
|
+
* per-envelope disposition of its own: `autoCommit` and `leafEquivocation`
|
|
42
|
+
* (both handled by the caller before this is reached), and
|
|
43
|
+
* `appliedNotifications`, `stateRevalidated`, `stateInvalidated` (keyed by
|
|
44
|
+
* commit digest rather than by envelope).
|
|
45
|
+
*/
|
|
46
|
+
async function outcomeFor(rt, result) {
|
|
47
|
+
const terminalOk = (envelope, disposition) => ({
|
|
48
|
+
eventId: disposition.eventId,
|
|
49
|
+
createdAt: envelope.created_at,
|
|
50
|
+
disposition,
|
|
51
|
+
terminal: true,
|
|
52
|
+
holds: false,
|
|
53
|
+
advances: true,
|
|
54
|
+
});
|
|
55
|
+
const terminalRejected = (envelope, reason) => {
|
|
56
|
+
const eventId = rt.engine.peeler.idOf(envelope);
|
|
57
|
+
return { eventId, createdAt: envelope.created_at, disposition: { kind: 'rejected', eventId, reason }, terminal: true, holds: false, advances: false };
|
|
58
|
+
};
|
|
59
|
+
const retryable = (envelope, reason, unauthenticated = false) => {
|
|
60
|
+
const eventId = rt.engine.peeler.idOf(envelope);
|
|
61
|
+
return { eventId, createdAt: envelope.created_at, disposition: { kind: 'retryable', eventId, reason }, terminal: false, holds: true, advances: false, unauthenticated };
|
|
62
|
+
};
|
|
63
|
+
switch (result.kind) {
|
|
64
|
+
case 'processed': {
|
|
65
|
+
const eventId = rt.engine.peeler.idOf(result.envelope);
|
|
66
|
+
if (result.result.kind === 'applicationMessage') {
|
|
67
|
+
const rumor = deserializeApplicationData(result.result.message);
|
|
68
|
+
// AC-HIST-1: `account` from the credential ts-mls authenticated (resolved against the message's own epoch),
|
|
69
|
+
// the epoch and generation from the message itself, the accepted branch from the retained state it decrypted under.
|
|
70
|
+
const orderingKey = await orderingKeyOf({
|
|
71
|
+
cs: rt.engine.ciphersuite,
|
|
72
|
+
message: result.message,
|
|
73
|
+
senderCredential: result.result.senderCredential,
|
|
74
|
+
senderLeafIndex: result.result.senderLeafIndex ?? -1,
|
|
75
|
+
states: [rt.engine.state, ...rt.engine.retainedStates()],
|
|
76
|
+
});
|
|
77
|
+
return { ...terminalOk(result.envelope, { kind: 'application', eventId, rumor, orderingKey }), message: { type: 'message', eventId, rumor, orderingKey } };
|
|
78
|
+
}
|
|
79
|
+
return terminalOk(result.envelope, { kind: 'applied', eventId });
|
|
80
|
+
}
|
|
81
|
+
case 'removed':
|
|
82
|
+
// DEVIATION: no distinct client-level disposition exists yet for "this
|
|
83
|
+
// commit removed us" (that surfaces as a `removed` client event once
|
|
84
|
+
// S9/S10 land) — treated as an ordinary applied commit for ingest
|
|
85
|
+
// bookkeeping; the tombstoned state itself still gets persisted below.
|
|
86
|
+
return terminalOk(result.envelope, { kind: 'applied', eventId: rt.engine.peeler.idOf(result.envelope) });
|
|
87
|
+
case 'rejected':
|
|
88
|
+
return terminalRejected(result.envelope, result.reason ?? 'rejected');
|
|
89
|
+
case 'skipped':
|
|
90
|
+
if (result.reason === 'duplicate' || result.reason === 'self-echo') {
|
|
91
|
+
return terminalOk(result.envelope, { kind: 'duplicate', eventId: rt.engine.peeler.idOf(result.envelope) });
|
|
92
|
+
}
|
|
93
|
+
if (result.reason === 'past-epoch') {
|
|
94
|
+
// Terminal (ledgered, so never re-fed), but the engine skipped it
|
|
95
|
+
// without authenticating it: it must not advance the cursor.
|
|
96
|
+
return { ...terminalOk(result.envelope, { kind: 'duplicate', eventId: rt.engine.peeler.idOf(result.envelope) }), advances: false, ledgerAs: 'past-epoch' };
|
|
97
|
+
}
|
|
98
|
+
return terminalRejected(result.envelope, result.reason);
|
|
99
|
+
case 'deferred':
|
|
100
|
+
return retryable(result.envelope, result.reason);
|
|
101
|
+
case 'refused':
|
|
102
|
+
// The engine refuses decrypt-failure envelopes for capacity too, so a
|
|
103
|
+
// refusal says nothing about authenticity: the evictable tier.
|
|
104
|
+
return retryable(result.envelope, 'capacity', true);
|
|
105
|
+
case 'unreadable': {
|
|
106
|
+
// A decrypt failure the engine surfaces (rather than pooling) is still
|
|
107
|
+
// retryable: a later commit may unlock its epoch.
|
|
108
|
+
if (result.decryptFailure === true)
|
|
109
|
+
return retryable(result.envelope, 'undecryptable', true);
|
|
110
|
+
// AC-HIST-4: replaying a commit whose path this device has no key for — its own confirmed commit after a
|
|
111
|
+
// restore — is not an unreadable event but a fenced group. Not ledgered: the fence ends all processing.
|
|
112
|
+
const fence = fencingErrorIn(result.errors);
|
|
113
|
+
if (fence !== undefined)
|
|
114
|
+
return { ...terminalRejected(result.envelope, 'fenced'), terminal: false, fence };
|
|
115
|
+
return terminalRejected(result.envelope, 'unreadable');
|
|
116
|
+
}
|
|
117
|
+
case 'invalidated': {
|
|
118
|
+
// A rewind abandoned the branch this earlier-delivered message decrypted on (M7): it is withdrawn
|
|
119
|
+
// (`message-withdrawn`, and from the history backend) — see planTransition. Not a disposition of
|
|
120
|
+
// the envelope being ingested: `record` keeps the message's own ledger entry for the withdrawal.
|
|
121
|
+
const eventId = rt.engine.peeler.idOf(result.envelope);
|
|
122
|
+
return {
|
|
123
|
+
eventId,
|
|
124
|
+
createdAt: result.envelope.created_at,
|
|
125
|
+
disposition: { kind: 'rejected', eventId, reason: 'invalidated' },
|
|
126
|
+
terminal: false,
|
|
127
|
+
holds: false,
|
|
128
|
+
advances: false,
|
|
129
|
+
retracted: true,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
case 'autoCommit':
|
|
133
|
+
case 'leafEquivocation':
|
|
134
|
+
case 'appliedNotifications':
|
|
135
|
+
case 'stateRevalidated':
|
|
136
|
+
case 'stateInvalidated':
|
|
137
|
+
return undefined;
|
|
138
|
+
default: {
|
|
139
|
+
// A result kind added to the engine's union must be mapped here, not silently dropped.
|
|
140
|
+
const unhandled = result;
|
|
141
|
+
throw new Error(`ingest: unhandled engine result kind ${String(unhandled.kind)}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
// The ADR-012 post-apply proof gate around one engine ingest call
|
|
147
|
+
// ---------------------------------------------------------------------------
|
|
148
|
+
/**
|
|
149
|
+
* Whether `next` can differ from `parent` in anything the proof gate checks:
|
|
150
|
+
* the leaf set (ratchet tree), the epoch, or the staged proposals. An
|
|
151
|
+
* application message only ratchets `secretTree` and shares all three by
|
|
152
|
+
* reference, so it skips the gate.
|
|
153
|
+
*/
|
|
154
|
+
function needsProofGate(parent, next) {
|
|
155
|
+
return (parent.groupContext.epoch !== next.groupContext.epoch ||
|
|
156
|
+
parent.ratchetTree !== next.ratchetTree ||
|
|
157
|
+
parent.unappliedProposals !== next.unappliedProposals);
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* One `engine.ingest(envelopes)` drain followed by the ADR-012 post-apply
|
|
161
|
+
* gate (AC-PROOF-2/3): the engine's `adminPolicyCallback` never sees a
|
|
162
|
+
* commit's UpdatePath leaf, so the resulting state is re-checked here against
|
|
163
|
+
* what the engine actually produced — and so are newly staged proposals,
|
|
164
|
+
* which the callback judges too since S9's
|
|
165
|
+
* `seam-engine-ingest-3-proposal-policy` (this re-check is the backstop). On failure the engine is rebuilt over a FRESH copy of
|
|
166
|
+
* the parent bytes (never hand one `ClientState` object to two engines —
|
|
167
|
+
* ts-mls mutates `secretTree` in place), discarding outcomes this drain
|
|
168
|
+
* surfaced, and the envelopes the discarded engine was pooling (future-epoch
|
|
169
|
+
* messages awaiting a commit) are re-queued into the rebuilt one.
|
|
170
|
+
*
|
|
171
|
+
* `settle: false` (batch settle, spec Design Decision 6) says more events of the caller's batch follow: the engine
|
|
172
|
+
* holds a could-win opaque candidate instead of fencing it, and the caller's final settling call decides it.
|
|
173
|
+
*/
|
|
174
|
+
async function ingestThroughGate(rt, envelopes, settle = true) {
|
|
175
|
+
const parent = rt.engine.state;
|
|
176
|
+
const pooledBefore = rt.engine.pendingEnvelopes();
|
|
177
|
+
/**
|
|
178
|
+
* The engine's held opaque candidates and `U` before this drain (LD-2): a rollback keeps exactly that, like the
|
|
179
|
+
* state and the history. For the batch's first drain it IS the pre-batch record (`rt.opaque`, advanced only after a
|
|
180
|
+
* commit); for a later one it also holds what the batch's earlier steps learned — those steps' provisional
|
|
181
|
+
* `skipped` is ledgered with the batch, so the candidate it held must not be lost with this drain's rollback.
|
|
182
|
+
*/
|
|
183
|
+
const opaqueBefore = { record: rt.engine.opaque };
|
|
184
|
+
/** The engine's history before this drain: a rollback keeps exactly that (the batch's earlier steps included). */
|
|
185
|
+
const treeBefore = new Set(rt.engine.history.tags());
|
|
186
|
+
const rollbackWindow = async () => restrictWindow(await captureWindow(rt.engine.history, parent, undefined, rt.window), treeBefore);
|
|
187
|
+
const outcomes = [];
|
|
188
|
+
const evidence = [];
|
|
189
|
+
for await (const result of rt.engine.ingest([...envelopes], ...(settle ? [] : [{ settle: false }]))) {
|
|
190
|
+
if (result.kind === 'leafEquivocation') {
|
|
191
|
+
evidence.push(result.evidence);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
if (result.kind === 'autoCommit') {
|
|
195
|
+
// The engine elected this client to commit the staged SelfRemoves (AC-ADMIN-9). Its staged
|
|
196
|
+
// commit is not durable: rolled back to Stable here, and — once this batch is persisted —
|
|
197
|
+
// rebuilt as a durable pipeline commit of the same staged set (scheduleSelfRemoveCommit).
|
|
198
|
+
rt.engine.publishFailed(result.pending);
|
|
199
|
+
rt.selfRemoveElection = electionOf(rt.engine.state);
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
const outcome = await outcomeFor(rt, result);
|
|
203
|
+
if (outcome !== undefined)
|
|
204
|
+
outcomes.push(outcome);
|
|
205
|
+
}
|
|
206
|
+
assertNotDisposed(rt);
|
|
207
|
+
const next = rt.engine.state;
|
|
208
|
+
const fence = outcomes.find((o) => o.fence !== undefined)?.fence;
|
|
209
|
+
if (fence !== undefined) {
|
|
210
|
+
// A fenced group never processes anything again, so nothing this drain applied besides the fencing
|
|
211
|
+
// envelope is kept: the engine goes back to the parent (a pooled envelope the drain applied first
|
|
212
|
+
// never reached the proof gate) and only the fence marker is persisted — with any equivocation evidence
|
|
213
|
+
// the drain reported (architecture.md OQ-6): evidence is a fact about signed commits the device saw, not
|
|
214
|
+
// about the state it keeps, and recording it changes nothing the fence decides.
|
|
215
|
+
if (next !== parent)
|
|
216
|
+
await rt.rebuildEngine(deserializeClientState(serializeClientState(parent)), await rollbackWindow(), opaqueBefore);
|
|
217
|
+
return { outcomes: outcomes.filter((o) => o.fence !== undefined), evidence, fence, rejected: false, stateChanged: false, rejectedPooled: [] };
|
|
218
|
+
}
|
|
219
|
+
if (next === parent)
|
|
220
|
+
return { outcomes, evidence, rejected: false, stateChanged: false, rejectedPooled: [] };
|
|
221
|
+
if (!needsProofGate(parent, next))
|
|
222
|
+
return { outcomes, evidence, rejected: false, stateChanged: true, rejectedPooled: [] };
|
|
223
|
+
const failures = [...enforceStateTransition(parent, next), ...enforceStagedProposals(parent, next)];
|
|
224
|
+
if (failures.length === 0)
|
|
225
|
+
return { outcomes, evidence, rejected: false, stateChanged: true, rejectedPooled: [] };
|
|
226
|
+
// The rebuilt engine starts with an empty pool, and with the history it held at `parent` — earlier steps of this
|
|
227
|
+
// batch included, which the durable window (still the pre-batch tip) cannot name. The drain's outcomes go; its
|
|
228
|
+
// evidence stays (OQ-6): it is MLS-authenticated signed commits the device saw, and the ADR-012 gate judges Marmot
|
|
229
|
+
// rules on the resulting state, not authentication — refusing the transition does not unsee the equivocation.
|
|
230
|
+
// The record handed back is the drain-start one, except in the terminal case (nothing was pooled, so the rejected
|
|
231
|
+
// envelope is the only input and is never re-fed: it is ledgered `rejected`): an id that LEFT `U` during this drain
|
|
232
|
+
// was opened by a held state (the engine drops an id from `U` only on a decrypt), and a rejected commit that was
|
|
233
|
+
// opened is no hidden edge, so it must not stay in `U` from the drain-start record. Read from the engine's own
|
|
234
|
+
// bookkeeping at the point of decryption, never inferred from the inputs: an id still undecryptable at rollback
|
|
235
|
+
// (junk) stays, and so does every id of an envelope that is re-fed below (the nested runs reach this line alone).
|
|
236
|
+
// Single-trigger invariant: every caller hands ONE event per call (ingestEvents, requeueIndividually, the nested
|
|
237
|
+
// re-run), so with an empty pool the rejected envelope is `envelopes[0]`; the reduction is guarded on that and only
|
|
238
|
+
// ever removes the trigger's own id, never another id that happened to leave `U`.
|
|
239
|
+
const triggerIds = new Set(envelopes.map((e) => e.id));
|
|
240
|
+
// A saturated `U` (the engine's cap) holds no ids: an id missing from it says nothing about a decrypt, so none is
|
|
241
|
+
// taken as decrypted — the drain-start record is handed back whole (never an undercount).
|
|
242
|
+
const saturatedNow = rt.engine.opaque?.undecryptableSaturated === true;
|
|
243
|
+
const decrypted = new Set(saturatedNow ? [] : (opaqueBefore.record?.undecryptableIds ?? []).filter((id) => triggerIds.has(id) && !(rt.engine.opaque?.undecryptableIds ?? []).includes(id)));
|
|
244
|
+
const handedBack = pooledBefore.length === 0 && envelopes.length === 1 && opaqueBefore.record !== undefined && decrypted.size > 0
|
|
245
|
+
? { record: { ...opaqueBefore.record, undecryptableIds: opaqueBefore.record.undecryptableIds.filter((id) => !decrypted.has(id)) } }
|
|
246
|
+
: opaqueBefore;
|
|
247
|
+
await rt.rebuildEngine(deserializeClientState(serializeClientState(parent)), await rollbackWindow(), handedBack);
|
|
248
|
+
if (pooledBefore.length === 0)
|
|
249
|
+
return { outcomes: [], evidence, rejected: true, stateChanged: false, rejectedPooled: [] };
|
|
250
|
+
// The drain also applied envelopes the engine had pooled, so the failing transition may be one of
|
|
251
|
+
// theirs rather than the trigger's. Re-run the trigger alone, then feed the pooled envelopes back one
|
|
252
|
+
// at a time: each is gated on its own, so only an envelope whose own transition fails is rejected.
|
|
253
|
+
const alone = await ingestThroughGate(rt, envelopes, settle);
|
|
254
|
+
const requeued = await requeueIndividually(rt, pooledBefore, settle);
|
|
255
|
+
// The first fence wins, and none may be dropped by the aggregation: a held opaque candidate that fenced in an isolated
|
|
256
|
+
// replay is released by the engine once it yielded the verdict, and the batch must persist the marker for it.
|
|
257
|
+
const nestedFence = alone.fence ?? requeued.fence;
|
|
258
|
+
return {
|
|
259
|
+
...(nestedFence === undefined ? {} : { fence: nestedFence }),
|
|
260
|
+
outcomes: [...alone.outcomes, ...requeued.outcomes],
|
|
261
|
+
evidence: [...evidence, ...alone.evidence, ...requeued.evidence],
|
|
262
|
+
rejected: alone.rejected,
|
|
263
|
+
stateChanged: alone.stateChanged || requeued.stateChanged,
|
|
264
|
+
rejectedPooled: [...alone.rejectedPooled, ...requeued.rejectedPooled],
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
async function requeueIndividually(rt, envelopes, settle = true) {
|
|
268
|
+
const outcomes = [];
|
|
269
|
+
const evidence = [];
|
|
270
|
+
const rejectedPooled = [];
|
|
271
|
+
let stateChanged = false;
|
|
272
|
+
let fence;
|
|
273
|
+
for (const envelope of envelopes) {
|
|
274
|
+
const step = await ingestThroughGate(rt, [envelope], settle);
|
|
275
|
+
fence ??= step.fence;
|
|
276
|
+
outcomes.push(...step.outcomes);
|
|
277
|
+
evidence.push(...step.evidence);
|
|
278
|
+
rejectedPooled.push(...step.rejectedPooled);
|
|
279
|
+
if (step.rejected)
|
|
280
|
+
rejectedPooled.push(envelope);
|
|
281
|
+
stateChanged ||= step.stateChanged;
|
|
282
|
+
}
|
|
283
|
+
return { ...(fence === undefined ? {} : { fence }), outcomes, evidence, rejected: false, stateChanged, rejectedPooled };
|
|
284
|
+
}
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
// ingestEvents — MarmotGroup.ingest's body, and every sync batch's body
|
|
287
|
+
// ---------------------------------------------------------------------------
|
|
288
|
+
/** Per-batch cursor bookkeeping (see transport-policy `nextCursor`). */
|
|
289
|
+
class CursorBatch {
|
|
290
|
+
preJoinBefore;
|
|
291
|
+
advanced = [];
|
|
292
|
+
/** Unauthenticated holds (the cursor's evictable tier). */
|
|
293
|
+
held = new Map();
|
|
294
|
+
/** Group-content holds (the cursor's pinned tier); disjoint from `held`. */
|
|
295
|
+
pinned = new Map();
|
|
296
|
+
/** Events dropped unclassified as `group-busy` (the cursor's bounded busy tier). */
|
|
297
|
+
busy = new Map();
|
|
298
|
+
released = new Set();
|
|
299
|
+
constructor(preJoinBefore) {
|
|
300
|
+
this.preJoinBefore = preJoinBefore;
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* Holds `eventId` for redelivery in the tier its latest classification
|
|
304
|
+
* names (a `group-busy` pin that turns out undecryptable moves to `held`).
|
|
305
|
+
*/
|
|
306
|
+
hold(eventId, createdAt, unauthenticated) {
|
|
307
|
+
this.held.delete(eventId);
|
|
308
|
+
this.pinned.delete(eventId);
|
|
309
|
+
// An unauthenticated event stamped before this member joined can never
|
|
310
|
+
// be read (no pre-join epoch key is ever held): holding it would pin the
|
|
311
|
+
// cursor for nothing — and any earlier hold on it is given up.
|
|
312
|
+
if (unauthenticated && this.preJoinBefore !== undefined && createdAt < this.preJoinBefore) {
|
|
313
|
+
this.released.add(eventId);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
this.released.delete(eventId);
|
|
317
|
+
(unauthenticated ? this.held : this.pinned).set(eventId, createdAt);
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Holds `eventId`, dropped as `group-busy` before anything authenticated
|
|
321
|
+
* it, in the bounded busy tier — unless stamped provably pre-join (it could
|
|
322
|
+
* never be read). An earlier classification of the id keeps precedence.
|
|
323
|
+
*/
|
|
324
|
+
holdBusy(eventId, createdAt) {
|
|
325
|
+
if (this.preJoinBefore !== undefined && createdAt < this.preJoinBefore)
|
|
326
|
+
return;
|
|
327
|
+
this.busy.set(eventId, createdAt);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* The engine's convergence continuation: input that arrived after a convergence
|
|
332
|
+
* deadline is retained and only released by `driveConvergence()`. The released
|
|
333
|
+
* envelopes are not trusted as applied by that drain — the engine is rolled back
|
|
334
|
+
* to the durable state and they (with anything it was pooling) are re-ingested
|
|
335
|
+
* through {@link ingestEvents}, so the proof gate, persistence, ledger and cursor
|
|
336
|
+
* apply exactly as for any other delivery.
|
|
337
|
+
*/
|
|
338
|
+
export async function drainConvergence(rt) {
|
|
339
|
+
const pooledBefore = rt.engine.pendingEnvelopes();
|
|
340
|
+
const results = await rt.engine.driveConvergence();
|
|
341
|
+
// BACKLOG marmot-disband-settled-store-divergence: a drive that selected a peer's disband ended the group. The
|
|
342
|
+
// marker is persisted (a reload stays disbanded) and announced; the engine keeps its terminal in-memory state —
|
|
343
|
+
// nothing it selected is persisted as group state (no proof gate judged it) and nothing is re-ingested.
|
|
344
|
+
const disband = await recordSelectedDisband(rt);
|
|
345
|
+
if (disband !== undefined) {
|
|
346
|
+
await recordEquivocations(rt, results.flatMap((r) => (r.kind === 'leafEquivocation' ? [r.evidence] : []))).catch(() => undefined);
|
|
347
|
+
stopGroupSync(rt);
|
|
348
|
+
announceNotifications(rt, [disband]);
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
if (results.length === 0)
|
|
352
|
+
return;
|
|
353
|
+
// ADR-016 D6 (held opaque candidates): a drive can fence — a candidate held `skipped` became able to win during the
|
|
354
|
+
// pass, and the engine yields its `unreadable` carrying the lost-path-secrets error. The fence is group state, not a
|
|
355
|
+
// per-event disposition: persisted BEFORE the engine goes back to the durable state, or the reload below would
|
|
356
|
+
// discard the only thing that knows the group must stop.
|
|
357
|
+
const driveFence = results.flatMap((r) => (r.kind === 'unreadable' ? [fencingErrorIn(r.errors)] : [])).find((e) => e !== undefined);
|
|
358
|
+
if (driveFence !== undefined) {
|
|
359
|
+
await fenceFromDrive(rt, driveFence, results.flatMap((r) => (r.kind === 'leafEquivocation' ? [r.evidence] : [])));
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
// ADR-016: a drive whose only results are the disband settlement's equivocation report (nothing released) records
|
|
363
|
+
// it and stops, exactly as a drive returning nothing does — the settlement alone never re-reports it to a batch.
|
|
364
|
+
if (results.every((r) => r.kind === 'leafEquivocation')) {
|
|
365
|
+
await recordEquivocations(rt, results.flatMap((r) => (r.kind === 'leafEquivocation' ? [r.evidence] : []))).catch(() => undefined);
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
const released = [];
|
|
369
|
+
const evidence = [];
|
|
370
|
+
for (const result of results) {
|
|
371
|
+
if (result.kind === 'leafEquivocation') {
|
|
372
|
+
evidence.push(result.evidence);
|
|
373
|
+
continue;
|
|
374
|
+
}
|
|
375
|
+
if (result.kind === 'autoCommit') {
|
|
376
|
+
rt.engine.publishFailed(result.pending);
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
const envelope = result.envelope;
|
|
380
|
+
if (envelope !== undefined && !released.some((e) => e.id === envelope.id))
|
|
381
|
+
released.push(envelope);
|
|
382
|
+
}
|
|
383
|
+
// What the drive itself set aside (released input deferred again, e.g. a commit of a later epoch) is in the pool
|
|
384
|
+
// only now, and surfaces as no result: taken before the reload discards that pool, or it waits for a redelivery.
|
|
385
|
+
const pooledAfter = rt.engine.pendingEnvelopes();
|
|
386
|
+
// The drive moved the engine onto a result no proof gate has judged: it goes back to the durable state before
|
|
387
|
+
// anything else can fail (AC-PROOF-2 — the engine never runs on state that was not gated and persisted).
|
|
388
|
+
await rt.reloadFromDurable();
|
|
389
|
+
// ADR-016: the drive's equivocation evidence is recorded now — when nothing is released and nothing was pooled,
|
|
390
|
+
// no batch follows to re-report it — and kept whatever the re-ingest's proof gate decides (evidence is kept on
|
|
391
|
+
// every path, OQ-6). Recorded before the re-ingest, so the batch finds its keys present and announces nothing
|
|
392
|
+
// twice; best-effort, since the state is already safe: a refused write leaves the evidence to the re-ingest's
|
|
393
|
+
// own selection pass (which records it in the batch's transaction) or to the next pass over its fork.
|
|
394
|
+
await recordEquivocations(rt, evidence).catch(() => undefined);
|
|
395
|
+
const again = [...released];
|
|
396
|
+
for (const e of [...pooledAfter, ...pooledBefore])
|
|
397
|
+
if (!again.some((r) => r.id === e.id))
|
|
398
|
+
again.push(e);
|
|
399
|
+
if (again.length > 0)
|
|
400
|
+
await ingestEvents(rt, again);
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* The drive fenced the group (see {@link drainConvergence}): the fence marker is written first, in one transaction
|
|
404
|
+
* with the deletion of the held opaque record (a fenced group keeps none: a reload must not resurrect them), then
|
|
405
|
+
* the runtime is fenced, its sync stopped, its awaiting-sync intents failed, and the engine put back on the durable
|
|
406
|
+
* state. Nothing the drive selected or released is applied or re-ingested: a fenced group processes nothing again.
|
|
407
|
+
* Equivocation evidence the same drive reported is recorded best-effort after the marker (OQ-6), as everywhere a
|
|
408
|
+
* drain's evidence is kept beside a fence. Idempotent for a group that is fenced already.
|
|
409
|
+
*/
|
|
410
|
+
async function fenceFromDrive(rt, fence, reported) {
|
|
411
|
+
if (rt.fence === undefined) {
|
|
412
|
+
// A failed marker write still fences this session and marks the runtime for a reload (see fenceGroup).
|
|
413
|
+
await fenceGroup(rt, fence).finally(() => stopGroupSync(rt));
|
|
414
|
+
}
|
|
415
|
+
await rt.reloadFromDurable();
|
|
416
|
+
await recordEquivocations(rt, reported).catch(() => undefined);
|
|
417
|
+
await failAwaitingIntents(rt).catch(() => undefined);
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* S20: settles the group's revisable removal once its window passed (client/group-removal.ts): the engine treats
|
|
421
|
+
* it as final from now on (input `self-evicted` again), the record is deleted, and the rumors queued while it was
|
|
422
|
+
* revisable are dropped by the next queue drain. Runs inside a group op, before anything else touches the engine.
|
|
423
|
+
* AC-RACE-1: a re-enqueued self-update left envelope-less is ALSO stranded once this settles for good — every
|
|
424
|
+
* drive path for it refuses a `removedFromGroup` group — and is failed the same way fencing fails one.
|
|
425
|
+
*/
|
|
426
|
+
async function settleDueRemoval(rt) {
|
|
427
|
+
const due = settlesAt(rt.removal, rt.ctx.removalSettleMs);
|
|
428
|
+
if (due === undefined || rt.ctx.clock.now() < due)
|
|
429
|
+
return;
|
|
430
|
+
// Read before the settle: settleRemoval() forgets the removal and its senders. Best-effort (decision 19).
|
|
431
|
+
const actor = await rt.engine.removalActor().catch(() => undefined);
|
|
432
|
+
// One transaction: the marker never goes without its stranded re-enqueues (a crash in between would leave them
|
|
433
|
+
// durable with nothing left to settle, so never reported).
|
|
434
|
+
const stranded = await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
435
|
+
await clearRemoval(tx.extension(REMOVAL_EXTENSION));
|
|
436
|
+
return takeStrandedSelfUpdateReenqueues(tx);
|
|
437
|
+
}));
|
|
438
|
+
rt.engine.settleRemoval();
|
|
439
|
+
rt.removal = undefined;
|
|
440
|
+
await reportStrandedSelfUpdateReenqueues(rt, stranded, new Error(`group ${rt.groupId}'s removal of the local leaf settled for good`));
|
|
441
|
+
rt.scheduleQueueDrain();
|
|
442
|
+
// Owner decision 19 (2026-10-29): the removal is final now — announce it (unless this device asked for it).
|
|
443
|
+
await rt.removalSettled(actor);
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* S20: settles the group's held removal at its settle time even when no further input arrives (a removed device
|
|
447
|
+
* gets nothing it can read): armed on load and whenever a removal is newly held. A timer that fires early (the
|
|
448
|
+
* client clock lags) re-arms. Never throws.
|
|
449
|
+
*/
|
|
450
|
+
export function scheduleRemovalSettle(rt) {
|
|
451
|
+
const due = settlesAt(rt.removal, rt.ctx.removalSettleMs);
|
|
452
|
+
if (due === undefined || rt.disposed || rt.ctx.isDisposed())
|
|
453
|
+
return;
|
|
454
|
+
const handle = rt.ctx.timers.set(() => {
|
|
455
|
+
rt.pipelineTimers.delete(handle);
|
|
456
|
+
// AC-RE-1: deferred background work, scheduled via a raw timer from a call site that can
|
|
457
|
+
// itself be nested inside a `devices.*` operation's live DEVICE_SCOPE frame (`loadRuntime` re-arms this on
|
|
458
|
+
// every load) — `guard.detached` clears that frame before the body runs, the same way `setPipelineTimer`
|
|
459
|
+
// already does for its own callbacks.
|
|
460
|
+
rt.guard.detached(() => {
|
|
461
|
+
if (rt.disposed || rt.abandoned || rt.ctx.isDisposed())
|
|
462
|
+
return;
|
|
463
|
+
void rt
|
|
464
|
+
.run('settleRemoval', async (rtc) => {
|
|
465
|
+
await settleDueRemoval(rtc);
|
|
466
|
+
if (rtc.removal !== undefined && settlesAt(rtc.removal, rtc.ctx.removalSettleMs) === due)
|
|
467
|
+
scheduleRemovalSettle(rtc);
|
|
468
|
+
})
|
|
469
|
+
.catch(() => undefined);
|
|
470
|
+
});
|
|
471
|
+
// Clamped to the 32-bit timer ceiling (a longer delay would fire at once): a far settle time re-arms on firing.
|
|
472
|
+
}, Math.min(MAX_TIMER_DELAY_MS, Math.max(0, due - rt.ctx.clock.now())));
|
|
473
|
+
rt.pipelineTimers.add(handle);
|
|
474
|
+
}
|
|
475
|
+
export async function ingestEvents(rt, events, opts) {
|
|
476
|
+
// AC-HIST-4: a fenced group processes nothing more — its state can never catch up with the group's.
|
|
477
|
+
if (rt.fence !== undefined)
|
|
478
|
+
return events.map((event) => ({ kind: 'rejected', eventId: event.id, reason: 'fenced' }));
|
|
479
|
+
// A disbanded group processes nothing more either (client/group-disband.ts).
|
|
480
|
+
if (rt.disbanded !== undefined)
|
|
481
|
+
return events.map((event) => ({ kind: 'rejected', eventId: event.id, reason: 'group-disbanded' }));
|
|
482
|
+
await settleDueRemoval(rt);
|
|
483
|
+
if (opts?.fromSync !== true && opts?.restaged !== true && opts?.treeFed !== true) {
|
|
484
|
+
// Crash-recovery resume (AC-PIPE-1): a direct `group.ingest()` call —
|
|
485
|
+
// exactly like `group.send()` — first drains any durable pending-intent
|
|
486
|
+
// record left over from a previous process. A sync-driven batch does
|
|
487
|
+
// NOT: the live subscription redelivers a member's own just-published
|
|
488
|
+
// commit back to itself (relays don't suppress self-echo) on every
|
|
489
|
+
// publish, and eagerly re-attempting a currently-blocked confirmation
|
|
490
|
+
// on every such incidental delivery would race AC-PIPE-13's own
|
|
491
|
+
// backoff-scheduled confirm retry instead of leaving it in sole control
|
|
492
|
+
// of the retry cadence. The pending-confirm gate right below still
|
|
493
|
+
// applies to sync batches unconditionally — this only skips the ACTIVE
|
|
494
|
+
// resume, not the check.
|
|
495
|
+
await resumePending(rt);
|
|
496
|
+
assertNotDisposed(rt);
|
|
497
|
+
}
|
|
498
|
+
const nostrGroupId = getNostrGroupIdHex(rt.engine.state);
|
|
499
|
+
const ours = (event) => event.kind === GROUP_EVENT_KIND && hTagOf(event) === nostrGroupId;
|
|
500
|
+
const batch = new CursorBatch(rt.joinedAtMs === undefined ? undefined : Math.floor(rt.joinedAtMs / 1000) - CURSOR_FUTURE_SKEW_S);
|
|
501
|
+
const pendingView = rt.groupStore.artifact('pendingIntents');
|
|
502
|
+
const pendingRecords = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(pendingView));
|
|
503
|
+
if (pendingRecords.some((r) => r.state === 'pending-confirm')) {
|
|
504
|
+
// Confirmation is blocked (AC-PIPE-13) — nothing can safely be applied
|
|
505
|
+
// on top of a state the group hasn't durably settled into yet. A direct
|
|
506
|
+
// caller retries itself; a sync batch holds the cursor at these events
|
|
507
|
+
// and asks for a history pass once the block clears (the confirmation
|
|
508
|
+
// that clears it calls `rt.resync()`).
|
|
509
|
+
if (opts?.fromSync === true && events.some(ours)) {
|
|
510
|
+
for (const event of events)
|
|
511
|
+
if (ours(event))
|
|
512
|
+
batch.holdBusy(event.id, event.created_at);
|
|
513
|
+
rt.resyncNeeded = true;
|
|
514
|
+
await persistCursor(rt, batch);
|
|
515
|
+
}
|
|
516
|
+
return events.map((event) => ({ kind: 'retryable', eventId: event.id, reason: 'group-busy' }));
|
|
517
|
+
}
|
|
518
|
+
if (events.length === 0 && opts?.treeFed !== true)
|
|
519
|
+
return [];
|
|
520
|
+
// Epic marmot-leaf-backup-pre-publish S2 (Design Decisions 7 and 7a), before the engine is fed with anything: a group
|
|
521
|
+
// restored from a pending-tagged snapshot checks every envelope's commit digest against the commit it waits for.
|
|
522
|
+
// - While the adoption's marker is present (C not seen yet), a match - or any envelope that opens under S_C's own epoch
|
|
523
|
+
// key, which only a holder of S_C can write - is recognised here and the marker is cleared in
|
|
524
|
+
// THIS batch's transaction, beside its cursor and ledger write: a duplicate disposition advances the cursor with no
|
|
525
|
+
// ledger entry, so a crash between this batch and the resolution step must not resume past C and rewind a public C.
|
|
526
|
+
// - While the rewound-away node is parked (C was unseen at the first sync), a match reinstates it as an ordinary fork
|
|
527
|
+
// node and rebuilds the engine; a pass over the rebuilt tree follows, and the resync re-offers whatever the lost pool held.
|
|
528
|
+
// - A parked node whose parent left the window can never be reinstated: dropped, and C is processed under today's rules.
|
|
529
|
+
let recognisedPending = false;
|
|
530
|
+
let treeFedNeeded = false;
|
|
531
|
+
if (rt.pendingLeaf !== undefined && !reinstatable(rt.window, rt.pendingLeaf))
|
|
532
|
+
await dropPendingLeaf(rt);
|
|
533
|
+
const watch = pendingWatch(rt);
|
|
534
|
+
if (watch !== undefined) {
|
|
535
|
+
const priorLedger = rt.groupStore.artifact('terminalLedger');
|
|
536
|
+
const commits = [];
|
|
537
|
+
const copies = [];
|
|
538
|
+
for (const event of events) {
|
|
539
|
+
if (!ours(event))
|
|
540
|
+
continue;
|
|
541
|
+
// Already terminal: neither evidence nor a carrier (and no trial decryption for it).
|
|
542
|
+
if ((await rt.guard.call(rt.groupId, 'store', () => readTerminalLedgerEntry(priorLedger, event.id))) !== undefined)
|
|
543
|
+
continue;
|
|
544
|
+
const sighting = await watch.sight(event);
|
|
545
|
+
if (sighting === undefined)
|
|
546
|
+
continue;
|
|
547
|
+
// The consumer's verifier decides first, exactly as for the loop below: an event it refuses is neither recognised nor ledgered.
|
|
548
|
+
if (rt.ctx.verifier !== undefined) {
|
|
549
|
+
let verified;
|
|
550
|
+
try {
|
|
551
|
+
verified = await rt.guard.call(rt.groupId, 'verifier', () => rt.ctx.verifier(event));
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
verified = false;
|
|
555
|
+
}
|
|
556
|
+
if (!verified)
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
if (sighting === 'copy')
|
|
560
|
+
copies.push(event);
|
|
561
|
+
else if (sighting === 'public')
|
|
562
|
+
recognisedPending = true;
|
|
563
|
+
else if (rt.pendingLeaf !== undefined)
|
|
564
|
+
commits.push(event);
|
|
565
|
+
else
|
|
566
|
+
recognisedPending = true;
|
|
567
|
+
}
|
|
568
|
+
if (commits.length > 0 && (await reinstatePendingLeaf(rt, commits)))
|
|
569
|
+
treeFedNeeded = true;
|
|
570
|
+
await ledgerOwnCopies(rt, copies);
|
|
571
|
+
assertNotDisposed(rt);
|
|
572
|
+
}
|
|
573
|
+
// An own envelope whose intent is still recorded (published, not yet confirmed — a probe that timed out after a
|
|
574
|
+
// crash) is not a foreign event: the engine cannot process its own commit (it would read as a fenced replay).
|
|
575
|
+
// Held for redelivery; the intent's own resume confirms it, and ledgers it `own`.
|
|
576
|
+
const ownPending = new Set(pendingRecords.flatMap((r) => (r.envelope === undefined ? [] : [r.envelope.id])));
|
|
577
|
+
const stateBefore = rt.engine.state;
|
|
578
|
+
const windowBefore = rt.window;
|
|
579
|
+
const treeSizeBefore = rt.engine.history.size;
|
|
580
|
+
/** S10: accepted application messages of this batch, by event id (a later outcome for the id replaces or drops it). */
|
|
581
|
+
const messages = new Map();
|
|
582
|
+
/** S10: earlier-delivered messages the engine retracted in this batch (a rewind abandoned their branch). */
|
|
583
|
+
const retracted = new Set();
|
|
584
|
+
/** S10: whether the engine's history tree may have changed without the state changing (a sibling commit, a proposal staged on a retained node). */
|
|
585
|
+
let handshake = false;
|
|
586
|
+
let fence;
|
|
587
|
+
/**
|
|
588
|
+
* ADR-016: equivocation evidence the engine reported in this batch, by record key — the engine re-reports the same
|
|
589
|
+
* evidence on every selection pass, so it is collected once per key (further sibling digests of the same leaf at
|
|
590
|
+
* the same node merged in). Recorded with the batch, announced only for keys the batch's transaction newly wrote.
|
|
591
|
+
*/
|
|
592
|
+
const evidence = new Map();
|
|
593
|
+
const noteEvidence = (reported) => noteEquivocations(evidence, reported);
|
|
594
|
+
/** The records this batch's transaction wrote (absent before it): the only ones announced. */
|
|
595
|
+
let recorded = [];
|
|
596
|
+
const ledgerView = rt.groupStore.artifact('terminalLedger');
|
|
597
|
+
const dispositions = new Array(events.length);
|
|
598
|
+
const ledgerWrites = new Map();
|
|
599
|
+
/** Event ids the engine authenticated and processed terminally in this batch (a later copy is a duplicate). */
|
|
600
|
+
const inBatch = new Set();
|
|
601
|
+
let stateChanged = false;
|
|
602
|
+
// Set once the engine is first touched: if anything fails before the batch
|
|
603
|
+
// is persisted, the engine is rebuilt from the durable state (which is what
|
|
604
|
+
// it held when the batch started) plus the envelopes it was pooling then
|
|
605
|
+
// (see `restorePreBatchState`).
|
|
606
|
+
let pooledBefore;
|
|
607
|
+
/** The persisted transition's plan: delivered after the try — nothing after the commit may roll the batch back. */
|
|
608
|
+
let committed;
|
|
609
|
+
let restoredReenqueues = [];
|
|
610
|
+
const record = (outcome) => {
|
|
611
|
+
// An event already terminal in this batch keeps that outcome: a later copy's (a duplicate, a pool retry)
|
|
612
|
+
// must not drop its notification or overwrite its ledger entry. A retraction is not such a copy.
|
|
613
|
+
if (outcome.retracted !== true && ledgerWrites.has(outcome.eventId) && outcome.disposition.kind === 'duplicate')
|
|
614
|
+
return;
|
|
615
|
+
if (outcome.message !== undefined)
|
|
616
|
+
messages.set(outcome.eventId, outcome.message);
|
|
617
|
+
else
|
|
618
|
+
messages.delete(outcome.eventId);
|
|
619
|
+
if (outcome.disposition.kind === 'applied' || outcome.disposition.kind === 'rejected')
|
|
620
|
+
handshake = true;
|
|
621
|
+
if (outcome.retracted === true) {
|
|
622
|
+
retracted.add(outcome.eventId);
|
|
623
|
+
// Delivered earlier in this very batch: never announced, so only its ledger entry changes.
|
|
624
|
+
const pending = ledgerWrites.get(outcome.eventId);
|
|
625
|
+
if (pending !== undefined)
|
|
626
|
+
ledgerWrites.set(outcome.eventId, { disposition: WITHDRAWN_DISPOSITION, createdAt: pending.createdAt });
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
if (outcome.terminal) {
|
|
630
|
+
ledgerWrites.set(outcome.eventId, {
|
|
631
|
+
disposition: outcome.ledgerAs ?? outcome.disposition.kind,
|
|
632
|
+
createdAt: outcome.createdAt,
|
|
633
|
+
...(outcome.message === undefined ? {} : { node: outcome.message.orderingKey.acceptedDigest }),
|
|
634
|
+
});
|
|
635
|
+
batch.released.add(outcome.eventId);
|
|
636
|
+
if (outcome.advances)
|
|
637
|
+
batch.advanced.push(outcome.createdAt);
|
|
638
|
+
}
|
|
639
|
+
else if (outcome.holds) {
|
|
640
|
+
batch.hold(outcome.eventId, outcome.createdAt, outcome.unauthenticated === true);
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
try {
|
|
644
|
+
for (let i = 0; i < events.length; i++) {
|
|
645
|
+
const event = events[i];
|
|
646
|
+
if (fence !== undefined) {
|
|
647
|
+
dispositions[i] = { kind: 'rejected', eventId: event.id, reason: 'fenced' };
|
|
648
|
+
continue;
|
|
649
|
+
}
|
|
650
|
+
if (!ours(event)) {
|
|
651
|
+
// Not ledgered: an id the relay filter should never have matched says
|
|
652
|
+
// nothing about this group, and the ledger is this group's dedup set.
|
|
653
|
+
dispositions[i] = { kind: 'rejected', eventId: event.id, reason: 'wrong-group' };
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
const existingLedgerEntry = await rt.guard.call(rt.groupId, 'store', () => readTerminalLedgerEntry(ledgerView, event.id));
|
|
657
|
+
if (existingLedgerEntry !== undefined) {
|
|
658
|
+
// Already terminal from an earlier call (AC-PIPE-5) — neither the
|
|
659
|
+
// verifier nor the engine is consulted for it again. So this copy's
|
|
660
|
+
// `created_at` is unverified (the id alone matched): only the timestamp
|
|
661
|
+
// recorded when the event was processed may advance the cursor, and an
|
|
662
|
+
// entry without one never does.
|
|
663
|
+
dispositions[i] = { kind: 'duplicate', eventId: event.id };
|
|
664
|
+
batch.released.add(event.id);
|
|
665
|
+
if (AUTHENTICATED_TERMINAL.has(existingLedgerEntry.disposition) && existingLedgerEntry.createdAt !== undefined) {
|
|
666
|
+
batch.advanced.push(existingLedgerEntry.createdAt);
|
|
667
|
+
}
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (rt.ctx.verifier !== undefined) {
|
|
671
|
+
let verified;
|
|
672
|
+
try {
|
|
673
|
+
verified = await rt.guard.call(rt.groupId, 'verifier', () => rt.ctx.verifier(event));
|
|
674
|
+
}
|
|
675
|
+
catch {
|
|
676
|
+
verified = false;
|
|
677
|
+
}
|
|
678
|
+
if (!verified) {
|
|
679
|
+
// Not ledgered: a verifier verdict may be transient, and a redelivery
|
|
680
|
+
// is simply verified again.
|
|
681
|
+
dispositions[i] = { kind: 'rejected', eventId: event.id, reason: 'verifier' };
|
|
682
|
+
continue;
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
if (inBatch.has(event.id)) {
|
|
686
|
+
// The same event twice in one batch: the first copy the engine authenticated decides (its ledger entry is
|
|
687
|
+
// written with the batch). Only such a copy reserves the id — a forged, misrouted or unreadable envelope
|
|
688
|
+
// carrying a real event's id never suppresses the real one.
|
|
689
|
+
dispositions[i] = { kind: 'duplicate', eventId: event.id };
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
if (ownPending.has(event.id)) {
|
|
693
|
+
dispositions[i] = { kind: 'retryable', eventId: event.id, reason: 'own-pending' };
|
|
694
|
+
batch.hold(event.id, event.created_at, false);
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
if (pooledBefore === undefined)
|
|
698
|
+
pooledBefore = rt.engine.pendingEnvelopes();
|
|
699
|
+
// Batch settle: one engine step per event, none of them settling — a could-win opaque candidate is held, and
|
|
700
|
+
// decided once the whole call's material has been seen (below).
|
|
701
|
+
const step = await ingestThroughGate(rt, [event], false);
|
|
702
|
+
stateChanged ||= step.stateChanged;
|
|
703
|
+
noteEvidence(step.evidence);
|
|
704
|
+
if (step.fence !== undefined)
|
|
705
|
+
fence = step.fence;
|
|
706
|
+
let own;
|
|
707
|
+
for (const outcome of step.outcomes) {
|
|
708
|
+
if (outcome.eventId === event.id)
|
|
709
|
+
own = outcome;
|
|
710
|
+
else
|
|
711
|
+
record(outcome); // pool retries this event unlocked
|
|
712
|
+
}
|
|
713
|
+
for (const bad of step.rejectedPooled) {
|
|
714
|
+
record({
|
|
715
|
+
eventId: bad.id,
|
|
716
|
+
createdAt: bad.created_at,
|
|
717
|
+
disposition: { kind: 'rejected', eventId: bad.id, reason: 'identity-proof' },
|
|
718
|
+
terminal: true,
|
|
719
|
+
holds: false,
|
|
720
|
+
advances: false,
|
|
721
|
+
});
|
|
722
|
+
}
|
|
723
|
+
if (step.rejected) {
|
|
724
|
+
dispositions[i] = { kind: 'rejected', eventId: event.id, reason: 'identity-proof' };
|
|
725
|
+
record({ eventId: event.id, createdAt: event.created_at, disposition: dispositions[i], terminal: true, holds: false, advances: false });
|
|
726
|
+
}
|
|
727
|
+
else if (own === undefined) {
|
|
728
|
+
// The engine pooled the envelope without yielding a result for it: it
|
|
729
|
+
// does not decrypt against any state held yet (a future epoch whose
|
|
730
|
+
// commit has not arrived — or a pre-join epoch, which never will).
|
|
731
|
+
const pooled = rt.engine.pendingEnvelopes().some((e) => e.id === event.id);
|
|
732
|
+
dispositions[i] = { kind: 'retryable', eventId: event.id, reason: pooled ? 'undecryptable' : 'no-disposition' };
|
|
733
|
+
// A pooled envelope, or one the engine retained for a later convergence pass: either way it is
|
|
734
|
+
// owed a disposition, so it holds the cursor until it ends terminally.
|
|
735
|
+
batch.hold(event.id, event.created_at, true);
|
|
736
|
+
}
|
|
737
|
+
else {
|
|
738
|
+
dispositions[i] = own.disposition;
|
|
739
|
+
record(own);
|
|
740
|
+
if (own.terminal && own.advances)
|
|
741
|
+
inBatch.add(event.id);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
// Batch settle (spec Design Decision 6, AC-HOLD-1): the candidates the steps held are decided against everything
|
|
745
|
+
// this call brought — a lone own-leaf commit with no sibling fences, one the batch outdid stays held. Whenever a
|
|
746
|
+
// candidate is held or the engine reports its batch open (a candidate the steps held may have left the held set,
|
|
747
|
+
// e.g. past the horizon, and its bookkeeping must not outlive the call); a seam-off batch never opens one.
|
|
748
|
+
// The tree-fed pass of a pending-restore resolution or reinstatement runs here too (client/pending-leaf.ts): a pass over
|
|
749
|
+
// a tree rebuilt without any envelope to trigger one.
|
|
750
|
+
if (fence === undefined && ((rt.engine.opaque?.candidates.length ?? 0) > 0 || rt.engine.batchOpen || opts?.treeFed === true || treeFedNeeded)) {
|
|
751
|
+
const settled = await ingestThroughGate(rt, []);
|
|
752
|
+
stateChanged ||= settled.stateChanged;
|
|
753
|
+
noteEvidence(settled.evidence);
|
|
754
|
+
if (settled.fence !== undefined)
|
|
755
|
+
fence = settled.fence;
|
|
756
|
+
// Every position of an event id (the same event can appear twice in one call): each is overwritten.
|
|
757
|
+
const indexOf = new Map();
|
|
758
|
+
for (const [at, event] of events.entries())
|
|
759
|
+
indexOf.set(event.id, [...(indexOf.get(event.id) ?? []), at]);
|
|
760
|
+
for (const outcome of settled.outcomes) {
|
|
761
|
+
// The envelope the settle fences was provisionally ledgered `past-epoch` by its deferred step: the fence ends
|
|
762
|
+
// all processing and nothing about the fencing envelope is terminal, so that entry goes.
|
|
763
|
+
if (outcome.fence !== undefined)
|
|
764
|
+
ledgerWrites.delete(outcome.eventId);
|
|
765
|
+
record(outcome);
|
|
766
|
+
for (const at of indexOf.get(outcome.eventId) ?? [])
|
|
767
|
+
dispositions[at] = outcome.disposition;
|
|
768
|
+
}
|
|
769
|
+
for (const bad of settled.rejectedPooled) {
|
|
770
|
+
record({
|
|
771
|
+
eventId: bad.id,
|
|
772
|
+
createdAt: bad.created_at,
|
|
773
|
+
disposition: { kind: 'rejected', eventId: bad.id, reason: 'identity-proof' },
|
|
774
|
+
terminal: true,
|
|
775
|
+
holds: false,
|
|
776
|
+
advances: false,
|
|
777
|
+
});
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
assertNotDisposed(rt);
|
|
781
|
+
// S20: the removal the engine now holds (a new one, a competing one joining it, or none after a rewind out of it),
|
|
782
|
+
// persisted with the state it belongs to.
|
|
783
|
+
const removalNext = fence === undefined ? nextHeldRemoval(rt.removal, rt.engine.removal, rt.ctx.clock.now()) : undefined;
|
|
784
|
+
// The held opaque candidates and `U` (client/opaque-candidates.ts), read only now that every drain ran to its end (a
|
|
785
|
+
// fenced candidate's record is released only after its `unreadable` was yielded). A change of the held set OR of
|
|
786
|
+
// `U` alone is written, in the transaction that ledgers the provisional `skipped` it belongs to. A batch that fences
|
|
787
|
+
// keeps none of them: the group is Unrecoverable and a reload must not resurrect them (`null` = delete).
|
|
788
|
+
const opaqueNext = fence !== undefined ? (isEmptyOpaque(rt.opaque) ? undefined : null) : nextOpaque(rt.opaque, rt.engine.opaque);
|
|
789
|
+
if (stateChanged || ledgerWrites.size > 0 || retracted.size > 0 || handshake || opts?.fromSync === true || fence !== undefined || removalNext !== undefined || opaqueNext !== undefined || evidence.size > 0 || recognisedPending) {
|
|
790
|
+
const finalState = rt.engine.state;
|
|
791
|
+
// S10: the rewind window moves with the engine's history tree — the tip, and fork material the engine retained
|
|
792
|
+
// without moving it (a losing sibling commit, a proposal staged on a retained node): written in the same
|
|
793
|
+
// transaction as the state it describes, so every engine rebuilt over that state can still converge.
|
|
794
|
+
const captured = handshake || tagOf(finalState) !== windowBefore.tipTag || rt.engine.history.size !== treeSizeBefore ? await captureWindow(rt.engine.history, finalState, undefined, windowBefore) : undefined;
|
|
795
|
+
const window = captured !== undefined && !sameWindow(captured, windowBefore) ? captured : undefined;
|
|
796
|
+
// A batch that moved further than the rewind horizon: its earliest commits (and an abandoned commit below the
|
|
797
|
+
// new floor) are already pruned from `window` — the notifications are diffed against one reaching down to the
|
|
798
|
+
// previous window's floor.
|
|
799
|
+
const span = Number(finalState.groupContext.epoch) - (canonicalPath(windowBefore)[0]?.epoch ?? 0);
|
|
800
|
+
const diffWindow = window !== undefined && span > REWIND_HORIZON ? await captureWindow(rt.engine.history, finalState, span, windowBefore) : undefined;
|
|
801
|
+
const fenceRecord = fence === undefined ? undefined : { at: rt.ctx.clock.now(), epoch: String(finalState.groupContext.epoch), message: fence.message };
|
|
802
|
+
// AC-HIST-2/3: what the batch applied, withdrew and delivered — planned now, announced once it is durable.
|
|
803
|
+
const plan = await planTransition(rt, windowBefore, stateBefore, window ?? windowBefore, finalState, [...messages.values()], retracted, diffWindow ?? window ?? windowBefore);
|
|
804
|
+
// AC-RATCH-1 (BACKLOG ratchet-skip-undelivered-marker-horizon-rearm, step 1): during the first post-load sync of
|
|
805
|
+
// a load that did not re-arm the skip (its marker undelivered), an own application message a relay returned at
|
|
806
|
+
// or after the skipped generation on the marker's epoch state proves the skip was delivered after all — a
|
|
807
|
+
// backup taken before that. The marker is marked delivered with the batch and the skip re-armed, so the first
|
|
808
|
+
// post-gate message also clears own messages this sync could not see (held only by silence-excluded relays).
|
|
809
|
+
// Own messages the sync DID see need no skip: ingesting them already moved the own sender ratchet past them.
|
|
810
|
+
const ownLeaf = Number(finalState.privatePath.leafIndex);
|
|
811
|
+
const ownSkipEvidence = opts?.fromSync === true && !rt.sendGateOpen && !rt.ratchetSkipPending
|
|
812
|
+
? [...messages.values()].flatMap((m) => (m.orderingKey.leafIndex === ownLeaf && !retracted.has(m.eventId) ? [m.orderingKey] : []))
|
|
813
|
+
: [];
|
|
814
|
+
let skipDelivered = false;
|
|
815
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
816
|
+
if (stateChanged)
|
|
817
|
+
await writeGroupState(tx.artifact('clientState'), finalState);
|
|
818
|
+
if (window !== undefined)
|
|
819
|
+
await writeWindow(tx.artifact('rewindTree'), window);
|
|
820
|
+
if (fenceRecord !== undefined)
|
|
821
|
+
await writeFence(tx.extension(FENCE_EXTENSION), fenceRecord);
|
|
822
|
+
// ADR-016 (SC-6): each piece of evidence is recorded once — only when its key is absent — atomically with
|
|
823
|
+
// the batch; reset per run of this callback, so only what the committed run wrote is announced.
|
|
824
|
+
const wrote = [];
|
|
825
|
+
for (const record of evidence.values())
|
|
826
|
+
if (await writeEquivocation(tx.extension(EQUIVOCATION_EXTENSION), record))
|
|
827
|
+
wrote.push(record);
|
|
828
|
+
recorded = wrote;
|
|
829
|
+
if (removalNext === null)
|
|
830
|
+
await clearRemoval(tx.extension(REMOVAL_EXTENSION));
|
|
831
|
+
else if (removalNext !== undefined)
|
|
832
|
+
await writeRemoval(tx.extension(REMOVAL_EXTENSION), removalNext);
|
|
833
|
+
if (opaqueNext === null)
|
|
834
|
+
await clearOpaque(tx.extension(OPAQUE_EXTENSION));
|
|
835
|
+
else if (opaqueNext !== undefined)
|
|
836
|
+
await writeOpaque(tx.extension(OPAQUE_EXTENSION), opaqueNext);
|
|
837
|
+
// Recognising C is durable: the marker goes with the cursor and ledger write that moves past C (Design Decision 7).
|
|
838
|
+
if (recognisedPending)
|
|
839
|
+
await clearPendingCommit(tx.extension(PENDING_COMMIT_EXTENSION));
|
|
840
|
+
for (const [eventId, { disposition, createdAt, node }] of ledgerWrites) {
|
|
841
|
+
await writeTerminalLedgerEntry(tx.artifact('terminalLedger'), eventId, { disposition, at: rt.ctx.clock.now(), createdAt, ...(node === undefined ? {} : { node }) });
|
|
842
|
+
}
|
|
843
|
+
restoredReenqueues = await recordTransition(rt, tx, plan, finalState.groupContext.epoch);
|
|
844
|
+
if (opts?.fromSync === true)
|
|
845
|
+
await writeNextCursor(rt, tx.artifact('ingestCursor'), batch);
|
|
846
|
+
skipDelivered = false;
|
|
847
|
+
for (const ordering of ownSkipEvidence)
|
|
848
|
+
if (await markRatchetSkipDelivered(tx.extension(RATCHET_SKIP_EXTENSION), ordering))
|
|
849
|
+
skipDelivered = true;
|
|
850
|
+
}));
|
|
851
|
+
// `leaf-equivocation` rides the transition's delivery: dispatched after the commit, after the batch's own notifications.
|
|
852
|
+
committed = recorded.length === 0 ? plan : { ...plan, notifications: [...plan.notifications, ...recorded.map(equivocationNotification)] };
|
|
853
|
+
if (removalNext !== undefined)
|
|
854
|
+
rt.removal = removalNext ?? undefined;
|
|
855
|
+
if (removalNext !== undefined && removalNext !== null)
|
|
856
|
+
scheduleRemovalSettle(rt);
|
|
857
|
+
// The mirror follows the commit (never before it: a failed transaction leaves the pre-batch record for the rollback).
|
|
858
|
+
if (opaqueNext !== undefined)
|
|
859
|
+
rt.opaque = opaqueNext ?? undefined;
|
|
860
|
+
if (recognisedPending)
|
|
861
|
+
rt.pendingCommit = undefined;
|
|
862
|
+
// Exposed only now: the state passed the proof gate and is durable (AC-PROOF-2).
|
|
863
|
+
if (stateChanged)
|
|
864
|
+
rt.exposedState = finalState;
|
|
865
|
+
if (skipDelivered)
|
|
866
|
+
rt.ratchetSkipPending = true;
|
|
867
|
+
if (fence !== undefined) {
|
|
868
|
+
rt.fence = fence;
|
|
869
|
+
// Nothing it fetches could ever be processed again.
|
|
870
|
+
stopGroupSync(rt);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
// Unchanged by content: the mirror re-points at the engine's (equal) record, so the next batch's check is an identity test again.
|
|
874
|
+
if (opaqueNext === undefined && fence === undefined)
|
|
875
|
+
rt.opaque = rt.engine.opaque;
|
|
876
|
+
}
|
|
877
|
+
catch (error) {
|
|
878
|
+
if (pooledBefore !== undefined) {
|
|
879
|
+
try {
|
|
880
|
+
await restorePreBatchState(rt, pooledBefore, opts?.fromSync === true);
|
|
881
|
+
}
|
|
882
|
+
catch (restoreError) {
|
|
883
|
+
// The engine may be left anywhere between the batch's state and the
|
|
884
|
+
// durable one: never let the next op run on it.
|
|
885
|
+
rt.needsReload = true;
|
|
886
|
+
rt.ctx.emit('groupStateFailed', {
|
|
887
|
+
groupId: rt.groupId,
|
|
888
|
+
error: new MarmotError('store-failed', `ingest: group ${rt.groupId} could not be rolled back to its durable state; it reloads before the next operation`, {
|
|
889
|
+
cause: restoreError,
|
|
890
|
+
details: { groupId: rt.groupId },
|
|
891
|
+
}),
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
// Always the batch's own failure, never the rollback's.
|
|
896
|
+
throw error;
|
|
897
|
+
}
|
|
898
|
+
if (committed !== undefined)
|
|
899
|
+
await deliverTransition(rt, committed);
|
|
900
|
+
// AC-RACE-1 / epic marmot-rewound-own-commit-no-signal: this batch's rewind withdrew an own self-update or own change commit — re-drive it at the new tip.
|
|
901
|
+
scheduleWithdrawnRedrives(rt, committed);
|
|
902
|
+
// ...and cancelled a re-enqueue whose original rotation it restored: a sibling that waited behind it goes next.
|
|
903
|
+
for (const intentId of restoredReenqueues)
|
|
904
|
+
scheduleDeferredReenqueueRetry(rt, intentId);
|
|
905
|
+
// AC-PIPE-15: this batch fenced the group (the marker is durable): every awaiting-sync intent fails `fenced`,
|
|
906
|
+
// never published. Its own failure never fails the batch — a load of the fenced group repeats the cleanup.
|
|
907
|
+
if (fence !== undefined && rt.fence === fence)
|
|
908
|
+
await failAwaitingIntents(rt).catch(() => undefined);
|
|
909
|
+
// An accepted commit may have moved the group's relays or Nostr group id.
|
|
910
|
+
if (stateChanged)
|
|
911
|
+
rt.checkSyncRouting();
|
|
912
|
+
// AC-ADMIN-9: the engine's SelfRemove election, now that the staged set it decided on is durable.
|
|
913
|
+
const election = rt.selfRemoveElection;
|
|
914
|
+
rt.selfRemoveElection = undefined;
|
|
915
|
+
if (election !== undefined)
|
|
916
|
+
scheduleSelfRemoveCommit(rt, election);
|
|
917
|
+
// ...and may be the commit queued rumors wait for (AC-INVITE-3), or a proposal that changes
|
|
918
|
+
// what the next commit carries. An application message (secret tree only) changes neither:
|
|
919
|
+
// no drain, so a deferred proposal is not asked about again on every message.
|
|
920
|
+
const after = rt.exposedState;
|
|
921
|
+
// A changed staged set is a genuine reason to try a held queue commit again.
|
|
922
|
+
if (stateChanged && after.unappliedProposals !== stateBefore.unappliedProposals) {
|
|
923
|
+
releaseQueueCommitHold(rt);
|
|
924
|
+
// AC-RACE-2: the election this client was waiting on (or watching for) may no longer apply.
|
|
925
|
+
releaseSelfRemoveElectionWait(rt);
|
|
926
|
+
}
|
|
927
|
+
if (stateChanged &&
|
|
928
|
+
(after.groupContext.epoch !== stateBefore.groupContext.epoch ||
|
|
929
|
+
after.unappliedProposals !== stateBefore.unappliedProposals ||
|
|
930
|
+
after.groupActiveState !== stateBefore.groupActiveState)) {
|
|
931
|
+
rt.scheduleQueueDrain();
|
|
932
|
+
}
|
|
933
|
+
// S15 round 2: a commit waiting for a proposal this client dropped by application policy (rejected or expired)
|
|
934
|
+
// at this epoch — a peer admin committed it by reference — gets that proposal re-staged and is retried.
|
|
935
|
+
// S20: not for a removed leaf — its `missing_parent` commits are siblings of its removal, read at the removal's parent.
|
|
936
|
+
if (opts?.restaged !== true && rt.engine.state.groupActiveState.kind !== 'removedFromGroup' && dispositions.some((d) => d?.kind === 'retryable' && d.reason === MISSING_PARENT)) {
|
|
937
|
+
const waiting = events.filter((_, i) => dispositions[i]?.kind === 'retryable' && dispositions[i].reason === MISSING_PARENT);
|
|
938
|
+
const retried = await restageExcludedProposals(rt, waiting);
|
|
939
|
+
// Round 3: the caller sees what the retry made of this batch's events (a `missing_parent` commit now
|
|
940
|
+
// `applied`, an undecryptable message it unlocked …). Only still-retryable dispositions are replaced:
|
|
941
|
+
// nothing terminal here was retried. The retry persisted and delivered its own outcomes (ledger,
|
|
942
|
+
// notifications, history outbox) — this batch recorded none for those events, so none is doubled.
|
|
943
|
+
for (let i = 0; i < events.length; i++) {
|
|
944
|
+
const outcome = retried.get(events[i].id);
|
|
945
|
+
if (outcome !== undefined && dispositions[i]?.kind === 'retryable')
|
|
946
|
+
dispositions[i] = outcome;
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
return dispositions;
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Re-stages the proposals this client dropped by application policy
|
|
953
|
+
* (invitations/proposal-deadline.ts, Exclusions) wherever a waiting commit
|
|
954
|
+
* may need them, and retries `waiting` — the commits that named a proposal
|
|
955
|
+
* not staged here — together with whatever the engine still pools:
|
|
956
|
+
* - those dropped from the CURRENT state (same tag) into it;
|
|
957
|
+
* - those dropped from an earlier state within the rollback horizon into
|
|
958
|
+
* the retained window node with THAT tag (round 4; never a same-epoch
|
|
959
|
+
* sibling, whose members never validated them): a peer commit at that epoch
|
|
960
|
+
* is replayed by fork recovery against the retained state, which must
|
|
961
|
+
* stage it too. Same tag, re-snapshotted — as the engine itself does when
|
|
962
|
+
* a proposal is staged onto a retained node.
|
|
963
|
+
* Persisted like the drop itself (state and window written, engine rebuilt
|
|
964
|
+
* over fresh copies). A commit that then applies moves the epoch; one that
|
|
965
|
+
* still does not leaves the proposals staged, and the policy in front of the
|
|
966
|
+
* next own commit drops the current epoch's again (an exclusion is
|
|
967
|
+
* authoritative at its epoch). Runs inside a group op.
|
|
968
|
+
*/
|
|
969
|
+
async function restageExcludedProposals(rt, waiting) {
|
|
970
|
+
const state = rt.engine.state;
|
|
971
|
+
const all = await rt.guard.call(rt.groupId, 'store', () => readExclusions(rt.groupStore.extension(EXCLUDED_PROPOSALS_EXTENSION)));
|
|
972
|
+
const { byTag } = exclusionsAt(all, state.groupContext.epoch, tagOf(state), REWIND_HORIZON);
|
|
973
|
+
/** The proposals excluded at the branch node `tag` (only ever that node — never a same-epoch sibling) not staged in `staged`. */
|
|
974
|
+
const missingAt = (tag, staged) => {
|
|
975
|
+
const snapshot = all.snapshots.get(tag);
|
|
976
|
+
const records = byTag.get(tag) ?? [];
|
|
977
|
+
if (snapshot === undefined || records.length === 0)
|
|
978
|
+
return {};
|
|
979
|
+
const kept = deserializeClientState(snapshot).unappliedProposals;
|
|
980
|
+
const out = {};
|
|
981
|
+
for (const { proposalRef } of records)
|
|
982
|
+
if (!(proposalRef in staged) && kept[proposalRef] !== undefined)
|
|
983
|
+
out[proposalRef] = kept[proposalRef];
|
|
984
|
+
return out;
|
|
985
|
+
};
|
|
986
|
+
const restored = missingAt(tagOf(state), state.unappliedProposals);
|
|
987
|
+
const next = { ...state, unappliedProposals: { ...state.unappliedProposals, ...restored } };
|
|
988
|
+
// Retained nodes (never the tip: the stored clientState is it) a proposal was excluded at — matched by tag.
|
|
989
|
+
const nodes = new Map(rt.window.nodes);
|
|
990
|
+
let windowChanged = false;
|
|
991
|
+
for (const node of rt.window.nodes.values()) {
|
|
992
|
+
if (node.tag === rt.window.tipTag || node.snapshot === undefined || !byTag.has(node.tag))
|
|
993
|
+
continue;
|
|
994
|
+
const retained = deserializeClientState(node.snapshot);
|
|
995
|
+
const add = missingAt(node.tag, retained.unappliedProposals);
|
|
996
|
+
if (Object.keys(add).length === 0)
|
|
997
|
+
continue;
|
|
998
|
+
nodes.set(node.tag, { ...node, snapshot: serializeClientState({ ...retained, unappliedProposals: { ...retained.unappliedProposals, ...add } }) });
|
|
999
|
+
windowChanged = true;
|
|
1000
|
+
}
|
|
1001
|
+
if (Object.keys(restored).length === 0 && !windowChanged)
|
|
1002
|
+
return new Map();
|
|
1003
|
+
const window = { tipTag: rt.window.tipTag, nodes };
|
|
1004
|
+
const pooled = rt.engine.pendingEnvelopes();
|
|
1005
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1006
|
+
if (Object.keys(restored).length > 0)
|
|
1007
|
+
await writeGroupState(tx.artifact('clientState'), next);
|
|
1008
|
+
if (windowChanged)
|
|
1009
|
+
await writeWindow(tx.artifact('rewindTree'), window);
|
|
1010
|
+
}));
|
|
1011
|
+
assertNotDisposed(rt);
|
|
1012
|
+
try {
|
|
1013
|
+
// A fresh copy: never hand one ClientState object to two engines (ts-mls mutates it in place).
|
|
1014
|
+
await rt.rebuildEngine(deserializeClientState(serializeClientState(next)), window);
|
|
1015
|
+
}
|
|
1016
|
+
catch (error) {
|
|
1017
|
+
rt.needsReload = true;
|
|
1018
|
+
throw error;
|
|
1019
|
+
}
|
|
1020
|
+
rt.exposedState = rt.engine.state;
|
|
1021
|
+
const again = [...waiting, ...pooled.filter((e) => !waiting.some((w) => w.id === e.id))];
|
|
1022
|
+
const outcomes = await ingestEvents(rt, again, { restaged: true });
|
|
1023
|
+
return new Map(again.map((event, i) => [event.id, outcomes[i]]).filter(([, d]) => d !== undefined));
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Rolls the engine back to the state a failed batch started from. Without
|
|
1027
|
+
* this, a batch whose persistence transaction (or any step before it) failed
|
|
1028
|
+
* would leave the engine advanced — ratchets, in-memory dedup — over a
|
|
1029
|
+
* durable state that never saw the batch: a retry would dedup the event as
|
|
1030
|
+
* `duplicate` and ledger it without its state, and after a restart it would
|
|
1031
|
+
* be skipped forever. Every group op persists before returning, so the
|
|
1032
|
+
* durable `clientState` IS the state the batch started from — no per-batch
|
|
1033
|
+
* copy is taken. The rebuilt engine starts from those fresh bytes; envelopes
|
|
1034
|
+
* the old one was pooling from before the batch are re-queued (they do not
|
|
1035
|
+
* decrypt against this state, so they only pool again). A sync batch's
|
|
1036
|
+
* events are refetched by a history pass from the (unwritten) cursor.
|
|
1037
|
+
*/
|
|
1038
|
+
async function restorePreBatchState(rt, pooledBefore, fromSync) {
|
|
1039
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
1040
|
+
return;
|
|
1041
|
+
await rt.reloadFromDurable();
|
|
1042
|
+
if (pooledBefore.length > 0) {
|
|
1043
|
+
const evidence = [];
|
|
1044
|
+
let fence;
|
|
1045
|
+
for await (const result of rt.engine.ingest([...pooledBefore])) {
|
|
1046
|
+
if (result.kind === 'autoCommit')
|
|
1047
|
+
rt.engine.publishFailed(result.pending);
|
|
1048
|
+
else if (result.kind === 'leafEquivocation')
|
|
1049
|
+
evidence.push(result.evidence);
|
|
1050
|
+
// A held opaque candidate the rebuilt engine decides at its first pass (one it could not verify against the
|
|
1051
|
+
// history, say) is fenced HERE: the engine releases its record once this verdict is yielded, so a verdict
|
|
1052
|
+
// dropped on the floor is a candidate forgotten. Persisted like any fence; best effort for the rest.
|
|
1053
|
+
else if (result.kind === 'unreadable')
|
|
1054
|
+
fence ??= fencingErrorIn(result.errors);
|
|
1055
|
+
}
|
|
1056
|
+
// ADR-016: what this drain's selection pass reported is recorded — best effort: the batch's own failure is what the
|
|
1057
|
+
// caller reports, and a store that refuses this write leaves the evidence to the next pass over its fork.
|
|
1058
|
+
await recordEquivocations(rt, evidence).catch(() => undefined);
|
|
1059
|
+
if (fence !== undefined)
|
|
1060
|
+
await fenceFromDrive(rt, fence, []);
|
|
1061
|
+
}
|
|
1062
|
+
if (fromSync) {
|
|
1063
|
+
rt.resyncNeeded = true;
|
|
1064
|
+
rt.resync();
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
async function writeNextCursor(rt, view, batch) {
|
|
1068
|
+
const current = await readIngestCursor(view);
|
|
1069
|
+
const next = nextCursor(current, batch, Math.floor(rt.ctx.clock.now() / 1000));
|
|
1070
|
+
if (next !== undefined && next !== current)
|
|
1071
|
+
await writeIngestCursor(view, next);
|
|
1072
|
+
}
|
|
1073
|
+
async function persistCursor(rt, batch) {
|
|
1074
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writeNextCursor(rt, tx.artifact('ingestCursor'), batch)));
|
|
1075
|
+
}
|
|
1076
|
+
/** Retention for the terminal ledger (see `pruneTerminalLedger`): entries far below the cursor's `since` are dropped, except those naming a node of the stored rewind window. */
|
|
1077
|
+
async function pruneLedger(rt) {
|
|
1078
|
+
const cursor = await rt.guard.call(rt.groupId, 'store', () => readIngestCursor(rt.groupStore.artifact('ingestCursor')));
|
|
1079
|
+
if (cursor === undefined)
|
|
1080
|
+
return;
|
|
1081
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1082
|
+
const retained = await retainedLedgerNodes(tx);
|
|
1083
|
+
await pruneTerminalLedger(tx.artifact('terminalLedger'), cursor.since - LEDGER_RETENTION_S, retained);
|
|
1084
|
+
// S9 (AC-MARK-4, DD6): a confirm-time marker lives exactly as long as its node is in the stored window.
|
|
1085
|
+
await pruneOwnCommitMarkers(tx.extension(OWN_COMMIT_EXTENSION), retained);
|
|
1086
|
+
}));
|
|
1087
|
+
}
|
|
1088
|
+
function syncRoutingOf(rt) {
|
|
1089
|
+
const view = getMarmotGroupView(rt.exposedState);
|
|
1090
|
+
const plane = groupRelayPlane(view?.relays ?? [], rt.ctx.relays.defaults);
|
|
1091
|
+
const nostrGroupId = getNostrGroupIdHex(rt.exposedState);
|
|
1092
|
+
return { plane, nostrGroupId, key: `${nostrGroupId}\u241f${[...plane].sort().join(' ')}` };
|
|
1093
|
+
}
|
|
1094
|
+
/**
|
|
1095
|
+
* The live subscription listens on the routing it was opened with; a commit
|
|
1096
|
+
* (inbound or own) changing the group's relays or Nostr group id would leave
|
|
1097
|
+
* it listening where the group no longer publishes. Restarts the sync on the
|
|
1098
|
+
* new routing: the old subscription is dropped, and a history pass from the
|
|
1099
|
+
* cursor over the new plane runs now (not on the backoff — nothing failed, and
|
|
1100
|
+
* not after a pass already in flight: it supersedes it), ending in a new
|
|
1101
|
+
* subscription. A pass already in flight on the old routing notices the change
|
|
1102
|
+
* and asks for a rerun (see `runHistoryPass`). Never throws:
|
|
1103
|
+
* it runs right after a batch or confirmation was persisted.
|
|
1104
|
+
*/
|
|
1105
|
+
export function restartSyncOnRoutingChange(rt) {
|
|
1106
|
+
if (!stillSyncing(rt) || rt.syncSubscription === undefined)
|
|
1107
|
+
return;
|
|
1108
|
+
if (rt.syncRouting === syncRoutingOf(rt).key)
|
|
1109
|
+
return;
|
|
1110
|
+
try {
|
|
1111
|
+
dropLiveSubscription(rt);
|
|
1112
|
+
}
|
|
1113
|
+
catch {
|
|
1114
|
+
// A transport that fails to unsubscribe: the subscription is forgotten either way.
|
|
1115
|
+
}
|
|
1116
|
+
if (rt.syncHistoryRetryTimer !== undefined) {
|
|
1117
|
+
rt.ctx.timers.clear(rt.syncHistoryRetryTimer);
|
|
1118
|
+
rt.syncHistoryRetryTimer = undefined;
|
|
1119
|
+
}
|
|
1120
|
+
void runHistoryPass(rt, 1, true);
|
|
1121
|
+
}
|
|
1122
|
+
/** Idempotent: a no-op once already started (a cached, still-live `groups.get` never restarts it). */
|
|
1123
|
+
export function startGroupSync(rt) {
|
|
1124
|
+
if (rt.syncStarted)
|
|
1125
|
+
return;
|
|
1126
|
+
rt.syncStarted = true;
|
|
1127
|
+
void runHistoryPass(rt, 1);
|
|
1128
|
+
}
|
|
1129
|
+
/**
|
|
1130
|
+
* Unsubscribes the live subscription and clears any pending history-retry
|
|
1131
|
+
* timer. Idempotent. The transport unsubscribe runs last: if it throws, the
|
|
1132
|
+
* sync is already stopped and the timer already cleared.
|
|
1133
|
+
*/
|
|
1134
|
+
export function stopGroupSync(rt) {
|
|
1135
|
+
rt.syncStarted = false;
|
|
1136
|
+
if (rt.syncHistoryRetryTimer !== undefined) {
|
|
1137
|
+
rt.ctx.timers.clear(rt.syncHistoryRetryTimer);
|
|
1138
|
+
rt.syncHistoryRetryTimer = undefined;
|
|
1139
|
+
}
|
|
1140
|
+
dropLiveSubscription(rt);
|
|
1141
|
+
}
|
|
1142
|
+
/** A connectivity gap: not caught up, and every history pass already under way is too old to say otherwise (GroupRuntime.caughtUp). */
|
|
1143
|
+
function markConnectivityGap(rt) {
|
|
1144
|
+
rt.caughtUp = false;
|
|
1145
|
+
rt.connectivityGeneration += 1;
|
|
1146
|
+
}
|
|
1147
|
+
/** Forgets the live subscription (and its routing) before unsubscribing it, so a throwing unsubscribe never leaves it installed. */
|
|
1148
|
+
function dropLiveSubscription(rt) {
|
|
1149
|
+
// No live delivery any more: a connectivity gap until the next complete history pass (GroupRuntime.caughtUp).
|
|
1150
|
+
markConnectivityGap(rt);
|
|
1151
|
+
const subscription = rt.syncSubscription;
|
|
1152
|
+
rt.syncSubscription = undefined;
|
|
1153
|
+
rt.syncRouting = undefined;
|
|
1154
|
+
subscription?.unsubscribe();
|
|
1155
|
+
}
|
|
1156
|
+
/**
|
|
1157
|
+
* A history pass from the persisted cursor, now — used once a blocked
|
|
1158
|
+
* confirmation clears so that events dropped as `group-busy` meanwhile (the
|
|
1159
|
+
* cursor holds them) are refetched. A no-op when not syncing, or when a
|
|
1160
|
+
* retry pass is already scheduled (it reads the same cursor).
|
|
1161
|
+
*/
|
|
1162
|
+
export function requestHistoryResync(rt) {
|
|
1163
|
+
if (!stillSyncing(rt) || rt.syncHistoryRetryTimer !== undefined)
|
|
1164
|
+
return;
|
|
1165
|
+
void runHistoryPass(rt, 1);
|
|
1166
|
+
}
|
|
1167
|
+
function stillSyncing(rt) {
|
|
1168
|
+
return rt.syncStarted && !rt.disposed && !rt.ctx.isDisposed();
|
|
1169
|
+
}
|
|
1170
|
+
/**
|
|
1171
|
+
* `startGroupSync`, `scheduleHistoryRetry`'s timer callback,
|
|
1172
|
+
* `requestHistoryResync` and `startLiveSubscription`'s `error` observer all
|
|
1173
|
+
* invoke this fire-and-forget (`void runHistoryPass(...)`, never awaited by
|
|
1174
|
+
* their caller) — so EVERY error path here must be handled internally; an
|
|
1175
|
+
* uncaught rejection would otherwise surface as an unhandled-rejection crash
|
|
1176
|
+
* instead of a retry. In particular, a store or transport call losing its
|
|
1177
|
+
* lease/connection between `stillSyncing(rt)` checks (e.g. the consumer
|
|
1178
|
+
* disposes the client, or closes the stores out from under an in-flight
|
|
1179
|
+
* background pass) must be swallowed exactly like a transport timeout — the
|
|
1180
|
+
* next scheduled retry (or, once actually disposed, `stillSyncing`'s own
|
|
1181
|
+
* guard) resolves it.
|
|
1182
|
+
*/
|
|
1183
|
+
async function runHistoryPass(rt, attempt, supersede = false) {
|
|
1184
|
+
// Single-flight per group: a pass requested while one runs (the catch-up resync of a subscription opened
|
|
1185
|
+
// mid-walk, a retry timer firing) would otherwise start a second, concurrent chain of passes over the
|
|
1186
|
+
// same windows. It only marks a rerun instead, kept across the running chain's resumed walk passes and
|
|
1187
|
+
// consumed by the next pass that reads the cursor and `catchUpSince` afresh (see `historyPass`). Should
|
|
1188
|
+
// the chain end with it still pending, exactly one pass follows — the rerun, or the retry already pending
|
|
1189
|
+
// (it reads the same cursor and floor, so one lowered meanwhile is honoured either way).
|
|
1190
|
+
// A routing restart (`supersede`) does not wait: the pass in flight fetched from the old routing and will
|
|
1191
|
+
// discard its batch; the restart's chain owns the sync from now on, and the stale pass's own rerun folds
|
|
1192
|
+
// into it like any other request.
|
|
1193
|
+
const flight = historyFlightOf(rt);
|
|
1194
|
+
if (flight.owner !== undefined && !supersede) {
|
|
1195
|
+
flight.rerun = Math.max(flight.rerun ?? 0, attempt);
|
|
1196
|
+
return;
|
|
1197
|
+
}
|
|
1198
|
+
const token = {};
|
|
1199
|
+
flight.owner = token;
|
|
1200
|
+
let next;
|
|
1201
|
+
let owned = false;
|
|
1202
|
+
try {
|
|
1203
|
+
next = await historyPass(rt, attempt);
|
|
1204
|
+
}
|
|
1205
|
+
finally {
|
|
1206
|
+
// Released even should the pass ever reject: a stuck owner would fold every later request away.
|
|
1207
|
+
owned = flight.owner === token;
|
|
1208
|
+
if (owned)
|
|
1209
|
+
flight.owner = undefined;
|
|
1210
|
+
}
|
|
1211
|
+
let then = next;
|
|
1212
|
+
if (owned && then === undefined && flight.rerun !== undefined) {
|
|
1213
|
+
const rerun = flight.rerun;
|
|
1214
|
+
flight.rerun = undefined;
|
|
1215
|
+
if (rt.syncHistoryRetryTimer === undefined)
|
|
1216
|
+
then = rerun;
|
|
1217
|
+
}
|
|
1218
|
+
if (then !== undefined)
|
|
1219
|
+
void runHistoryPass(rt, then);
|
|
1220
|
+
}
|
|
1221
|
+
// Kept beside the runtime rather than on it: in-memory sync bookkeeping private to this module.
|
|
1222
|
+
const historyFlights = new WeakMap();
|
|
1223
|
+
function historyFlightOf(rt) {
|
|
1224
|
+
let flight = historyFlights.get(rt);
|
|
1225
|
+
if (flight === undefined) {
|
|
1226
|
+
flight = { owner: undefined, rerun: undefined };
|
|
1227
|
+
historyFlights.set(rt, flight);
|
|
1228
|
+
}
|
|
1229
|
+
return flight;
|
|
1230
|
+
}
|
|
1231
|
+
/** One history pass; resolves to the attempt the pass continues with (the walk goes on, or restarts on a new routing), if any. */
|
|
1232
|
+
async function historyPass(rt, attempt) {
|
|
1233
|
+
try {
|
|
1234
|
+
if (!stillSyncing(rt))
|
|
1235
|
+
return undefined;
|
|
1236
|
+
rt.resyncNeeded = false;
|
|
1237
|
+
// GroupRuntime.connectivityGeneration at this pass's start: it may mark the group caught up only if unchanged.
|
|
1238
|
+
const generation = rt.connectivityGeneration;
|
|
1239
|
+
const routing = syncRoutingOf(rt);
|
|
1240
|
+
const { plane, nostrGroupId } = routing;
|
|
1241
|
+
// Checked inside each queued batch: an ingest ahead of it may have moved
|
|
1242
|
+
// the routing, and then this pass fetched from the wrong relays or `h`.
|
|
1243
|
+
const stale = (rtc) => syncRoutingOf(rtc).key !== routing.key;
|
|
1244
|
+
const requestRelays = rt.historyTracker.historyRelays(plane);
|
|
1245
|
+
const cursor = await rt.guard.call(rt.groupId, 'store', () => readIngestCursor(rt.groupStore.artifact('ingestCursor')));
|
|
1246
|
+
if (!stillSyncing(rt))
|
|
1247
|
+
return undefined;
|
|
1248
|
+
const nowSeconds = Math.floor(rt.ctx.clock.now() / 1000);
|
|
1249
|
+
// Paginated per relay (history-pages.ts): relays cap each response, so
|
|
1250
|
+
// one request over a larger backlog would silently return its newest part.
|
|
1251
|
+
// A backlog too deep for one pass is walked down first and then ingested
|
|
1252
|
+
// oldest-first, window by window (IngestCursor.resume).
|
|
1253
|
+
const resuming = cursor?.resume;
|
|
1254
|
+
if (resuming === undefined) {
|
|
1255
|
+
// No walk in progress: this pass begins one.
|
|
1256
|
+
rt.walkPasses = 0;
|
|
1257
|
+
rt.walkStartedAt = nowSeconds;
|
|
1258
|
+
}
|
|
1259
|
+
rt.walkPasses += 1;
|
|
1260
|
+
// The post-subscribe catch-up reaches back to the walk's start (see startLiveSubscription).
|
|
1261
|
+
const catchUp = resuming === undefined ? rt.catchUpSince : undefined;
|
|
1262
|
+
// A pass reading the cursor and the catch-up floor afresh is what a rerun asked for: it is satisfied.
|
|
1263
|
+
// (A resumed walk pass reads neither floor; the rerun waits for the walk's end.)
|
|
1264
|
+
if (resuming === undefined)
|
|
1265
|
+
historyFlightOf(rt).rerun = undefined;
|
|
1266
|
+
const since = resuming !== undefined ? resuming.from : catchUp === undefined ? cursor?.since : Math.min(cursor?.since ?? catchUp, catchUp);
|
|
1267
|
+
const window = resuming ?? { from: since ?? 0 };
|
|
1268
|
+
const filter = {
|
|
1269
|
+
...groupEventFilter(nostrGroupId, since),
|
|
1270
|
+
...(window.until === undefined ? {} : { until: window.until }),
|
|
1271
|
+
};
|
|
1272
|
+
const result = await fetchHistoryPages((relays, pageFilter) => rt.guard.callSync(rt.groupId, 'transport', () => rt.ctx.transport.requestWithDeadline(relays, pageFilter, { timeoutMs: rt.ctx.historyTimeoutMs })), requestRelays, filter, {
|
|
1273
|
+
active: () => stillSyncing(rt),
|
|
1274
|
+
nowSeconds: nowSeconds + CURSOR_FUTURE_SKEW_S,
|
|
1275
|
+
// A resumed window was measured by an earlier walk: allow the re-walk's two extra requests.
|
|
1276
|
+
...(resuming === undefined ? {} : { maxPages: HISTORY_MAX_PAGES + HISTORY_REWALK_SLACK_PAGES }),
|
|
1277
|
+
});
|
|
1278
|
+
if (!stillSyncing(rt))
|
|
1279
|
+
return undefined;
|
|
1280
|
+
const { newlySilent } = rt.historyTracker.record(requestRelays, result);
|
|
1281
|
+
for (const relayUrl of newlySilent)
|
|
1282
|
+
rt.ctx.emit('relaySilent', { groupId: rt.groupId, relayUrl, reason: 'timeout' });
|
|
1283
|
+
for (const gap of result.gaps) {
|
|
1284
|
+
const key = `${gap.relay}\u241f${gap.createdAt}`;
|
|
1285
|
+
if (!gap.confirmed || rt.historyGapsReported.has(key))
|
|
1286
|
+
continue;
|
|
1287
|
+
rt.historyGapsReported.add(key);
|
|
1288
|
+
rt.ctx.emit('historyGap', { groupId: rt.groupId, relayUrl: gap.relay, createdAt: gap.createdAt });
|
|
1289
|
+
}
|
|
1290
|
+
if (result.timedOut) {
|
|
1291
|
+
// AC-PIPE-7: the whole batch — every page of the pass — is discarded, cursor untouched.
|
|
1292
|
+
scheduleHistoryRetry(rt, attempt);
|
|
1293
|
+
return undefined;
|
|
1294
|
+
}
|
|
1295
|
+
recordWalkProgress(rt, result.walks);
|
|
1296
|
+
const noTiers = { advanced: [], held: new Map(), pinned: new Map(), released: new Set() };
|
|
1297
|
+
if (!result.complete) {
|
|
1298
|
+
// Descending: ingest nothing yet — the events fetched may depend on
|
|
1299
|
+
// older ones below the edge this pass reached. Record the edge and
|
|
1300
|
+
// walk on down now; a pass that got no further backs off.
|
|
1301
|
+
const edge = result.resumeUntil;
|
|
1302
|
+
if (edge === undefined || edge < window.from || (window.until !== undefined && edge >= window.until)) {
|
|
1303
|
+
startLiveIfWalkIsLong(rt);
|
|
1304
|
+
scheduleHistoryRetry(rt, attempt);
|
|
1305
|
+
return undefined;
|
|
1306
|
+
}
|
|
1307
|
+
const above = window.until === undefined ? (window.above ?? []) : [window.until, ...(window.above ?? [])];
|
|
1308
|
+
const descended = { from: window.from, until: edge, ...(above.length === 0 ? {} : { above }) };
|
|
1309
|
+
const moved = await rt.run('ingest', async (rtc) => {
|
|
1310
|
+
if (stale(rtc))
|
|
1311
|
+
return true;
|
|
1312
|
+
await persistCursor(rtc, { ...noTiers, resume: descended });
|
|
1313
|
+
return false;
|
|
1314
|
+
});
|
|
1315
|
+
if (moved) {
|
|
1316
|
+
return attempt;
|
|
1317
|
+
}
|
|
1318
|
+
if (catchUp !== undefined && rt.catchUpSince === catchUp)
|
|
1319
|
+
rt.catchUpSince = undefined;
|
|
1320
|
+
if (!stillSyncing(rt))
|
|
1321
|
+
return undefined;
|
|
1322
|
+
startLiveIfWalkIsLong(rt);
|
|
1323
|
+
return attempt;
|
|
1324
|
+
}
|
|
1325
|
+
// A complete window. A pin or busy entry it should have returned but did
|
|
1326
|
+
// not has no relay left to redeliver it — release it, or it would hold
|
|
1327
|
+
// the cursor floor forever. Only entries of the cursor read before the
|
|
1328
|
+
// request was sent qualify (a later one may postdate the relay's EOSE),
|
|
1329
|
+
// and none stamped on a second a relay may have truncated.
|
|
1330
|
+
const absent = absentPins(cursor, {
|
|
1331
|
+
complete: true,
|
|
1332
|
+
since: filter.since,
|
|
1333
|
+
until: filter.until,
|
|
1334
|
+
gaps: result.gaps.map((gap) => gap.createdAt),
|
|
1335
|
+
wholePlane: plane.every((relay) => requestRelays.includes(relay)),
|
|
1336
|
+
}, new Set(result.events.map((e) => e.id)));
|
|
1337
|
+
// Ascending: the window above, if any, is next; the open-topped one's completion ends the walk.
|
|
1338
|
+
const next = resuming === undefined
|
|
1339
|
+
? undefined
|
|
1340
|
+
: window.until === undefined
|
|
1341
|
+
? null
|
|
1342
|
+
: {
|
|
1343
|
+
from: window.until + 1,
|
|
1344
|
+
...(window.above?.[0] === undefined ? {} : { until: window.above[0] }),
|
|
1345
|
+
...((window.above?.length ?? 0) > 1 ? { above: window.above.slice(1) } : {}),
|
|
1346
|
+
};
|
|
1347
|
+
const sorted = [...result.events].sort((a, b) => a.created_at - b.created_at);
|
|
1348
|
+
/** AC-PIPE-15: this pass is the group's first complete sync since it was loaded — the send gate opened. */
|
|
1349
|
+
let gateOpened = false;
|
|
1350
|
+
// A failed batch is not processed: it propagates to the catch below,
|
|
1351
|
+
// which retries the pass on the backoff (the cursor was not written).
|
|
1352
|
+
const moved = await rt.run('ingest', async (rtc) => {
|
|
1353
|
+
if (stale(rtc))
|
|
1354
|
+
return true;
|
|
1355
|
+
if (absent.size > 0)
|
|
1356
|
+
await persistCursor(rtc, { ...noTiers, released: absent });
|
|
1357
|
+
if (sorted.length > 0)
|
|
1358
|
+
await ingestEvents(rtc, sorted, { fromSync: true });
|
|
1359
|
+
// Only once the window is ingested may the walk move past it.
|
|
1360
|
+
if (next !== undefined)
|
|
1361
|
+
await persistCursor(rtc, { ...noTiers, resume: next });
|
|
1362
|
+
await pruneLedger(rtc);
|
|
1363
|
+
// A complete pass with no walk left to continue: the group caught up (GroupRuntime.caughtUp) — unless
|
|
1364
|
+
// connectivity was lost since this pass began (round 3): its fetch may predate the gap.
|
|
1365
|
+
if ((next === undefined || next === null) && rtc.connectivityGeneration === generation) {
|
|
1366
|
+
rtc.caughtUp = true;
|
|
1367
|
+
// AC-PIPE-15: the first complete sync since the load opens the send gate (it latches; a fence set by this
|
|
1368
|
+
// very batch stopped the sync and bumped the generation, so a fencing pass never gets here).
|
|
1369
|
+
if (!rtc.sendGateOpen) {
|
|
1370
|
+
rtc.sendGateOpen = true;
|
|
1371
|
+
gateOpened = true;
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
// S15 (AC-INVITE-4): each sync pass evaluates the deferral deadlines over the state it just caught up to
|
|
1375
|
+
// (a no-op for expiry until the marker is set). Its own failure never fails the pass: the deadline is
|
|
1376
|
+
// durable and the next evaluation point retries.
|
|
1377
|
+
await expireDeferredProposals(rtc).catch(() => undefined);
|
|
1378
|
+
return false;
|
|
1379
|
+
});
|
|
1380
|
+
if (moved) {
|
|
1381
|
+
// Fetched from the old routing: nothing of it was ingested or released. Again, on the new one.
|
|
1382
|
+
return attempt;
|
|
1383
|
+
}
|
|
1384
|
+
if (catchUp !== undefined && rt.catchUpSince === catchUp)
|
|
1385
|
+
rt.catchUpSince = undefined;
|
|
1386
|
+
// The awaiting-sync intents go out now, encrypted under the synced state.
|
|
1387
|
+
if (gateOpened)
|
|
1388
|
+
releaseAwaitingSync(rt);
|
|
1389
|
+
if (!stillSyncing(rt))
|
|
1390
|
+
return undefined;
|
|
1391
|
+
// Mid-walk, history owns the backlog: a live subscription started now
|
|
1392
|
+
// would replay the relay's newest events ahead of the windows below them
|
|
1393
|
+
// — unless the walk has run long (liveness bound).
|
|
1394
|
+
if (next !== undefined && next !== null) {
|
|
1395
|
+
startLiveIfWalkIsLong(rt);
|
|
1396
|
+
return attempt;
|
|
1397
|
+
}
|
|
1398
|
+
rt.walkPasses = 0;
|
|
1399
|
+
startLiveSubscription(rt);
|
|
1400
|
+
// The walk is done. Live events that arrived meanwhile (on a subscription
|
|
1401
|
+
// already open before it began) ran ahead of the backlog and may sit
|
|
1402
|
+
// deferred in the cursor's tiers: one ordinary pass from the cursor, on
|
|
1403
|
+
// the backoff, redelivers them.
|
|
1404
|
+
if (next === null)
|
|
1405
|
+
scheduleHistoryRetry(rt, attempt);
|
|
1406
|
+
}
|
|
1407
|
+
catch {
|
|
1408
|
+
// A transport-level rejection (not a `timedOut: true` result), a failed
|
|
1409
|
+
// ingest batch, or a store call that lost its lease mid-flight — retried
|
|
1410
|
+
// on the same backoff as an ordinary timeout, only while still syncing.
|
|
1411
|
+
if (stillSyncing(rt))
|
|
1412
|
+
scheduleHistoryRetry(rt, attempt);
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
/**
|
|
1416
|
+
* Liveness of the history walk: a relay whose walk ends incomplete without
|
|
1417
|
+
* a single event inside the window (it ignores `until`, or answers with
|
|
1418
|
+
* events outside the filter) never lets the walk advance, yet is never
|
|
1419
|
+
* silent. After `historySilenceThreshold` such consecutive passes it is
|
|
1420
|
+
* excluded from history like a silent relay (`relaySilent`, reason
|
|
1421
|
+
* `no-progress`); any pass in which its walk completes or advances resets it.
|
|
1422
|
+
*/
|
|
1423
|
+
function recordWalkProgress(rt, walks) {
|
|
1424
|
+
for (const walk of walks) {
|
|
1425
|
+
if (walk.complete || walk.advanced) {
|
|
1426
|
+
rt.historyStalls.delete(walk.relay);
|
|
1427
|
+
continue;
|
|
1428
|
+
}
|
|
1429
|
+
const stalls = (rt.historyStalls.get(walk.relay) ?? 0) + 1;
|
|
1430
|
+
rt.historyStalls.set(walk.relay, stalls);
|
|
1431
|
+
if (stalls >= rt.ctx.historySilenceThreshold && rt.historyTracker.exclude(walk.relay)) {
|
|
1432
|
+
rt.ctx.emit('relaySilent', { groupId: rt.groupId, relayUrl: walk.relay, reason: 'no-progress' });
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
/**
|
|
1437
|
+
* The live subscription waits for the backlog walk — but not forever: after
|
|
1438
|
+
* {@link HISTORY_LIVE_AFTER_WALK_PASSES} passes of one walk it starts anyway,
|
|
1439
|
+
* and events it delivers ahead of the backlog are held or deferred in the
|
|
1440
|
+
* cursor's tiers until the walk (and the pass that follows it) redelivers them.
|
|
1441
|
+
*/
|
|
1442
|
+
function startLiveIfWalkIsLong(rt) {
|
|
1443
|
+
if (rt.walkPasses >= HISTORY_LIVE_AFTER_WALK_PASSES)
|
|
1444
|
+
startLiveSubscription(rt);
|
|
1445
|
+
}
|
|
1446
|
+
function scheduleHistoryRetry(rt, attempt) {
|
|
1447
|
+
if (!stillSyncing(rt) || rt.syncHistoryRetryTimer !== undefined)
|
|
1448
|
+
return;
|
|
1449
|
+
rt.syncHistoryRetryTimer = rt.ctx.timers.set(() => {
|
|
1450
|
+
rt.syncHistoryRetryTimer = undefined;
|
|
1451
|
+
void runHistoryPass(rt, attempt + 1);
|
|
1452
|
+
}, backoffMs(attempt));
|
|
1453
|
+
}
|
|
1454
|
+
function startLiveSubscription(rt) {
|
|
1455
|
+
if (!stillSyncing(rt) || rt.syncSubscription !== undefined)
|
|
1456
|
+
return;
|
|
1457
|
+
void rt.guard
|
|
1458
|
+
.call(rt.groupId, 'store', () => readIngestCursor(rt.groupStore.artifact('ingestCursor')))
|
|
1459
|
+
.then((cursor) => {
|
|
1460
|
+
if (!stillSyncing(rt) || rt.syncSubscription !== undefined)
|
|
1461
|
+
return;
|
|
1462
|
+
// The FULL plane — AC-PIPE-14's silence exclusion is history-only. Read
|
|
1463
|
+
// now, not before the cursor read: the routing may have moved meanwhile.
|
|
1464
|
+
const routing = syncRoutingOf(rt);
|
|
1465
|
+
const { plane, nostrGroupId } = routing;
|
|
1466
|
+
// History owns the backlog: live starts at the cursor's `high` (the
|
|
1467
|
+
// newest AUTHENTICATED event processed) minus the margin, so a relay's
|
|
1468
|
+
// capped replay of an old window never races the oldest-first walk.
|
|
1469
|
+
// Not the local clock (one running ahead would push `since` past live
|
|
1470
|
+
// events), and not any fetched event's `created_at` — that is stamped
|
|
1471
|
+
// by its publisher, so unauthenticated junk could set it anywhere.
|
|
1472
|
+
// Whatever reached a relay between the history pass and this REQ below
|
|
1473
|
+
// `since` is fetched by the catch-up pass that follows the subscribe.
|
|
1474
|
+
const since = cursor?.high === undefined ? cursor?.since : Math.max(cursor.since, cursor.high - CURSOR_REDELIVERY_MARGIN_S);
|
|
1475
|
+
const filter = groupEventFilter(nostrGroupId, since);
|
|
1476
|
+
let errored = false;
|
|
1477
|
+
let subscription;
|
|
1478
|
+
const ended = () => {
|
|
1479
|
+
if (errored)
|
|
1480
|
+
return;
|
|
1481
|
+
errored = true;
|
|
1482
|
+
// The transport ended live delivery (raw-ws: every relay socket of the plane closed): a connectivity gap.
|
|
1483
|
+
markConnectivityGap(rt);
|
|
1484
|
+
if (subscription !== undefined && rt.syncSubscription === subscription)
|
|
1485
|
+
rt.syncSubscription = undefined;
|
|
1486
|
+
// Back off before re-syncing: a relay plane that ends every
|
|
1487
|
+
// subscription at once would otherwise spin history → subscribe → end.
|
|
1488
|
+
scheduleHistoryRetry(rt, 1);
|
|
1489
|
+
};
|
|
1490
|
+
const subscribable = rt.guard.callSync(rt.groupId, 'transport', () => rt.ctx.transport.subscription(plane, filter));
|
|
1491
|
+
subscription = rt.guard.callSync(rt.groupId, 'transport', () => subscribable.subscribe({
|
|
1492
|
+
next: (event) => {
|
|
1493
|
+
// Deferred past the transport frame: a transport may deliver
|
|
1494
|
+
// synchronously from inside subscribe().
|
|
1495
|
+
void Promise.resolve()
|
|
1496
|
+
.then(() => rt.run('ingest', (rtc) => ingestEvents(rtc, [event], { fromSync: true })))
|
|
1497
|
+
.catch(() => retryFailedLiveEvent(rt, event));
|
|
1498
|
+
},
|
|
1499
|
+
error: () => ended(),
|
|
1500
|
+
// A transport may also end a subscription cleanly; treat it like an error, or live delivery stops for good.
|
|
1501
|
+
complete: () => ended(),
|
|
1502
|
+
}));
|
|
1503
|
+
if (errored || !stillSyncing(rt)) {
|
|
1504
|
+
// Errored synchronously inside subscribe() (or torn down meanwhile):
|
|
1505
|
+
// never keep a dead subscription, or no later pass would resubscribe.
|
|
1506
|
+
subscription.unsubscribe();
|
|
1507
|
+
return;
|
|
1508
|
+
}
|
|
1509
|
+
rt.syncSubscription = subscription;
|
|
1510
|
+
rt.syncRouting = routing.key;
|
|
1511
|
+
// Catch-up: one history pass reaching back to the start of the last
|
|
1512
|
+
// walk (minus the margin) fetches whatever reached a relay after that
|
|
1513
|
+
// walk read it but below this subscription's `since` (never raising a
|
|
1514
|
+
// floor a failed live event already lowered it to).
|
|
1515
|
+
lowerCatchUpSince(rt, (rt.walkStartedAt ?? Math.floor(rt.ctx.clock.now() / 1000)) - CURSOR_REDELIVERY_MARGIN_S);
|
|
1516
|
+
requestHistoryResync(rt);
|
|
1517
|
+
})
|
|
1518
|
+
.catch(() => {
|
|
1519
|
+
// The cursor read lost its lease (e.g. stores closed mid-flight, or
|
|
1520
|
+
// the client was disposed), or the transport threw synchronously:
|
|
1521
|
+
// retried on the history backoff while still syncing; `stopGroupSync`
|
|
1522
|
+
// already tore down anything that needed tearing down otherwise.
|
|
1523
|
+
if (stillSyncing(rt))
|
|
1524
|
+
scheduleHistoryRetry(rt, 1);
|
|
1525
|
+
});
|
|
1526
|
+
}
|
|
1527
|
+
function lowerCatchUpSince(rt, since) {
|
|
1528
|
+
// Never below the join floor (owner decision 20; less the redelivery margin every reach-back keeps): nothing before it is readable here.
|
|
1529
|
+
if (rt.joinedAtMs !== undefined)
|
|
1530
|
+
since = Math.max(since, Math.floor(rt.joinedAtMs / 1000) - CURSOR_FUTURE_SKEW_S - CURSOR_REDELIVERY_MARGIN_S);
|
|
1531
|
+
rt.catchUpSince = rt.catchUpSince === undefined ? since : Math.min(rt.catchUpSince, since);
|
|
1532
|
+
}
|
|
1533
|
+
/**
|
|
1534
|
+
* A live event whose ingestion failed (a transient store or transport
|
|
1535
|
+
* failure — possibly before the engine was touched, so no rollback asked for
|
|
1536
|
+
* a resync) is never dropped: the next ordinary history pass reaches back to
|
|
1537
|
+
* it (its `created_at`, capped at now and floored at the provable pre-join
|
|
1538
|
+
* bound, minus the margin — later events may meanwhile have advanced the
|
|
1539
|
+
* cursor past it) and runs on the backoff. The ledger dedups whatever that
|
|
1540
|
+
* pass refetches that did get processed.
|
|
1541
|
+
*/
|
|
1542
|
+
function retryFailedLiveEvent(rt, event) {
|
|
1543
|
+
if (!stillSyncing(rt))
|
|
1544
|
+
return;
|
|
1545
|
+
let at = Math.min(event.created_at, Math.floor(rt.ctx.clock.now() / 1000));
|
|
1546
|
+
if (rt.joinedAtMs !== undefined)
|
|
1547
|
+
at = Math.max(at, Math.floor(rt.joinedAtMs / 1000) - CURSOR_FUTURE_SKEW_S);
|
|
1548
|
+
lowerCatchUpSince(rt, at - CURSOR_REDELIVERY_MARGIN_S);
|
|
1549
|
+
scheduleHistoryRetry(rt, 1);
|
|
1550
|
+
}
|