@941design/marmot 0.0.0-stage → 0.2.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +709 -0
- package/LICENSE +21 -0
- package/README.md +33 -2
- package/dist/account/admin.d.ts +72 -0
- package/dist/account/admin.js +209 -0
- package/dist/account/device-deps.d.ts +379 -0
- package/dist/account/device-deps.js +36 -0
- package/dist/account/device-lineage.d.ts +199 -0
- package/dist/account/device-lineage.js +390 -0
- package/dist/account/device-registry.d.ts +93 -0
- package/dist/account/device-registry.js +1290 -0
- package/dist/account/device-rotation.d.ts +61 -0
- package/dist/account/device-rotation.js +827 -0
- package/dist/account/enrolment.d.ts +339 -0
- package/dist/account/enrolment.js +1247 -0
- package/dist/account/identity-proof.d.ts +79 -0
- package/dist/account/identity-proof.js +264 -0
- package/dist/account/index.d.ts +17 -0
- package/dist/account/index.js +16 -0
- package/dist/account/restore-rotation.d.ts +21 -0
- package/dist/account/restore-rotation.js +22 -0
- package/dist/account/same-account-authorization.d.ts +86 -0
- package/dist/account/same-account-authorization.js +180 -0
- package/dist/account/teardown.d.ts +127 -0
- package/dist/account/teardown.js +856 -0
- package/dist/client/before-create-group.d.ts +18 -0
- package/dist/client/before-create-group.js +65 -0
- package/dist/client/before-publish-hooks.d.ts +121 -0
- package/dist/client/before-publish-hooks.js +180 -0
- package/dist/client/client-config.d.ts +42 -0
- package/dist/client/client-config.js +13 -0
- package/dist/client/commit-redrive.d.ts +68 -0
- package/dist/client/commit-redrive.js +221 -0
- package/dist/client/component-update.d.ts +43 -0
- package/dist/client/component-update.js +53 -0
- package/dist/client/confirm-hooks.d.ts +50 -0
- package/dist/client/confirm-hooks.js +59 -0
- package/dist/client/create-client.d.ts +1022 -0
- package/dist/client/create-client.js +1125 -0
- package/dist/client/group-disband.d.ts +28 -0
- package/dist/client/group-disband.js +64 -0
- package/dist/client/group-history.d.ts +435 -0
- package/dist/client/group-history.js +1295 -0
- package/dist/client/group-profile.d.ts +31 -0
- package/dist/client/group-profile.js +62 -0
- package/dist/client/group-records.d.ts +485 -0
- package/dist/client/group-records.js +1034 -0
- package/dist/client/group-removal.d.ts +36 -0
- package/dist/client/group-removal.js +127 -0
- package/dist/client/group-remove-member.d.ts +49 -0
- package/dist/client/group-remove-member.js +163 -0
- package/dist/client/group-runtime.d.ts +388 -0
- package/dist/client/group-runtime.js +2701 -0
- package/dist/client/group-snapshot.d.ts +98 -0
- package/dist/client/group-snapshot.js +453 -0
- package/dist/client/history-pages.d.ts +107 -0
- package/dist/client/history-pages.js +194 -0
- package/dist/client/inbox-send.d.ts +19 -0
- package/dist/client/inbox-send.js +71 -0
- package/dist/client/index.d.ts +21 -0
- package/dist/client/index.js +13 -0
- package/dist/client/ingest.d.ts +82 -0
- package/dist/client/ingest.js +1550 -0
- package/dist/client/lifecycle-events.d.ts +63 -0
- package/dist/client/lifecycle-events.js +35 -0
- package/dist/client/opaque-candidates.d.ts +81 -0
- package/dist/client/opaque-candidates.js +169 -0
- package/dist/client/own-change-outbox.d.ts +66 -0
- package/dist/client/own-change-outbox.js +156 -0
- package/dist/client/pending-leaf.d.ts +116 -0
- package/dist/client/pending-leaf.js +378 -0
- package/dist/client/permissions.d.ts +9 -0
- package/dist/client/permissions.js +12 -0
- package/dist/client/pipeline.d.ts +554 -0
- package/dist/client/pipeline.js +3545 -0
- package/dist/client/reentrancy-guard.d.ts +96 -0
- package/dist/client/reentrancy-guard.js +153 -0
- package/dist/client/rekey.d.ts +58 -0
- package/dist/client/rekey.js +149 -0
- package/dist/client/transport-policy.d.ts +181 -0
- package/dist/client/transport-policy.js +214 -0
- package/dist/history/equivocation.d.ts +55 -0
- package/dist/history/equivocation.js +113 -0
- package/dist/history/fencing.d.ts +25 -0
- package/dist/history/fencing.js +61 -0
- package/dist/history/hex.d.ts +3 -0
- package/dist/history/hex.js +10 -0
- package/dist/history/index.d.ts +73 -0
- package/dist/history/index.js +31 -0
- package/dist/history/notifications.d.ts +270 -0
- package/dist/history/notifications.js +264 -0
- package/dist/history/ordering-key.d.ts +24 -0
- package/dist/history/ordering-key.js +101 -0
- package/dist/history/own-change-fork.d.ts +112 -0
- package/dist/history/own-change-fork.js +196 -0
- package/dist/history/own-change.d.ts +106 -0
- package/dist/history/own-change.js +146 -0
- package/dist/history/policy.d.ts +48 -0
- package/dist/history/policy.js +52 -0
- package/dist/history/retained-window.d.ts +215 -0
- package/dist/history/retained-window.js +565 -0
- package/dist/inbox/demux.d.ts +90 -0
- package/dist/inbox/demux.js +78 -0
- package/dist/inbox/handler-registry.d.ts +42 -0
- package/dist/inbox/handler-registry.js +37 -0
- package/dist/inbox/index.d.ts +11 -0
- package/dist/inbox/index.js +10 -0
- package/dist/inbox/seen-set.d.ts +82 -0
- package/dist/inbox/seen-set.js +202 -0
- package/dist/inbox/subscription.d.ts +236 -0
- package/dist/inbox/subscription.js +644 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +9 -0
- package/dist/invitations/competing-welcome.d.ts +223 -0
- package/dist/invitations/competing-welcome.js +128 -0
- package/dist/invitations/decline.d.ts +61 -0
- package/dist/invitations/decline.js +143 -0
- package/dist/invitations/expect-welcome.d.ts +66 -0
- package/dist/invitations/expect-welcome.js +138 -0
- package/dist/invitations/index.d.ts +16 -0
- package/dist/invitations/index.js +16 -0
- package/dist/invitations/invite.d.ts +303 -0
- package/dist/invitations/invite.js +966 -0
- package/dist/invitations/offers.d.ts +79 -0
- package/dist/invitations/offers.js +115 -0
- package/dist/invitations/on-proposal.d.ts +47 -0
- package/dist/invitations/on-proposal.js +40 -0
- package/dist/invitations/pending-proposal-policy.d.ts +52 -0
- package/dist/invitations/pending-proposal-policy.js +139 -0
- package/dist/invitations/proposal-deadline.d.ts +133 -0
- package/dist/invitations/proposal-deadline.js +164 -0
- package/dist/invitations/relay-planes.d.ts +43 -0
- package/dist/invitations/relay-planes.js +70 -0
- package/dist/invitations/welcome-processor.d.ts +253 -0
- package/dist/invitations/welcome-processor.js +1180 -0
- package/dist/key-packages/device-key.d.ts +131 -0
- package/dist/key-packages/device-key.js +272 -0
- package/dist/key-packages/device-rotation-records.d.ts +83 -0
- package/dist/key-packages/device-rotation-records.js +173 -0
- package/dist/key-packages/group-refs.d.ts +40 -0
- package/dist/key-packages/group-refs.js +79 -0
- package/dist/key-packages/index.d.ts +18 -0
- package/dist/key-packages/index.js +18 -0
- package/dist/key-packages/key-package-ref.d.ts +16 -0
- package/dist/key-packages/key-package-ref.js +21 -0
- package/dist/key-packages/observation-stream.d.ts +4 -0
- package/dist/key-packages/observation-stream.js +120 -0
- package/dist/key-packages/offer-rumor.d.ts +26 -0
- package/dist/key-packages/offer-rumor.js +69 -0
- package/dist/key-packages/publisher.d.ts +174 -0
- package/dist/key-packages/publisher.js +1247 -0
- package/dist/key-packages/select-key-package.d.ts +24 -0
- package/dist/key-packages/select-key-package.js +90 -0
- package/dist/key-packages/slot-manager.d.ts +167 -0
- package/dist/key-packages/slot-manager.js +327 -0
- package/dist/key-packages/validate-key-package-event.d.ts +3 -0
- package/dist/key-packages/validate-key-package-event.js +49 -0
- package/dist/media/cache.d.ts +14 -0
- package/dist/media/cache.js +10 -0
- package/dist/media/index.d.ts +52 -0
- package/dist/media/index.js +71 -0
- package/dist/mls.d.ts +15 -0
- package/dist/mls.js +15 -0
- package/dist/shared/account-pubkey.d.ts +4 -0
- package/dist/shared/account-pubkey.js +10 -0
- package/dist/shared/errors.d.ts +21 -0
- package/dist/shared/errors.js +85 -0
- package/dist/shared/hex.d.ts +4 -0
- package/dist/shared/hex.js +13 -0
- package/dist/shared/index.d.ts +14 -0
- package/dist/shared/index.js +14 -0
- package/dist/shared/kinds.d.ts +8 -0
- package/dist/shared/kinds.js +28 -0
- package/dist/shared/publish-classification.d.ts +26 -0
- package/dist/shared/publish-classification.js +31 -0
- package/dist/storage/envelope.d.ts +39 -0
- package/dist/storage/envelope.js +201 -0
- package/dist/storage/index.d.ts +227 -0
- package/dist/storage/index.js +459 -0
- package/dist/storage/indexeddb/idb-types.d.ts +68 -0
- package/dist/storage/indexeddb/idb-types.js +1 -0
- package/dist/storage/indexeddb/index.d.ts +17 -0
- package/dist/storage/indexeddb/index.js +141 -0
- package/dist/storage/lease.d.ts +41 -0
- package/dist/storage/lease.js +117 -0
- package/dist/storage/memory/deterministic-signer.d.ts +27 -0
- package/dist/storage/memory/deterministic-signer.js +65 -0
- package/dist/storage/memory/index.d.ts +32 -0
- package/dist/storage/memory/index.js +87 -0
- package/dist/storage/memory/lock-manager.d.ts +8 -0
- package/dist/storage/memory/lock-manager.js +71 -0
- package/package.json +66 -3
|
@@ -0,0 +1,3545 @@
|
|
|
1
|
+
import { commitDigest, deserializeClientState, getMarmotGroupView, getKeyPackage, GROUP_PROFILE_COMPONENT_ID, getPubkeyLeafNodes, serializeApplicationRumor, serializeClientState, } from '@941design/marmot-engine/core';
|
|
2
|
+
import { decideAutoCommit, decodeOwnCommitRecord, deriveStateNotifications, encodeOwnCommitRecord, nextOwnApplicationGeneration } from '@941design/marmot-engine/engine';
|
|
3
|
+
import { decode, defaultProposalTypes, encode, getCredentialFromLeafIndex, mlsMessageDecoder, mlsMessageEncoder, nodeTypes, selfRemoveProposalType, } from '@941design/marmot-engine/mls';
|
|
4
|
+
import { adminPolicyProposal, directedAncestors, enforceStateTransition, isActiveAdmin, readRegistryEdges, REOPENED_LEAF_STILL_CANONICAL, validateAdminList } from '../account/index.js';
|
|
5
|
+
import { addProposalsFor, buildWelcomeRumors, deleteDeferral, deleteExclusionEpochs, DEFERRED_PROPOSALS_EXTENSION, EXCLUDED_PROPOSALS_EXTENSION, exclusionsAt, hasStagedAdds, isAuthorisedCommitter, mustQueue, nextQueueStep, planDeferrals, queueBarrier, readDeferrals, readExclusions, recipientsOfWelcome, resolveProposals, STAGED_ADD_LOOKUP_TIMEOUT_MS, stagedAddRecipients, stagedProposals, validateInvitees, withoutProposals, writeDeferral, writeExclusion, writeExclusionSnapshot, } from '../invitations/index.js';
|
|
6
|
+
import { isHex64, MarmotError, isMarmotError } from '../shared/index.js';
|
|
7
|
+
import { readDeviceKeyLink } from '../key-packages/index.js';
|
|
8
|
+
import { assertNoStagedComponentUpdate, componentUpdateProposal, genericComponentGuard, guardedComponentUpdate } from './component-update.js';
|
|
9
|
+
import { assertGroupProfile, groupProfileAction, profileUpdateGuard } from './group-profile.js';
|
|
10
|
+
import { assertAccountPubkey, holdsLeafOfAny, holdsSnapshotLeaf, leafKeysOfAccount, removeProposalsForAccounts, stagedAddConflict } from './group-remove-member.js';
|
|
11
|
+
import { canonicalPath, commitViewOf, extendWindow, holdsLeafKey as holdsLeafKeyOfView, withForkNode, fencedError, fencingErrorIn, isFencingError, orderingKeyOf, REWIND_HORIZON, tagOf, windowFor, writeWindow, } from '../history/index.js';
|
|
12
|
+
import { assertNotDisbanded } from './group-disband.js';
|
|
13
|
+
import { changesNeedingKeyPackageCheck, decideChanges } from './commit-redrive.js';
|
|
14
|
+
import { announceTerminals, assertNotFenced, deliverTransition, dispatch, fenceGroup, lostMessagePlan, parkWithdrawnWelcomes, planTransition, recordEquivocations, recordTransition, sentOnCanonicalBranch, WITHDRAWN_DISPOSITION } from './group-history.js';
|
|
15
|
+
import { clearRestorePending, deletePendingIntentRecord, listPendingIntentRecords, listQueuedRumorRecords, markRatchetSkipDelivered, nextSeq, OWN_COMMIT_EXTENSION, RATCHET_SKIP_EXTENSION, randomIntentId, RESTORE_PENDING_EXTENSION, readPendingIntentRecord, readQueuedRumorRecord, writeGroupState, raiseRootSpend, writeOwnCommitMarker, writePendingIntentRecord, writeQueuedRumorRecord, writeRatchetSkipMarker, writeTerminalLedgerEntry, isCommitRedrive, } from './group-records.js';
|
|
16
|
+
import { enqueueTerminals, OWN_CHANGE_OUTBOX_EXTENSION, terminalKey } from './own-change-outbox.js';
|
|
17
|
+
import { runConfirmHooks } from './confirm-hooks.js';
|
|
18
|
+
import { accountRead, ownLeafKeyOf, planRekey, readSelfKey } from './rekey.js';
|
|
19
|
+
import { runBeforePublish } from './before-publish-hooks.js';
|
|
20
|
+
import { ACCOUNT_SCOPE } from './reentrancy-guard.js';
|
|
21
|
+
import { classifyPublish, groupRelayPlane } from './transport-policy.js';
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
// Small helpers
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
function toHex(bytes) {
|
|
26
|
+
return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
|
|
27
|
+
}
|
|
28
|
+
export function assertNotDisposed(rt) {
|
|
29
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
30
|
+
throw new MarmotError('disposed', 'group operation aborted: client has been disposed');
|
|
31
|
+
}
|
|
32
|
+
function relayPlaneFor(rt) {
|
|
33
|
+
const view = getMarmotGroupView(rt.engine.state);
|
|
34
|
+
return groupRelayPlane(view?.relays ?? [], rt.ctx.relays.defaults);
|
|
35
|
+
}
|
|
36
|
+
/** Commit-kind intents produce an own commit (or a proposal that changes the next commit) at the parent epoch. */
|
|
37
|
+
function isCommitKind(kind) {
|
|
38
|
+
return kind === 'commit' || kind === 'proposal' || kind === 'self-update';
|
|
39
|
+
}
|
|
40
|
+
export function backoffMs(count) {
|
|
41
|
+
if (count <= 0)
|
|
42
|
+
return 0;
|
|
43
|
+
return Math.min(1000 * 2 ** (count - 1), 60_000);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* S5 (AC-RACE-1): a record re-enqueued after a rewind reports failure under the ORIGINAL intent id
|
|
47
|
+
* (`reenqueuedFrom`) — the one a caller/consumer actually tracked — not the fresh id this specific
|
|
48
|
+
* re-enqueue attempt was given. A record that was never re-enqueued reports its own id, unchanged.
|
|
49
|
+
*/
|
|
50
|
+
function reportedIntentIdOf(record) {
|
|
51
|
+
return record.reenqueuedFrom ?? record.intentId;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Epic marmot-rewound-own-commit-no-signal (S9, AC-MARK-1/2, DD2/DD10/DD17): whether `record` is an own commit whose
|
|
55
|
+
* intent DECLARES a change a later rewind could lose — `removeAccounts`, `invite`, `admins`, `profile` or
|
|
56
|
+
* `components` (or a re-drive record's `changes`). Not a self-update / rekey (its own `selfUpdateIntentId` owner), not
|
|
57
|
+
* `removeLeafKeys` (`devices.forget`, `reopenForgetsAfterRewind`), not `origin: 'approval'` (enrolment,
|
|
58
|
+
* `reopenApprovalsAfterRewind`), and not a commit declaring nothing (`{ type: 'commit' }`: the S13 Add-commit, a bare
|
|
59
|
+
* `send`, disband) — those stay unmarked and unlinked. The one predicate behind the marker write and the DD17 link.
|
|
60
|
+
*/
|
|
61
|
+
function declaresOwnChange(record) {
|
|
62
|
+
const intent = record.intent;
|
|
63
|
+
// `kind` and `intent.type` agree for every record the pipeline writes (a self-update/rekey is `kind: 'self-update'` with
|
|
64
|
+
// a `self-update` intent; the rekey case is pinned in own-commit-marker-records.test.ts), so the kind test is a
|
|
65
|
+
// belt-and-braces guard that no input can make observable: it is deliberately redundant with the intent test.
|
|
66
|
+
if (record.kind !== 'commit' || intent.type !== 'commit')
|
|
67
|
+
return false;
|
|
68
|
+
if (intent.removeLeafKeys !== undefined || intent.origin !== undefined)
|
|
69
|
+
return false;
|
|
70
|
+
return intent.removeAccounts !== undefined || intent.invite !== undefined || intent.admins !== undefined || intent.profile !== undefined || intent.components !== undefined || record.changes !== undefined;
|
|
71
|
+
}
|
|
72
|
+
/** The changes an intent declares besides an invite (DD2): what the withdrawal-time breakdown keeps of the state diff. */
|
|
73
|
+
function declaredOf(intent) {
|
|
74
|
+
const components = [...new Set([...(intent.profile === undefined ? [] : [GROUP_PROFILE_COMPONENT_ID]), ...(intent.components ?? []).map((c) => c.id)])];
|
|
75
|
+
const declared = {
|
|
76
|
+
...(intent.removeAccounts === undefined ? {} : { removed: [...intent.removeAccounts] }),
|
|
77
|
+
...(intent.admins === undefined ? {} : { admins: true }),
|
|
78
|
+
...(components.length === 0 ? {} : { components }),
|
|
79
|
+
};
|
|
80
|
+
return Object.keys(declared).length === 0 ? undefined : declared;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* The marker an own commit that {@link declaresOwnChange} leaves at confirm. An intent's marker carries `invitees` (each
|
|
84
|
+
* KeyPackage event, its ref and its leaf signature key, DD8/DD16) and what the intent `declared` (DD2). A re-drive record's
|
|
85
|
+
* marker carries the record's own changes instead, verdict-stripped and without the ones already decided lost (the
|
|
86
|
+
* re-drive commit carries the survivors; a later withdrawal of it re-drives exactly those), its `forkEpoch`, and the invitees
|
|
87
|
+
* its `member-added` changes name — so each marked node of the root carries what a later withdrawal needs, even once the
|
|
88
|
+
* original's marker was pruned (DD6, S1 lead decision). The fork-point values on a marker's changes are NOT authoritative (the codec
|
|
89
|
+
* requires them): a withdrawal uses the list only as a filter by change identity and recomputes the values.
|
|
90
|
+
*/
|
|
91
|
+
function ownCommitMarkerOf(record, node) {
|
|
92
|
+
// DD9 (AC-DRIVE-4): the publish budget is carried across the re-drives of one root. A confirmed re-drive was published, which
|
|
93
|
+
// `record.attempts` (its failed publishes and re-mints so far) does not count yet: the marker, and with it the re-drive
|
|
94
|
+
// record a later withdrawal of this node writes, carries one more.
|
|
95
|
+
const base = { v: 1, root: reportedIntentIdOf(record), node, attempts: record.attempts + (isCommitRedrive(record) ? 1 : 0) };
|
|
96
|
+
if (record.changes !== undefined) {
|
|
97
|
+
const kept = record.changes.filter((c) => c.verdict === undefined || c.verdict === 'reapplied').map(({ verdict: _verdict, ...rest }) => rest);
|
|
98
|
+
const invitees = kept.flatMap((c) => (c.change.type === 'member-added' ? [{ event: c.keyPackage.event, inviteeRef: c.keyPackage.inviteeRef, leafKey: c.change.leafKey }] : []));
|
|
99
|
+
return {
|
|
100
|
+
...base,
|
|
101
|
+
...(invitees.length === 0 ? {} : { invitees }),
|
|
102
|
+
...(kept.length === 0 ? {} : { changes: kept, forkEpoch: record.forkEpoch }),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
const intent = record.intent.type === 'commit' ? record.intent : undefined;
|
|
106
|
+
const invite = intent?.invite;
|
|
107
|
+
const declared = intent === undefined ? undefined : declaredOf(intent);
|
|
108
|
+
return {
|
|
109
|
+
...base,
|
|
110
|
+
...(invite === undefined
|
|
111
|
+
? {}
|
|
112
|
+
: { invitees: invite.invitees.map((i) => ({ event: i.event, inviteeRef: i.inviteeRef, leafKey: toHex(getKeyPackage(i.event).leafNode.signaturePublicKey) })) }),
|
|
113
|
+
...(declared === undefined ? {} : { declared }),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
async function terminalFail(rt, record, hasCaller, cause, extraDetails = {}) {
|
|
117
|
+
const reportedIntentId = reportedIntentIdOf(record);
|
|
118
|
+
// A typed refusal raised while building (a MarmotError with a `details.reason`) keeps its reason on the terminal error.
|
|
119
|
+
const causeReason = isMarmotError(cause) ? cause.details?.reason : undefined;
|
|
120
|
+
const err = new MarmotError('send-rejected', `send: intent ${reportedIntentId} on group ${rt.groupId} is terminal`, {
|
|
121
|
+
cause,
|
|
122
|
+
details: { groupId: rt.groupId, intentId: reportedIntentId, ...(typeof causeReason === 'string' ? { reason: causeReason } : {}), ...extraDetails },
|
|
123
|
+
});
|
|
124
|
+
// An own commit refused at the epoch it was built for (every relay refused it, the engine or the
|
|
125
|
+
// proof gate did): the queue must not rebuild the same commit right away (see holdQueueCommit).
|
|
126
|
+
if ((record.kind === 'commit' || record.kind === 'self-update') && record.parentEpoch === rt.engine.state.groupContext.epoch)
|
|
127
|
+
holdQueueCommit(rt);
|
|
128
|
+
// AC-RACE-1: a commit-kind record (a proposal included — a started one blocks a re-enqueue's drive just the
|
|
129
|
+
// same) going terminal clears "another own commit pending" exactly like one confirming does — and is the more
|
|
130
|
+
// likely outcome for whatever a re-enqueue's drive was waiting behind, since the rewind that produced the
|
|
131
|
+
// re-enqueue tends to leave an own attempt epoch-stale. Retried the same way confirmRecord does, detached.
|
|
132
|
+
if (isCommitKind(record.kind))
|
|
133
|
+
scheduleDeferredReenqueueRetry(rt, record.intentId);
|
|
134
|
+
// Epic marmot-rewound-own-commit-no-signal S2 (DD7, AC-PROP-1): a commit re-drive that goes terminal ends every change it
|
|
135
|
+
// still owed with an `own-change-lost`, so no `own-change-withdrawn` is left without a terminal. `intentFailed` below is
|
|
136
|
+
// unchanged (it fires under the root id, as for a self-update re-enqueue). S11: the verdicts were written to the outbox in the
|
|
137
|
+
// transaction that deleted the record (`deleteEndingRecord`, with `lostReasonOf(extraDetails)`); this hands them over.
|
|
138
|
+
if (isCommitRedrive(record))
|
|
139
|
+
await announceTerminals(rt);
|
|
140
|
+
if (hasCaller) {
|
|
141
|
+
notifySettled(rt, record, err);
|
|
142
|
+
throw err;
|
|
143
|
+
}
|
|
144
|
+
reportIntentFailed(rt, record, err);
|
|
145
|
+
}
|
|
146
|
+
/** The reason a terminal failure's details give its still-owed changes: a spent or refused publish is `attempts-exhausted`, anything else `concurrent-change`. */
|
|
147
|
+
function lostReasonOf(details) {
|
|
148
|
+
return details?.reason === 'max-attempts' || details?.reason === 'publish-refused' ? 'attempts-exhausted' : 'concurrent-change';
|
|
149
|
+
}
|
|
150
|
+
/** `own-change-lost` terminals for `lost` (`root`: the root intent id), in order. */
|
|
151
|
+
function lostTerminals(root, lost) {
|
|
152
|
+
return lost.map((l) => ({ type: 'own-change-lost', intentId: root, change: l.change, reason: l.reason }));
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Epic marmot-rewound-own-commit-no-signal S11 (AC-COMP-10): deletes `record` in `tx` and, when it is a commit re-drive, writes the
|
|
156
|
+
* terminals that end it to the outbox in the SAME transaction: the verdicts this run decided before it ended (`newlyLost`, each with
|
|
157
|
+
* its reason) and then `own-change-lost` (`reason`) for every change that has no verdict yet (a verdict is persisted with the
|
|
158
|
+
* terminal that signals it, so one that exists was already written). Whatever deletes a commit re-drive record goes through here
|
|
159
|
+
* (or `takeStrandedSelfUpdateReenqueues`, `cancelRestoredReenqueues`): a delete that did not would leave the application holding an
|
|
160
|
+
* `own-change-withdrawn` with no terminal. The caller announces after the transaction commits (`terminalFail`, `announceTerminals`).
|
|
161
|
+
*/
|
|
162
|
+
async function deleteEndingRecord(tx, record, reason, newlyLost = []) {
|
|
163
|
+
await deletePendingIntentRecord(tx.artifact('pendingIntents'), record.intentId);
|
|
164
|
+
if (!isCommitRedrive(record))
|
|
165
|
+
return;
|
|
166
|
+
const root = reportedIntentIdOf(record);
|
|
167
|
+
// DD9: the publishes this record spent stay the root's, whatever ended it. The record is gone, so its spend is folded into every marker of
|
|
168
|
+
// the root in the same transaction. `attempts` only, never `redriveSpendOf`: the one caller that ends a pending-confirm re-drive
|
|
169
|
+
// (`remintStaleRedrive`) passes it with the confirm's publish already counted and writes that marker itself.
|
|
170
|
+
await raiseRootSpend(tx.extension(OWN_COMMIT_EXTENSION), root, record.attempts);
|
|
171
|
+
await enqueueTerminals(tx.extension(OWN_CHANGE_OUTBOX_EXTENSION), [
|
|
172
|
+
...lostTerminals(root, newlyLost),
|
|
173
|
+
...lostTerminals(root, (record.changes ?? []).filter((c) => c.verdict === undefined).map((c) => ({ change: c.change, reason }))),
|
|
174
|
+
]);
|
|
175
|
+
}
|
|
176
|
+
/** The `send-rejected` a self-update dropped with the staged-Adds commit it followed (`after`) settles with. */
|
|
177
|
+
function dependencyTerminalError(rt, record, after, cause) {
|
|
178
|
+
const reportedIntentId = reportedIntentIdOf(record);
|
|
179
|
+
return new MarmotError('send-rejected', `send: intent ${reportedIntentId} on group ${rt.groupId} is terminal`, {
|
|
180
|
+
cause,
|
|
181
|
+
details: { groupId: rt.groupId, intentId: reportedIntentId, reason: 'dependency-terminal', after },
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
/** The ledger fields of a self-update record's rekey target (AC-REKEY-7): its entry names the target so a rewind re-enqueues a rekey. */
|
|
185
|
+
function rekeyOf(record) {
|
|
186
|
+
return record.intent.type === 'self-update' && record.intent.rekey !== undefined ? { rekey: record.intent.rekey.target } : {};
|
|
187
|
+
}
|
|
188
|
+
/** Tells the runtime's settled-intent listeners (group-runtime.ts `groups.restore`) that `record` confirmed or went terminal. Never throws. */
|
|
189
|
+
export function notifySettled(rt, record, error) {
|
|
190
|
+
for (const listener of [...rt.settledListeners]) {
|
|
191
|
+
try {
|
|
192
|
+
listener({ intentId: record.intentId, rootIntentId: reportedIntentIdOf(record), kind: record.kind, ...(error === undefined ? {} : { error }) });
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
// A listener never breaks the operation that settled the intent.
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* The one report of an own intent that went terminal with no caller to reject: its settled-intent listeners are
|
|
201
|
+
* told, then `intentFailed` fires under the ROOT intent id (`reportedIntentIdOf`). Every terminal path without a
|
|
202
|
+
* caller — {@link terminalFail}, a stranded re-enqueue, an awaiting-sync intent a fence dropped — goes through
|
|
203
|
+
* here, so nothing waiting on an intent's settlement is ever left waiting.
|
|
204
|
+
*/
|
|
205
|
+
function reportIntentFailed(rt, record, error) {
|
|
206
|
+
notifySettled(rt, record, error);
|
|
207
|
+
rt.ctx.emit('intentFailed', { groupId: rt.groupId, intentId: reportedIntentIdOf(record), error });
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* After a terminal own commit, the queue's own commit (the one queued rumors
|
|
211
|
+
* wait for) is held at this epoch until the capped backoff timer fires — then
|
|
212
|
+
* one new attempt is allowed — or a genuine change releases it: an ingested
|
|
213
|
+
* change of the staged proposals, an epoch change (an ingested or own
|
|
214
|
+
* commit), a reload, a successful queue commit. A new application send is no
|
|
215
|
+
* such evidence and does not release it (a chatty application would otherwise
|
|
216
|
+
* re-drive the refused commit once per message). An explicit
|
|
217
|
+
* `send({ type: 'commit' | 'self-update' })` is never held: it is the
|
|
218
|
+
* application's deliberate retry, and its confirmation moves the epoch.
|
|
219
|
+
* Without the hold, a commit every relay refuses would be rebuilt and refused
|
|
220
|
+
* in a tight loop.
|
|
221
|
+
*/
|
|
222
|
+
function holdQueueCommit(rt) {
|
|
223
|
+
const previous = rt.queueCommitHold;
|
|
224
|
+
if (previous?.timer !== undefined) {
|
|
225
|
+
rt.ctx.timers.clear(previous.timer);
|
|
226
|
+
rt.pipelineTimers.delete(previous.timer);
|
|
227
|
+
}
|
|
228
|
+
const epoch = rt.engine.state.groupContext.epoch;
|
|
229
|
+
// A refusal at a new epoch starts again at the base backoff.
|
|
230
|
+
const failures = previous !== undefined && previous.epoch === epoch ? previous.failures + 1 : 1;
|
|
231
|
+
const handle = rt.ctx.timers.set(() => {
|
|
232
|
+
rt.pipelineTimers.delete(handle);
|
|
233
|
+
if (rt.queueCommitHold?.timer === handle)
|
|
234
|
+
rt.queueCommitHold = { epoch, failures, timer: undefined };
|
|
235
|
+
rt.scheduleQueueDrain();
|
|
236
|
+
}, backoffMs(failures));
|
|
237
|
+
rt.pipelineTimers.add(handle);
|
|
238
|
+
rt.queueCommitHold = { epoch, failures, timer: handle };
|
|
239
|
+
}
|
|
240
|
+
/** A genuine trigger (new send, ingested proposal): the held queue commit may be attempted again now. */
|
|
241
|
+
export function releaseQueueCommitHold(rt) {
|
|
242
|
+
const hold = rt.queueCommitHold;
|
|
243
|
+
if (hold?.timer === undefined)
|
|
244
|
+
return;
|
|
245
|
+
rt.ctx.timers.clear(hold.timer);
|
|
246
|
+
rt.pipelineTimers.delete(hold.timer);
|
|
247
|
+
rt.queueCommitHold = { ...hold, timer: undefined };
|
|
248
|
+
}
|
|
249
|
+
// ---------------------------------------------------------------------------
|
|
250
|
+
// GroupIntent <-> engine SendIntent / storage encoding
|
|
251
|
+
// ---------------------------------------------------------------------------
|
|
252
|
+
function validateIntent(intent) {
|
|
253
|
+
if (typeof intent !== 'object' || intent === null)
|
|
254
|
+
throw new MarmotError('invalid-options', 'send: intent must be an object');
|
|
255
|
+
const type = intent.type;
|
|
256
|
+
if (type !== 'application' && type !== 'proposal' && type !== 'commit' && type !== 'self-update') {
|
|
257
|
+
throw new MarmotError('invalid-options', `send: unknown intent.type ${JSON.stringify(type)}`);
|
|
258
|
+
}
|
|
259
|
+
if (type === 'application' && (typeof intent.rumor !== 'object' || intent.rumor === null)) {
|
|
260
|
+
throw new MarmotError('invalid-options', 'send: an application intent requires a rumor object');
|
|
261
|
+
}
|
|
262
|
+
if (type === 'proposal' && intent.proposal === undefined) {
|
|
263
|
+
throw new MarmotError('invalid-options', 'send: a proposal intent requires a proposal');
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
function encodeGroupIntent(intent) {
|
|
267
|
+
switch (intent.type) {
|
|
268
|
+
case 'application':
|
|
269
|
+
return { type: 'application', rumor: intent.rumor };
|
|
270
|
+
case 'proposal':
|
|
271
|
+
return { type: 'proposal', proposal: intent.proposal };
|
|
272
|
+
case 'commit':
|
|
273
|
+
return { type: 'commit' };
|
|
274
|
+
case 'self-update':
|
|
275
|
+
return { type: 'self-update' };
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* A self-update is always the engine's `selfUpdate` (it rotates the local
|
|
280
|
+
* leaf). It never reaches here while Adds are staged: `driveIntent` routes it
|
|
281
|
+
* to {@link driveSelfUpdateOverStagedAdds} first — an Add-only commit standing
|
|
282
|
+
* in for it would confirm without rotating anything.
|
|
283
|
+
*/
|
|
284
|
+
function toEngineIntent(intent, selfPubkey, state, groupId) {
|
|
285
|
+
switch (intent.type) {
|
|
286
|
+
case 'application':
|
|
287
|
+
return { kind: 'applicationMessage', payload: serializeApplicationRumor(intent.rumor) };
|
|
288
|
+
case 'proposal':
|
|
289
|
+
return { kind: 'proposal', proposal: intent.proposal };
|
|
290
|
+
case 'commit': {
|
|
291
|
+
// An invite commit's Adds are rebuilt from the stored KeyPackage events (a record resumed before staging);
|
|
292
|
+
// a forget's Removes from the current tree (the leaf may have moved, or already left, since it was recorded).
|
|
293
|
+
const admins = intent.admins;
|
|
294
|
+
const extraProposals = [
|
|
295
|
+
...(intent.invite === undefined ? [] : addProposalsFor(intent.invite.invitees.map((i) => i.event))),
|
|
296
|
+
...(intent.removeLeafKeys === undefined ? [] : removeProposalsFor(state, intent.removeLeafKeys)),
|
|
297
|
+
...(intent.removeAccounts === undefined ? [] : removeProposalsForAccounts(state, intent.removeAccounts)),
|
|
298
|
+
...(admins === undefined ? [] : [adminPolicyProposal(admins)]),
|
|
299
|
+
...(intent.profile === undefined ? [] : [groupProfileAction(intent.profile, groupId)]),
|
|
300
|
+
// The internal `components` field (DD2): applied when driven, first time as much as a re-drive -- each component's update, or its deletion.
|
|
301
|
+
...(intent.components ?? []).map((c) => guardedComponentUpdate(genericComponentGuard(c.id, groupId), () => componentUpdateProposal(c.id, c.data))),
|
|
302
|
+
];
|
|
303
|
+
return extraProposals.length === 0 ? { kind: 'commit', actorPubkey: selfPubkey } : { kind: 'commit', actorPubkey: selfPubkey, extraProposals };
|
|
304
|
+
}
|
|
305
|
+
case 'self-update':
|
|
306
|
+
return { kind: 'selfUpdate' };
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* One Remove per leaf of `state` whose signature key (hex) is `leafKeys` or in it: a single key, or a key set (a device
|
|
311
|
+
* chain, epic marmot-device-key-rotation AC-RM-3/4). The leaf index is resolved here, from the tree at hand.
|
|
312
|
+
*/
|
|
313
|
+
function removeProposalsFor(state, leafKeys) {
|
|
314
|
+
const wanted = new Set(typeof leafKeys === 'string' ? [leafKeys] : leafKeys);
|
|
315
|
+
const out = [];
|
|
316
|
+
const leafCount = Math.ceil(state.ratchetTree.length / 2);
|
|
317
|
+
for (let leafIndex = 0; leafIndex < leafCount; leafIndex++) {
|
|
318
|
+
const node = state.ratchetTree[2 * leafIndex];
|
|
319
|
+
if (node?.nodeType !== nodeTypes.leaf || !wanted.has(toHex(node.leaf.signaturePublicKey)))
|
|
320
|
+
continue;
|
|
321
|
+
out.push({ proposalType: defaultProposalTypes.remove, remove: { removed: leafIndex } });
|
|
322
|
+
}
|
|
323
|
+
return out;
|
|
324
|
+
}
|
|
325
|
+
/** Whether `state`'s tree holds a leaf carrying signature key `leafKey` (hex), or any key of the set `leafKey`. */
|
|
326
|
+
export function holdsLeafKey(state, leafKey) {
|
|
327
|
+
return removeProposalsFor(state, leafKey).length > 0;
|
|
328
|
+
}
|
|
329
|
+
function encodeStoredPending(pending) {
|
|
330
|
+
const commitMessage = pending.commitMessage === undefined ? undefined : encode(mlsMessageEncoder, pending.commitMessage);
|
|
331
|
+
return {
|
|
332
|
+
kind: pending.kind,
|
|
333
|
+
newState: serializeClientState(pending.newState),
|
|
334
|
+
...(pending.parentState === undefined ? {} : { parentState: serializeClientState(pending.parentState) }),
|
|
335
|
+
...(commitMessage === undefined ? {} : { commitMessage }),
|
|
336
|
+
...(commitMessage === undefined || pending.ownCommitStamp === undefined ? {} : { ownCommit: encodeOwnCommitRecord({ wireBytes: commitMessage, stamp: pending.ownCommitStamp }) }),
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
/** The own-commit stamp a stored pending state carries, if any (an unreadable record carries none). */
|
|
340
|
+
function storedStamp(pending) {
|
|
341
|
+
if (pending.ownCommit === undefined)
|
|
342
|
+
return undefined;
|
|
343
|
+
try {
|
|
344
|
+
const record = decodeOwnCommitRecord(pending.ownCommit);
|
|
345
|
+
return record.kind === 'stamped' ? record.stamp : undefined;
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
return undefined;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
/** The local leaf's credential in `state`. */
|
|
352
|
+
function ownCredential(state) {
|
|
353
|
+
try {
|
|
354
|
+
return getCredentialFromLeafIndex(state.ratchetTree, state.privatePath.leafIndex);
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
return undefined;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* S10: an own application message, confirmed, as the `message` notification
|
|
362
|
+
* (and history entry) every other member derives from it — the ordering key
|
|
363
|
+
* read from its own envelope under the state it was sent in.
|
|
364
|
+
*/
|
|
365
|
+
export async function ownMessage(rt, record, envelope) {
|
|
366
|
+
if (record.intent.type !== 'application')
|
|
367
|
+
return undefined;
|
|
368
|
+
const state = rt.engine.state;
|
|
369
|
+
const credential = ownCredential(state);
|
|
370
|
+
if (credential === undefined)
|
|
371
|
+
return undefined;
|
|
372
|
+
try {
|
|
373
|
+
const { read } = await rt.engine.peeler.peelGroupMessages([envelope], state);
|
|
374
|
+
const message = read[0]?.message;
|
|
375
|
+
if (message === undefined)
|
|
376
|
+
return undefined;
|
|
377
|
+
const orderingKey = await orderingKeyOf({
|
|
378
|
+
cs: rt.engine.ciphersuite,
|
|
379
|
+
message,
|
|
380
|
+
senderCredential: credential,
|
|
381
|
+
senderLeafIndex: Number(state.privatePath.leafIndex),
|
|
382
|
+
states: [state, ...rt.engine.retainedStates()],
|
|
383
|
+
});
|
|
384
|
+
return { type: 'message', eventId: envelope.id, rumor: record.intent.rumor, orderingKey };
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
return undefined;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
// ---------------------------------------------------------------------------
|
|
391
|
+
// Confirmation
|
|
392
|
+
// ---------------------------------------------------------------------------
|
|
393
|
+
/**
|
|
394
|
+
* The live state with an own proposal staged on top of it.
|
|
395
|
+
*
|
|
396
|
+
* A proposal's stored `pending.newState` is a snapshot of the whole group
|
|
397
|
+
* state taken when the proposal was built. Its publish may fail and be
|
|
398
|
+
* retried much later (or resumed after a reload) while application sends
|
|
399
|
+
* and inbound messages keep advancing that same epoch — ratchets, received
|
|
400
|
+
* proposals. Confirming the snapshot would rewind all of that (a sending-
|
|
401
|
+
* ratchet rewind is message-key reuse). Handshake content is PublicMessage
|
|
402
|
+
* (the engine always frames proposals so), so building the proposal changed
|
|
403
|
+
* nothing but `unappliedProposals`; and within one epoch that set only
|
|
404
|
+
* grows. So the snapshot's entries are merged into the CURRENT state.
|
|
405
|
+
* `undefined` when the epoch moved on — the proposal no longer applies.
|
|
406
|
+
*/
|
|
407
|
+
function withOwnProposal(current, staged) {
|
|
408
|
+
if (current.groupContext.epoch !== staged.groupContext.epoch)
|
|
409
|
+
return undefined;
|
|
410
|
+
return { ...current, unappliedProposals: { ...current.unappliedProposals, ...staged.unappliedProposals } };
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* A published own proposal whose epoch the group has since left (another
|
|
414
|
+
* writer of this store advanced it before the blocked confirmation ran):
|
|
415
|
+
* it can never be confirmed. The record is dropped and its envelope ledgered
|
|
416
|
+
* (a redelivery is a duplicate), no `onConfirm` hook runs, and the intent
|
|
417
|
+
* fails `send-rejected` with reason `stale-parent`.
|
|
418
|
+
*/
|
|
419
|
+
async function dropStaleProposal(rt, record, hasCaller) {
|
|
420
|
+
const envelope = record.envelope;
|
|
421
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
422
|
+
await deletePendingIntentRecord(tx.artifact('pendingIntents'), record.intentId);
|
|
423
|
+
if (envelope !== undefined) {
|
|
424
|
+
await writeTerminalLedgerEntry(tx.artifact('terminalLedger'), envelope.id, { disposition: 'own', at: rt.ctx.clock.now(), createdAt: envelope.created_at });
|
|
425
|
+
}
|
|
426
|
+
}));
|
|
427
|
+
// The blocked confirmation is gone: events dropped as `group-busy` meanwhile are refetched.
|
|
428
|
+
if (rt.resyncNeeded)
|
|
429
|
+
rt.resync();
|
|
430
|
+
await terminalFail(rt, record, hasCaller, new Error('the group left the proposal\'s epoch before it was confirmed'), {
|
|
431
|
+
reason: 'stale-parent',
|
|
432
|
+
parentEpoch: String(record.parentEpoch),
|
|
433
|
+
epoch: String(rt.engine.state.groupContext.epoch),
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* The rewind window an own commit produces: the runtime's window (or, when its tip is not the commit's parent, the
|
|
438
|
+
* parent alone) extended with the commit. The one definition behind what `confirmRecord` persists and what the
|
|
439
|
+
* `onBeforePublish` pending snapshot describes, so the two cannot drift.
|
|
440
|
+
*/
|
|
441
|
+
function windowAfterOwnCommit(rt, link) {
|
|
442
|
+
const base = windowFor(rt.window, link.parentState);
|
|
443
|
+
return extendWindow(base, { parentState: link.parentState, commit: link.commit, newState: link.newState, ...(link.stamp === undefined ? {} : { stamp: link.stamp }) });
|
|
444
|
+
}
|
|
445
|
+
async function confirmRecord(rt, record, attempt, live, hasCaller = false) {
|
|
446
|
+
if (isCommitRedrive(record)) {
|
|
447
|
+
// A restoring transition retires a commit re-drive in every state (group-history.ts cancelRestoredReenqueues, AC-REC-7): a
|
|
448
|
+
// confirm that was waiting on a retry finds no record and must announce nothing (the change has its terminal).
|
|
449
|
+
const stored = await rt.guard.call(rt.groupId, 'store', () => readPendingIntentRecord(rt.groupStore.artifact('pendingIntents'), record.intentId));
|
|
450
|
+
if (stored === undefined) {
|
|
451
|
+
// Defensive: the engine may still hold this commit's staged lifecycle; back to the durable state.
|
|
452
|
+
await adoptRetainedWindow(rt);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
// Published, but the group moved on before the confirmation could be applied: adopting its state would step back to a
|
|
456
|
+
// parent that is no longer the tip. Stale like an unconfirmed envelope (AC-DRIVE-8): re-minted at the tip, never confirmed.
|
|
457
|
+
if (record.parentEpoch !== rt.engine.state.groupContext.epoch) {
|
|
458
|
+
// Defensive, no known producer (every producer of a transition was checked: ingest refuses every batch while a record is pending-confirm (ingest.ts); an own confirmation
|
|
459
|
+
// never restores a marked node of another root's re-drive (its own record is deleted first, and an application confirm leaves the window as it is);
|
|
460
|
+
// resolvePendingCommit (pending-leaf.ts) refuses to rewind under any record that is not awaiting-sync; rebaseToWelcome (group-runtime.ts) has no
|
|
461
|
+
// such gate, but its canonical path is the joined tip alone, so it restores a marked node only when a Welcome names exactly the state that
|
|
462
|
+
// commit produced -- not constructible for a removal re-drive (the joiner is not the author).
|
|
463
|
+
// Kept because the state is representable. Treated as stored: a relay accepted it.
|
|
464
|
+
await remintStaleRedrive(rt, stored, false, true);
|
|
465
|
+
// The engine may still hold this commit's staged lifecycle (a confirmation blocked right after its publish): back to the durable state.
|
|
466
|
+
await adoptRetainedWindow(rt);
|
|
467
|
+
return;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
let newState;
|
|
471
|
+
let notifications = [];
|
|
472
|
+
/** S10: the rewind window the confirmed commit is persisted with — its own link and stamp on top of the parent's. */
|
|
473
|
+
let window;
|
|
474
|
+
if (record.pending !== undefined && record.kind === 'proposal') {
|
|
475
|
+
newState = withOwnProposal(rt.engine.state, deserializeClientState(record.pending.newState));
|
|
476
|
+
if (newState === undefined) {
|
|
477
|
+
await dropStaleProposal(rt, record, hasCaller);
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
else if (record.pending !== undefined) {
|
|
482
|
+
// An own commit / self-update starts a new epoch whose secret tree derives
|
|
483
|
+
// from the commit alone, so same-epoch progress made while it was pending
|
|
484
|
+
// publish (application sends, received messages) cannot be rewound by
|
|
485
|
+
// adopting its snapshot; uncommitted proposals are discarded at the epoch
|
|
486
|
+
// change by MLS itself.
|
|
487
|
+
newState = deserializeClientState(record.pending.newState);
|
|
488
|
+
if (record.pending.parentState !== undefined && record.pending.commitMessage !== undefined) {
|
|
489
|
+
const parentState = deserializeClientState(record.pending.parentState);
|
|
490
|
+
const decodedCommit = decode(mlsMessageDecoder, record.pending.commitMessage);
|
|
491
|
+
if (decodedCommit !== undefined) {
|
|
492
|
+
const digestBytes = commitDigest(record.pending.commitMessage);
|
|
493
|
+
const raw = deriveStateNotifications({ parentState, resultingState: newState, commitDigest: digestBytes });
|
|
494
|
+
notifications = raw.map((n) => ({ type: 'applied', commitDigest: toHex(n.commitDigest), kind: n.kind }));
|
|
495
|
+
}
|
|
496
|
+
// The stamp (live, or stored with the record) makes the own commit a fork-recovery candidate after any rebuild.
|
|
497
|
+
const stamp = live?.ownCommitStamp ?? storedStamp(record.pending);
|
|
498
|
+
window = windowAfterOwnCommit(rt, { parentState, commit: record.pending.commitMessage, newState, stamp });
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
const envelope = record.envelope;
|
|
502
|
+
const resultingEpoch = newState !== undefined ? newState.groupContext.epoch : rt.engine.state.groupContext.epoch;
|
|
503
|
+
// S10: what the confirmation announces — an own commit's applied changes (its window link), or an own message
|
|
504
|
+
// with its ordering key; the message's ledger entry records the state it went out under, so a rewind that
|
|
505
|
+
// abandons that state withdraws it like any received one.
|
|
506
|
+
const sent = record.intent.type !== 'application'
|
|
507
|
+
? undefined
|
|
508
|
+
: record.ordering !== undefined
|
|
509
|
+
? { type: 'message', eventId: envelope.id, rumor: record.intent.rumor, orderingKey: record.ordering }
|
|
510
|
+
: await ownMessage(rt, record, envelope);
|
|
511
|
+
// A message whose send-time branch a rewind left before this confirmation (its publish was retried meanwhile):
|
|
512
|
+
// no member on the canonical branch can read it. Never announced as kept nor appended to history; ledgered
|
|
513
|
+
// withdrawn with its node, and announced `message-withdrawn` — its `send()` may already have resolved, so this
|
|
514
|
+
// is the application's signal to send it again.
|
|
515
|
+
const lost = sent !== undefined && !sentOnCanonicalBranch(rt.window, sent.orderingKey) ? sent : undefined;
|
|
516
|
+
const message = lost === undefined ? sent : undefined;
|
|
517
|
+
let plan;
|
|
518
|
+
let deferred = new Set();
|
|
519
|
+
let restoredReenqueues = [];
|
|
520
|
+
// A flag, not `hookError !== undefined`: a hook may reject with `undefined`.
|
|
521
|
+
let blocked = false;
|
|
522
|
+
let hookError;
|
|
523
|
+
try {
|
|
524
|
+
plan =
|
|
525
|
+
window !== undefined && newState !== undefined
|
|
526
|
+
? await planTransition(rt, rt.window, rt.exposedState, window, newState, [])
|
|
527
|
+
: message !== undefined
|
|
528
|
+
? await planTransition(rt, rt.window, rt.exposedState, rt.window, rt.exposedState, [message])
|
|
529
|
+
: lost !== undefined
|
|
530
|
+
? lostMessagePlan(rt, lost)
|
|
531
|
+
: undefined;
|
|
532
|
+
const announced = plan;
|
|
533
|
+
// Epic marmot-rewound-own-commit-no-signal (S2 AC-DRIVE-2, DD7; S11): a confirmed commit re-drive tells the application each change it
|
|
534
|
+
// carries is in force again, naming the new commit. A change the build already decided lost (its verdict) was signalled
|
|
535
|
+
// then. Not when this very confirmation's transition restored the root's marked node: the plan then announced the
|
|
536
|
+
// original (a change is told exactly one terminal). Written to the outbox in the confirming transaction below.
|
|
537
|
+
const reapplied = isCommitRedrive(record) && record.pending?.commitMessage !== undefined && plan?.restoredOwnCommits?.some((r) => r.rootIntentId === reportedIntentIdOf(record)) !== true
|
|
538
|
+
? (record.changes ?? [])
|
|
539
|
+
.filter((c) => c.verdict === undefined || c.verdict === 'reapplied')
|
|
540
|
+
.map((c) => ({ type: 'own-change-reapplied', intentId: reportedIntentIdOf(record), commitDigest: toHex(commitDigest(record.pending.commitMessage)), change: c.change }))
|
|
541
|
+
: [];
|
|
542
|
+
// announced by the `announceTerminals` below, after the plan's own notifications (deliverTransition must not send them ahead)
|
|
543
|
+
deferred = new Set(reapplied.map(terminalKey));
|
|
544
|
+
// S5: a relay accepted this invite commit, so its Welcomes may now go out — they
|
|
545
|
+
// are handed to the account's `invites` record BEFORE the confirmation deletes
|
|
546
|
+
// this record (a crash in between only repeats the upsert, which keeps any
|
|
547
|
+
// delivery progress). A failed handoff blocks and retries like a hook.
|
|
548
|
+
const welcomes = record.welcomes;
|
|
549
|
+
if (welcomes !== undefined && welcomes.length > 0) {
|
|
550
|
+
// Framed on the group scope too (InvitationsDeps already frames the account scope): this runs
|
|
551
|
+
// while the confirmation holds the group's queue, so a consumer account store that awaits a
|
|
552
|
+
// mutating call on this group from inside it is rejected `reentrant` rather than deadlocking.
|
|
553
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.invitations.recordCommitted({ inviteId: record.intentId, groupId: rt.groupId, commitEventId: envelope.id, ...(newState === undefined ? {} : { commitTag: tagOf(newState) }), welcomes }));
|
|
554
|
+
}
|
|
555
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
556
|
+
if (newState !== undefined)
|
|
557
|
+
await writeGroupState(tx.artifact('clientState'), newState);
|
|
558
|
+
if (window !== undefined)
|
|
559
|
+
await writeWindow(tx.artifact('rewindTree'), window);
|
|
560
|
+
await deletePendingIntentRecord(tx.artifact('pendingIntents'), record.intentId);
|
|
561
|
+
const ledgered = message ?? lost;
|
|
562
|
+
// S5 (AC-RACE-1): a confirmed self-update's own commit is tagged with the node it produced too — same
|
|
563
|
+
// mechanism as an application message's `acceptedDigest` — so a later rewind that abandons this branch
|
|
564
|
+
// withdraws it the same way, and `selfUpdateIntentId` lets the withdrawal be re-enqueued (not just
|
|
565
|
+
// announced). Never set for `proposal`/`commit` records: only a self-update's own rotation is re-driven.
|
|
566
|
+
const ownSelfUpdateTag = ledgered === undefined && record.kind === 'self-update' && newState !== undefined ? tagOf(newState) : undefined;
|
|
567
|
+
// AC-RACE-1: selfUpdateIntentId always names the ROOT intent (reportedIntentIdOf) so a chain of
|
|
568
|
+
// withdrawal -> re-enqueue -> withdrawal again still dedups and reports against the one id a caller
|
|
569
|
+
// ever saw; attempts carries this record's own budget forward into whatever re-enqueue follows it.
|
|
570
|
+
// S9 (AC-MARK-1, DD2): an own commit declaring a change records the node it produced and its root intent on
|
|
571
|
+
// the ledger entry, and leaves the marker a later rewind re-drives from — same transaction, same key.
|
|
572
|
+
// `ledgered` (an application message, or a message lost to a rewind) is only ever set for an `application`
|
|
573
|
+
// record, which `declaresOwnChange` already refuses, so no commit confirm reaches here with it set: the first
|
|
574
|
+
// operand is a redundant guard that keeps an already-ledgered message entry from ever gaining a commit link.
|
|
575
|
+
const ownCommitMarker = ledgered === undefined && newState !== undefined && declaresOwnChange(record) ? ownCommitMarkerOf(record, tagOf(newState)) : undefined;
|
|
576
|
+
await writeTerminalLedgerEntry(tx.artifact('terminalLedger'), envelope.id, {
|
|
577
|
+
disposition: lost === undefined ? 'own' : WITHDRAWN_DISPOSITION,
|
|
578
|
+
at: rt.ctx.clock.now(),
|
|
579
|
+
createdAt: envelope.created_at,
|
|
580
|
+
...(ledgered === undefined || ledgered.orderingKey.acceptedDigest === '' ? {} : { node: ledgered.orderingKey.acceptedDigest }),
|
|
581
|
+
...(ownSelfUpdateTag === undefined ? {} : { node: ownSelfUpdateTag, selfUpdateIntentId: reportedIntentIdOf(record), attempts: record.attempts, ...rekeyOf(record) }),
|
|
582
|
+
...(ownCommitMarker === undefined ? {} : { node: ownCommitMarker.node, ownCommitRoot: ownCommitMarker.root }),
|
|
583
|
+
});
|
|
584
|
+
if (ownCommitMarker !== undefined)
|
|
585
|
+
await writeOwnCommitMarker(tx.extension(OWN_COMMIT_EXTENSION), envelope.id, ownCommitMarker);
|
|
586
|
+
if (announced !== undefined)
|
|
587
|
+
restoredReenqueues = await recordTransition(rt, tx, announced, newState?.groupContext.epoch ?? rt.engine.state.groupContext.epoch);
|
|
588
|
+
await enqueueTerminals(tx.extension(OWN_CHANGE_OUTBOX_EXTENSION), reapplied);
|
|
589
|
+
// AC-MX-2: an own self-update confirmed — a restored group's adoption is complete (atomically with it).
|
|
590
|
+
if (record.kind === 'self-update')
|
|
591
|
+
await clearRestorePending(tx.extension(RESTORE_PENDING_EXTENSION));
|
|
592
|
+
// AC-MX-1: the snapshot (built only if a hook reads it) describes what this transaction persists — the
|
|
593
|
+
// confirmed commit's state and window, else the exposed state and window: at an op boundary they ARE the
|
|
594
|
+
// stored ones (an application message's advanced ratchet was persisted when it was staged; a proposal's
|
|
595
|
+
// state keeps the tip's confirmation tag, so the window still names it).
|
|
596
|
+
await runConfirmHooks({ groupId: rt.groupId, guard: rt.guard, hooks: rt.confirmHooks }, {
|
|
597
|
+
intentId: record.intentId,
|
|
598
|
+
kind: record.kind,
|
|
599
|
+
epoch: resultingEpoch,
|
|
600
|
+
tx,
|
|
601
|
+
persisted: () => ({ state: newState ?? rt.exposedState, window: window ?? rt.window }),
|
|
602
|
+
notifications,
|
|
603
|
+
createdAt: rt.joinedAtMs,
|
|
604
|
+
// AC-MX-6: the commit `devices.forget` recorded (removeLeafIntent) is the only one carrying removeLeafKeys.
|
|
605
|
+
...(record.intent.type === 'commit' && record.intent.removeLeafKeys !== undefined ? { forgetting: { leafKeys: record.intent.removeLeafKeys } } : {}),
|
|
606
|
+
});
|
|
607
|
+
}));
|
|
608
|
+
}
|
|
609
|
+
catch (error) {
|
|
610
|
+
blocked = true;
|
|
611
|
+
hookError = error;
|
|
612
|
+
}
|
|
613
|
+
if (blocked) {
|
|
614
|
+
rt.ctx.emit('confirmBlocked', { groupId: rt.groupId, intentId: record.intentId, error: hookError });
|
|
615
|
+
scheduleConfirmRetry(rt, record.intentId, attempt);
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
618
|
+
// From here the confirmation is durable (committed above): the record is gone, and a self-update's transaction
|
|
619
|
+
// cleared a restore's unfinished mark, so nothing will ever confirm this intent again. Whatever fails while the
|
|
620
|
+
// engine takes it in (`confirmPublished`, or the `rebuildEngine` of a confirmation with no live engine attempt:
|
|
621
|
+
// a retried or resumed one — a resumed restore's self-update) can therefore never leave it unsettled: the intent
|
|
622
|
+
// is settled as confirmed — what the stores hold — and the rest of the confirmation runs; the engine, still at
|
|
623
|
+
// the parent state, is reloaded from the durable state before the next op runs on it (never let it judge an
|
|
624
|
+
// epoch: a dependent self-update would read as terminal). Without a caller the failure is rethrown last, so the
|
|
625
|
+
// background retry that drove this confirmation (resume, confirm or publish retry) reloads the group once more;
|
|
626
|
+
// a caller's send resolves — its intent IS confirmed, and failing it would have `groups.restore` take back a
|
|
627
|
+
// durably confirmed adoption — and its next op reloads. Without this, a `groups.restore` (or a resumed restore)
|
|
628
|
+
// waiting on the settlement waited forever (Codex P2).
|
|
629
|
+
let applied = { failed: false };
|
|
630
|
+
try {
|
|
631
|
+
// An application message was already applied synchronously inside engine.send().
|
|
632
|
+
if (record.kind === 'proposal') {
|
|
633
|
+
if (newState !== undefined)
|
|
634
|
+
rt.engine.confirmPublished({ kind: 'proposal', newState });
|
|
635
|
+
rt.exposedState = rt.engine.state;
|
|
636
|
+
}
|
|
637
|
+
else if (record.kind !== 'application') {
|
|
638
|
+
if (live !== undefined)
|
|
639
|
+
rt.engine.confirmPublished(live);
|
|
640
|
+
else if (newState !== undefined)
|
|
641
|
+
await rt.rebuildEngine(newState);
|
|
642
|
+
rt.exposedState = rt.engine.state;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
catch (error) {
|
|
646
|
+
rt.needsReload = true;
|
|
647
|
+
applied = { failed: true, error };
|
|
648
|
+
}
|
|
649
|
+
notifySettled(rt, record, undefined);
|
|
650
|
+
// S10: announced once the confirmation is applied.
|
|
651
|
+
if (plan !== undefined)
|
|
652
|
+
await deliverTransition(rt, plan, deferred);
|
|
653
|
+
// The re-drive's `own-change-reapplied` (outbox entries written in the confirming transaction), after the transition's own notifications.
|
|
654
|
+
await announceTerminals(rt);
|
|
655
|
+
// AC-RACE-1 / epic marmot-rewound-own-commit-no-signal S2: this confirmation's own transition withdrew an own commit (rare —
|
|
656
|
+
// usually ingest.ts's batch processing is where a rewind is discovered); re-drive it the same way.
|
|
657
|
+
scheduleWithdrawnRedrives(rt, plan);
|
|
658
|
+
// ...or cancelled a re-enqueue whose original rotation it restored: a sibling that waited behind it goes next.
|
|
659
|
+
for (const intentId of restoredReenqueues)
|
|
660
|
+
scheduleDeferredReenqueueRetry(rt, intentId);
|
|
661
|
+
// AC-RACE-1: this confirmation may be the started own commit (or the earlier sibling) a re-enqueued
|
|
662
|
+
// self-update was waiting behind (driveReenqueuedSelfUpdate) — re-evaluated directly, detached, since
|
|
663
|
+
// `scheduleQueueDrain` below only resumes such a record while `queuedHint` allows.
|
|
664
|
+
if (isCommitKind(record.kind))
|
|
665
|
+
scheduleDeferredReenqueueRetry(rt, record.intentId);
|
|
666
|
+
// An own commit may have moved the group's relays or Nostr group id: listen where it now publishes.
|
|
667
|
+
rt.checkSyncRouting();
|
|
668
|
+
// The block is cleared: sync-delivered events dropped as `group-busy` meanwhile are refetched from the cursor.
|
|
669
|
+
if (rt.resyncNeeded)
|
|
670
|
+
rt.resync();
|
|
671
|
+
// An invite confirmed without its `invite()` caller (a retry, a resume) delivers its Welcomes now.
|
|
672
|
+
// Only invite() delivers its own Welcomes inline; every other commit adding members (the queue's,
|
|
673
|
+
// an explicit commit/self-update send, a resumed one) delivers them in the background.
|
|
674
|
+
const inviteCaller = hasCaller && record.intent.type === 'commit' && record.intent.invite !== undefined;
|
|
675
|
+
if (record.welcomes !== undefined && !inviteCaller)
|
|
676
|
+
rt.invitations.deliverInBackground(rt.groupId, record.intentId);
|
|
677
|
+
// The confirmed commit (or proposal) may be what queued rumors wait for — and a confirmed
|
|
678
|
+
// queued rumor releases the one behind it (the drain stops at an unconfirmed head).
|
|
679
|
+
rt.scheduleQueueDrain();
|
|
680
|
+
if (applied.failed && !hasCaller)
|
|
681
|
+
throw applied.error;
|
|
682
|
+
}
|
|
683
|
+
/**
|
|
684
|
+
* Whether a background retry that failed with `error` must stop rather than
|
|
685
|
+
* reschedule: the client/runtime is gone, or the stores lost their lease (the
|
|
686
|
+
* next holder resumes the durable record). Anything else — a store or
|
|
687
|
+
* transport hiccup — is retried on the backoff.
|
|
688
|
+
*
|
|
689
|
+
* A lost lease is permanent for this client (every later write of its stores
|
|
690
|
+
* rejects `no-lease`), so no own intent of the group can settle here any more:
|
|
691
|
+
* the runtime's terminal signal fires (`reason: 'no-lease'`) so a wait on the
|
|
692
|
+
* group making progress (`groups.restore`'s self-update) ends instead of
|
|
693
|
+
* waiting for a settlement that never comes. `rt.leaseLost` remembers it for
|
|
694
|
+
* the checks made after the signal fired (a before-publish pass of a later
|
|
695
|
+
* resume must publish nothing).
|
|
696
|
+
*/
|
|
697
|
+
export function backgroundRetryEnds(rt, error) {
|
|
698
|
+
if (rt.disposed || rt.ctx.isDisposed() || isMarmotError(error, 'disposed'))
|
|
699
|
+
return true;
|
|
700
|
+
if (!isMarmotError(error, 'no-lease'))
|
|
701
|
+
return false;
|
|
702
|
+
rt.leaseLost ??= error;
|
|
703
|
+
const watchers = [...rt.terminalWatchers];
|
|
704
|
+
rt.terminalWatchers.clear();
|
|
705
|
+
for (const watcher of watchers) {
|
|
706
|
+
try {
|
|
707
|
+
watcher({ reason: 'no-lease', cause: error });
|
|
708
|
+
}
|
|
709
|
+
catch {
|
|
710
|
+
// A watcher never breaks the retry that observed the lost lease.
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return true;
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Schedules a pipeline retry whose handle the runtime tracks, so dispose() can cancel it. `ctx.timers.set`'s
|
|
717
|
+
* callback inherits whatever ALS context was live when `.set()` was called (Node's AsyncLocalStorage
|
|
718
|
+
* propagates through timers) — `guard.detached` clears it, since this is deferred background work, never
|
|
719
|
+
* inside the caller's own frame (S3 AC-RE-1).
|
|
720
|
+
*/
|
|
721
|
+
function setPipelineTimer(rt, callback, ms) {
|
|
722
|
+
const handle = rt.ctx.timers.set(() => {
|
|
723
|
+
rt.pipelineTimers.delete(handle);
|
|
724
|
+
rt.guard.detached(callback);
|
|
725
|
+
}, ms);
|
|
726
|
+
rt.pipelineTimers.add(handle);
|
|
727
|
+
}
|
|
728
|
+
function scheduleConfirmRetry(rt, intentId, failureCount) {
|
|
729
|
+
const delay = failureCount === 0 ? 0 : backoffMs(failureCount);
|
|
730
|
+
setPipelineTimer(rt, () => {
|
|
731
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
732
|
+
return;
|
|
733
|
+
rt.run('confirmRetry', async (rtc) => {
|
|
734
|
+
const view = rtc.groupStore.artifact('pendingIntents');
|
|
735
|
+
const current = await rtc.guard.call(rtc.groupId, 'store', () => readPendingIntentRecord(view, intentId));
|
|
736
|
+
if (current === undefined || current.state !== 'pending-confirm')
|
|
737
|
+
return;
|
|
738
|
+
await confirmRecord(rtc, current, failureCount + 1, undefined);
|
|
739
|
+
if (current.kind === 'commit')
|
|
740
|
+
await followDependents(rtc, intentId);
|
|
741
|
+
}).catch((error) => {
|
|
742
|
+
if (!backgroundRetryEnds(rt, error))
|
|
743
|
+
scheduleConfirmRetry(rt, intentId, failureCount + 1);
|
|
744
|
+
});
|
|
745
|
+
}, delay);
|
|
746
|
+
}
|
|
747
|
+
/** Drops a record whose envelope can never be accepted: `publishFailed` the live pending state, delete, surface `send-rejected`. */
|
|
748
|
+
async function onPublishTerminal(rt, record, opts, cause, details) {
|
|
749
|
+
if (opts.live !== undefined && record.kind !== 'application')
|
|
750
|
+
rt.engine.publishFailed(opts.live);
|
|
751
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deleteEndingRecord(tx, typeof details.attempts === 'number' ? { ...record, attempts: Math.max(record.attempts, details.attempts) } : record, lostReasonOf(details))));
|
|
752
|
+
// A dropped queued rumor (or commit) no longer holds the queue behind it.
|
|
753
|
+
rt.scheduleQueueDrain();
|
|
754
|
+
await terminalFail(rt, record, opts.hasCaller, cause, details);
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* A publish attempt no relay acknowledged. Returns `'stored'` when the retry ceiling is reached but a relay probe
|
|
758
|
+
* finds the own commit's envelope stored anyway (every ACK was lost): the caller then confirms it like a late ACK.
|
|
759
|
+
*
|
|
760
|
+
* Why the probe (`own-commit-ack-lost-max-attempts-terminal`, ADR-016 AR-10): no foreign commit moved the group
|
|
761
|
+
* (the stale branch runs before any publish), so a stored own commit IS the next canonical epoch for every peer that
|
|
762
|
+
* ingests it. Dropping it would let this device commit again at the same epoch — a real same-parent double commit by
|
|
763
|
+
* one leaf, which peers flag as equivocation and which can partition this client from peers extending the first.
|
|
764
|
+
* Only commit-kind records are probed; an absent envelope, or a probe that times out or fails, keeps today's drop.
|
|
765
|
+
*/
|
|
766
|
+
async function onPublishFailed(rt, record, opts, error) {
|
|
767
|
+
const attempts = record.attempts + 1;
|
|
768
|
+
if (attempts >= rt.ctx.maxPublishAttempts) {
|
|
769
|
+
if (isCommitKind(record.kind) && record.envelope !== undefined && (await probeStored(rt, record, record.envelope)) === 'stored')
|
|
770
|
+
return 'stored';
|
|
771
|
+
// The retry ceiling (options.maxPublishAttempts) turns a retryable intent terminal.
|
|
772
|
+
await onPublishTerminal(rt, record, opts, error, { reason: 'max-attempts', attempts });
|
|
773
|
+
return 'handled';
|
|
774
|
+
}
|
|
775
|
+
if (opts.live !== undefined && record.kind !== 'application')
|
|
776
|
+
rt.engine.publishFailed(opts.live);
|
|
777
|
+
const bumped = { ...record, attempts };
|
|
778
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), bumped)));
|
|
779
|
+
schedulePublishRetry(rt, bumped.intentId, bumped.attempts);
|
|
780
|
+
if (opts.hasCaller) {
|
|
781
|
+
throw new MarmotError('publish-failed', `send: publish of intent ${record.intentId} was not accepted by any relay`, {
|
|
782
|
+
cause: error,
|
|
783
|
+
details: { groupId: rt.groupId, intentId: record.intentId, retryable: true },
|
|
784
|
+
});
|
|
785
|
+
}
|
|
786
|
+
return 'handled';
|
|
787
|
+
}
|
|
788
|
+
/** `backoff` only lengthens the delay; the record's own `attempts` still drives the retry ceiling. */
|
|
789
|
+
function schedulePublishRetry(rt, intentId, backoff, staleToken) {
|
|
790
|
+
setPipelineTimer(rt, () => {
|
|
791
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
792
|
+
return;
|
|
793
|
+
// The armed stale-probe timer is consumed when it fires (before its probe); a stray one (token mismatch, or none)
|
|
794
|
+
// is an ordinary retry and never spends probe budget.
|
|
795
|
+
const armed = staleProbeArmed(rt);
|
|
796
|
+
const fromRetryTimer = staleToken !== undefined && armed.get(intentId) === staleToken;
|
|
797
|
+
if (fromRetryTimer)
|
|
798
|
+
armed.delete(intentId);
|
|
799
|
+
rt.run('publishRetry', async (rtc) => {
|
|
800
|
+
const view = rtc.groupStore.artifact('pendingIntents');
|
|
801
|
+
const current = await rtc.guard.call(rtc.groupId, 'store', () => readPendingIntentRecord(view, intentId));
|
|
802
|
+
if (current === undefined)
|
|
803
|
+
return;
|
|
804
|
+
// A failed earlier attempt of this retry may have got as far as pending-confirm.
|
|
805
|
+
if (current.state === 'pending-confirm')
|
|
806
|
+
await confirmRecord(rtc, current, 1, undefined);
|
|
807
|
+
else if (current.envelope !== undefined)
|
|
808
|
+
await settlePublishedRecord(rtc, current, { probeFirst: false, live: undefined, hasCaller: false, ...(fromRetryTimer ? { fromRetryTimer } : {}) });
|
|
809
|
+
else
|
|
810
|
+
return;
|
|
811
|
+
if (current.kind === 'commit')
|
|
812
|
+
await followDependents(rtc, intentId);
|
|
813
|
+
}).catch((error) => {
|
|
814
|
+
if (!backgroundRetryEnds(rt, error))
|
|
815
|
+
schedulePublishRetry(rt, intentId, backoff + 1);
|
|
816
|
+
});
|
|
817
|
+
}, backoffMs(backoff));
|
|
818
|
+
}
|
|
819
|
+
/**
|
|
820
|
+
* The rewind window with the stale own commit of `record` added as a fork node (never the tip), or `undefined` when
|
|
821
|
+
* it cannot be retained (nothing pending to build it from, or its parent left the window — the horizon).
|
|
822
|
+
*
|
|
823
|
+
* Why: a commit a foreign same-epoch commit made stale may still have been STORED by a relay (its publish ACK was
|
|
824
|
+
* lost), and peers that took it extend it. Fork choice can only adopt a branch the engine's history tree holds, and
|
|
825
|
+
* a committer cannot replay its own commit — so the node (with its stamp) is the only way this client can follow.
|
|
826
|
+
* Callers retain it ONLY once a relay probe found the envelope stored ({@link probeStored}): fork choice breaks an
|
|
827
|
+
* equal-depth tie by the tip committer's identity on every ingest, so an unstored (never extended) fork could win
|
|
828
|
+
* and move this client onto a branch no peer has. Once peers ingest the stored commit, the deterministic tie-break
|
|
829
|
+
* (or a deeper peer branch) may select it: that is ordinary same-epoch race semantics, not a new rule.
|
|
830
|
+
*/
|
|
831
|
+
function staleOwnCommitWindow(rt, record) {
|
|
832
|
+
const pending = record.pending;
|
|
833
|
+
if (pending?.parentState === undefined || pending.commitMessage === undefined || record.envelope === undefined)
|
|
834
|
+
return undefined;
|
|
835
|
+
const stamp = storedStamp(pending);
|
|
836
|
+
return withForkNode(rt.window, {
|
|
837
|
+
parentState: deserializeClientState(pending.parentState),
|
|
838
|
+
commit: pending.commitMessage,
|
|
839
|
+
newState: deserializeClientState(pending.newState),
|
|
840
|
+
...(stamp === undefined ? {} : { stamp }),
|
|
841
|
+
});
|
|
842
|
+
}
|
|
843
|
+
/**
|
|
844
|
+
* The relays an own commit's stale probe asks: the union of the plane of the state the commit was BUILT from (where
|
|
845
|
+
* it was published) and the current plane — a foreign same-epoch commit may have changed the group's relays, and
|
|
846
|
+
* `rt.engine.state` is already its state. Falls back to the current plane alone when the parent state is missing or
|
|
847
|
+
* unreadable.
|
|
848
|
+
*/
|
|
849
|
+
function stalePlaneFor(rt, record) {
|
|
850
|
+
const current = relayPlaneFor(rt);
|
|
851
|
+
if (record.pending?.parentState === undefined)
|
|
852
|
+
return current;
|
|
853
|
+
try {
|
|
854
|
+
const view = getMarmotGroupView(deserializeClientState(record.pending.parentState));
|
|
855
|
+
if (view === undefined || view === null)
|
|
856
|
+
return current;
|
|
857
|
+
return [...new Set([...groupRelayPlane(view.relays, rt.ctx.relays.defaults), ...current])];
|
|
858
|
+
}
|
|
859
|
+
catch {
|
|
860
|
+
return current;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
/**
|
|
864
|
+
* Whether a relay holds `envelope`: `timeout` when the probe could not tell (no relay answered in time). A probe
|
|
865
|
+
* that throws counts as unanswered too — otherwise a throwing probe would re-arm untokened retries forever, spending
|
|
866
|
+
* no budget — so it is armed/spent exactly like a timeout and the ceiling still ends it; a disposal propagates.
|
|
867
|
+
*/
|
|
868
|
+
async function probeStored(rt, record, envelope) {
|
|
869
|
+
const relays = stalePlaneFor(rt, record);
|
|
870
|
+
let result;
|
|
871
|
+
try {
|
|
872
|
+
result = await rt.guard.callSync(rt.groupId, 'transport', () => rt.ctx.transport.requestWithDeadline(relays, { ids: [envelope.id] }, { timeoutMs: rt.ctx.historyTimeoutMs }));
|
|
873
|
+
}
|
|
874
|
+
catch (error) {
|
|
875
|
+
if (rt.disposed || rt.ctx.isDisposed() || isMarmotError(error, 'disposed'))
|
|
876
|
+
throw error;
|
|
877
|
+
return 'timeout';
|
|
878
|
+
}
|
|
879
|
+
assertNotDisposed(rt);
|
|
880
|
+
// A hit wins over a partial timeout: the contract allows `timedOut` together with the events of the relays that did
|
|
881
|
+
// answer, and one silent relay must not hide an envelope another relay holds.
|
|
882
|
+
if (result.events.some((e) => e.id === envelope.id))
|
|
883
|
+
return 'stored';
|
|
884
|
+
return result.timedOut ? 'timeout' : 'absent';
|
|
885
|
+
}
|
|
886
|
+
const staleProbeArmedByRuntime = new WeakMap();
|
|
887
|
+
/** Per runtime: intent id -> token of the one armed stale-probe retry timer. */
|
|
888
|
+
function staleProbeArmed(rt) {
|
|
889
|
+
let armed = staleProbeArmedByRuntime.get(rt);
|
|
890
|
+
if (armed === undefined)
|
|
891
|
+
staleProbeArmedByRuntime.set(rt, (armed = new Map()));
|
|
892
|
+
return armed;
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Arms the stale-probe retry of `intentId` unless one is already armed (a resume/ingest/send burst during a probe
|
|
896
|
+
* outage must not stack timer chains). Scoped to this path: `schedulePublishRetry`'s other callers are unchanged and
|
|
897
|
+
* their timers, when they fire on a stale record, are ordinary passes that spend no probe budget.
|
|
898
|
+
*/
|
|
899
|
+
function armStaleProbeRetry(rt, intentId, backoff) {
|
|
900
|
+
const armed = staleProbeArmed(rt);
|
|
901
|
+
if (armed.has(intentId))
|
|
902
|
+
return;
|
|
903
|
+
const token = Symbol(intentId);
|
|
904
|
+
armed.set(intentId, token);
|
|
905
|
+
schedulePublishRetry(rt, intentId, backoff, token);
|
|
906
|
+
}
|
|
907
|
+
/**
|
|
908
|
+
* The stale-branch probe of an own commit (settle and remint share it). A probe timeout is BOUNDED, but only timer
|
|
909
|
+
* driven passes spend budget: when the armed stale-probe retry timer fires and the probe times out again it spends one
|
|
910
|
+
* persisted publish attempt and arms the next retry; once `maxPublishAttempts` is reached the answer is `unverified`
|
|
911
|
+
* and the caller takes the absent-envelope path (drop, `send-rejected`). Any other pass (resume, preflight, ingest,
|
|
912
|
+
* send) leaves the attempts alone and only arms a retry if none is armed. "Unverified after N probe timeouts" is NOT
|
|
913
|
+
* "a relay rejected it". Residual: if a relay did store the commit, this restores the pre-fix partition risk — the
|
|
914
|
+
* price of not wedging the group behind a relay that accepts publishes but never answers queries (a live record
|
|
915
|
+
* blocks every later own commit). Every settling answer clears the armed marker.
|
|
916
|
+
*/
|
|
917
|
+
async function staleProbe(rt, record, envelope, fromRetryTimer) {
|
|
918
|
+
const found = await probeStored(rt, record, envelope);
|
|
919
|
+
if (found !== 'timeout') {
|
|
920
|
+
staleProbeArmed(rt).delete(record.intentId);
|
|
921
|
+
return found;
|
|
922
|
+
}
|
|
923
|
+
if (!fromRetryTimer) {
|
|
924
|
+
armStaleProbeRetry(rt, record.intentId, record.attempts);
|
|
925
|
+
return 'retry';
|
|
926
|
+
}
|
|
927
|
+
const attempts = record.attempts + 1;
|
|
928
|
+
if (attempts >= rt.ctx.maxPublishAttempts) {
|
|
929
|
+
staleProbeArmed(rt).delete(record.intentId);
|
|
930
|
+
return 'unverified';
|
|
931
|
+
}
|
|
932
|
+
const bumped = { ...record, attempts };
|
|
933
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), bumped)));
|
|
934
|
+
armStaleProbeRetry(rt, record.intentId, attempts);
|
|
935
|
+
return 'retry';
|
|
936
|
+
}
|
|
937
|
+
/**
|
|
938
|
+
* Rebuilds the engine from the durable state and window, so a node just written to the window is in its history
|
|
939
|
+
* tree. Called after the transaction committed, so a failure must not skip the settlement that follows: the next op
|
|
940
|
+
* reloads first instead (as `restoreDurableEngine`).
|
|
941
|
+
*/
|
|
942
|
+
async function adoptRetainedWindow(rt) {
|
|
943
|
+
try {
|
|
944
|
+
await rt.reloadFromDurable();
|
|
945
|
+
}
|
|
946
|
+
catch {
|
|
947
|
+
rt.needsReload = true;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
/** Per runtime: the envelope ids whose `onBeforePublish` hooks already ran, so a publish retry never runs them again (AC-BAR-4). */
|
|
951
|
+
const beforePublishHandledByRuntime = new WeakMap();
|
|
952
|
+
/** Bounds {@link beforePublishHandledByRuntime}'s sets: only a recent envelope can be retried. */
|
|
953
|
+
const BEFORE_PUBLISH_HANDLED_CAP = 64;
|
|
954
|
+
/**
|
|
955
|
+
* `extensions.onBeforePublish` (AC-HOOK-*, AC-BAR-*): for an own `commit` / `self-update` whose staged envelope is
|
|
956
|
+
* about to be published for the first time in this runtime, runs the registered hooks (each in its own transaction,
|
|
957
|
+
* after the staging transaction committed) and awaits the barriers they returned under one deadline. Best-effort:
|
|
958
|
+
* nothing it does can hold the publish back beyond the deadline (see before-publish-hooks.ts).
|
|
959
|
+
*
|
|
960
|
+
* A no-op without a registered hook (the default behaviour is unchanged), for any other intent kind, and for a
|
|
961
|
+
* record staged without its pending state (nothing to describe). `resumed` is true when an earlier pass staged the
|
|
962
|
+
* envelope (the resume path, or a retry timer — including the retry that follows a resume probe timeout, which runs
|
|
963
|
+
* the hook as if unprobed: the envelope is not known stored).
|
|
964
|
+
*/
|
|
965
|
+
async function runBeforePublishHooks(rt, record, opts) {
|
|
966
|
+
if (record.kind !== 'commit' && record.kind !== 'self-update')
|
|
967
|
+
return true;
|
|
968
|
+
const registry = rt.ctx.beforePublish;
|
|
969
|
+
if (registry.hooks.length === 0)
|
|
970
|
+
return true;
|
|
971
|
+
const pending = record.pending;
|
|
972
|
+
if (pending?.parentState === undefined || pending.commitMessage === undefined || record.envelope === undefined)
|
|
973
|
+
return true;
|
|
974
|
+
const envelope = record.envelope;
|
|
975
|
+
let handled = beforePublishHandledByRuntime.get(rt);
|
|
976
|
+
if (handled === undefined)
|
|
977
|
+
beforePublishHandledByRuntime.set(rt, (handled = new Set()));
|
|
978
|
+
if (handled.has(envelope.id))
|
|
979
|
+
return true;
|
|
980
|
+
handled.add(envelope.id);
|
|
981
|
+
if (handled.size > BEFORE_PUBLISH_HANDLED_CAP)
|
|
982
|
+
handled.delete(handled.values().next().value);
|
|
983
|
+
const parentState = deserializeClientState(pending.parentState);
|
|
984
|
+
const newState = deserializeClientState(pending.newState);
|
|
985
|
+
const commitMessage = pending.commitMessage;
|
|
986
|
+
const stamp = opts.live?.ownCommitStamp ?? storedStamp(pending);
|
|
987
|
+
// Runtime end (dispose, fence, lost lease) as this run sees it: the terminal signal, or a state read directly (a
|
|
988
|
+
// watcher registered after the runtime ended would never be told).
|
|
989
|
+
let endedBy;
|
|
990
|
+
const subscribers = new Set();
|
|
991
|
+
const watcher = (end) => {
|
|
992
|
+
endedBy = end;
|
|
993
|
+
for (const notify of [...subscribers])
|
|
994
|
+
notify();
|
|
995
|
+
};
|
|
996
|
+
rt.terminalWatchers.add(watcher);
|
|
997
|
+
const isEnded = () => endedBy !== undefined || rt.disposed || rt.ctx.isDisposed() || rt.fence !== undefined || rt.leaseLost !== undefined || rt.abandoned;
|
|
998
|
+
let outcome;
|
|
999
|
+
try {
|
|
1000
|
+
outcome = await runBeforePublish({
|
|
1001
|
+
groupId: rt.groupId,
|
|
1002
|
+
guard: rt.guard,
|
|
1003
|
+
hooks: registry.hooks,
|
|
1004
|
+
timeoutMs: registry.timeoutMs,
|
|
1005
|
+
timers: rt.ctx.timers,
|
|
1006
|
+
transaction: (fn) => rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(fn)),
|
|
1007
|
+
skipped: (event) => rt.ctx.emit('beforePublishSkipped', event),
|
|
1008
|
+
ended: isEnded,
|
|
1009
|
+
onRuntimeEnd: (end) => {
|
|
1010
|
+
subscribers.add(end);
|
|
1011
|
+
if (isEnded())
|
|
1012
|
+
end();
|
|
1013
|
+
return () => void subscribers.delete(end);
|
|
1014
|
+
},
|
|
1015
|
+
}, {
|
|
1016
|
+
intentId: record.intentId,
|
|
1017
|
+
kind: record.kind,
|
|
1018
|
+
envelopeId: envelope.id,
|
|
1019
|
+
commitMessage,
|
|
1020
|
+
parentState,
|
|
1021
|
+
newState,
|
|
1022
|
+
// The window `confirmRecord` would persist for this commit, so the snapshot's tip edge is the commit itself.
|
|
1023
|
+
window: () => windowAfterOwnCommit(rt, { parentState, commit: commitMessage, newState, stamp }),
|
|
1024
|
+
createdAt: rt.joinedAtMs,
|
|
1025
|
+
// AC-MX-6 counterpart: the commit `devices.forget` recorded (removeLeafIntent) is the only one carrying removeLeafKeys.
|
|
1026
|
+
...(record.intent.type === 'commit' && record.intent.removeLeafKeys !== undefined ? { forgetting: { leafKeys: record.intent.removeLeafKeys } } : {}),
|
|
1027
|
+
resumed: opts.live === undefined,
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
finally {
|
|
1031
|
+
rt.terminalWatchers.delete(watcher);
|
|
1032
|
+
}
|
|
1033
|
+
assertNotDisposed(rt);
|
|
1034
|
+
if (outcome === 'done' && !isEnded())
|
|
1035
|
+
return true;
|
|
1036
|
+
// The hooks did not complete: a later pass of this runtime must not take the envelope as handled and publish it
|
|
1037
|
+
// without them (it runs them again, and finds the runtime ended).
|
|
1038
|
+
handled.delete(envelope.id);
|
|
1039
|
+
// The runtime ended (fenced, abandoned, or its stores lost their lease) while the hooks or barriers ran: publish
|
|
1040
|
+
// nothing from it. The durable intent stays as it is for the next holder of the stores to resume — the same as
|
|
1041
|
+
// every other pipeline path on a fenced runtime — and a waiting `send()` caller is told why.
|
|
1042
|
+
if (opts.hasCaller) {
|
|
1043
|
+
if (rt.fence !== undefined)
|
|
1044
|
+
throw fencedError(rt.groupId, 'send', rt.fence);
|
|
1045
|
+
throw new MarmotError('publish-failed', `send: the group ended before intent ${record.intentId} was published; its durable record is resumed by the next holder`, {
|
|
1046
|
+
details: { groupId: rt.groupId, intentId: record.intentId, retryable: true },
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
1049
|
+
return false;
|
|
1050
|
+
}
|
|
1051
|
+
async function settlePublishedRecord(rt, record, opts) {
|
|
1052
|
+
// DD1/AC-DRIVE-8: a commit re-drive never takes the stale-settle path below (which fails the record `intentFailed`): its changes are
|
|
1053
|
+
// still owed, so it is re-minted at the tip with every build-time check re-run.
|
|
1054
|
+
if (isCommitRedrive(record) && record.parentEpoch !== rt.engine.state.groupContext.epoch) {
|
|
1055
|
+
await remintStaleRedrive(rt, record, opts.fromRetryTimer === true);
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
if (isCommitKind(record.kind) && record.parentEpoch !== rt.engine.state.groupContext.epoch) {
|
|
1059
|
+
let remint;
|
|
1060
|
+
if (record.reenqueuedFrom !== undefined && record.intent.type === 'self-update') {
|
|
1061
|
+
remint = await remintStaleReenqueue(rt, record, opts.fromRetryTimer === true);
|
|
1062
|
+
if (remint === 'done')
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
const envelope = record.envelope;
|
|
1066
|
+
// Only a commit a relay holds can be extended by peers; probe before retaining. Fence/removal fails at once
|
|
1067
|
+
// (nothing can ever follow). A probe timeout keeps the record and retries; only timer-driven timeouts spend the
|
|
1068
|
+
// publish budget, so the intent fails after maxPublishAttempts timer-driven retries (see staleProbe).
|
|
1069
|
+
let retained = remint === 'fenced' ? undefined : staleOwnCommitWindow(rt, record);
|
|
1070
|
+
if (retained !== undefined) {
|
|
1071
|
+
const found = await staleProbe(rt, record, envelope, opts.fromRetryTimer === true);
|
|
1072
|
+
if (found === 'retry')
|
|
1073
|
+
return;
|
|
1074
|
+
if (found !== 'stored')
|
|
1075
|
+
retained = undefined; // absent, or unverified after the probe budget: today's drop
|
|
1076
|
+
}
|
|
1077
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1078
|
+
if (retained !== undefined) {
|
|
1079
|
+
await writeWindow(tx.artifact('rewindTree'), retained);
|
|
1080
|
+
// The retained commit is ledgered like a confirmed one, so a relay redelivery of it is a duplicate
|
|
1081
|
+
// (the engine's own-echo dedup is in-memory only and would replay it as a foreign commit).
|
|
1082
|
+
await writeTerminalLedgerEntry(tx.artifact('terminalLedger'), envelope.id, {
|
|
1083
|
+
disposition: 'own',
|
|
1084
|
+
at: rt.ctx.clock.now(),
|
|
1085
|
+
createdAt: envelope.created_at,
|
|
1086
|
+
...(record.kind === 'self-update' ? { node: tagOf(deserializeClientState(record.pending.newState)), selfUpdateIntentId: reportedIntentIdOf(record), attempts: record.attempts, ...rekeyOf(record) } : {}),
|
|
1087
|
+
// S9 (DD17): the node link and root only — NO marker. This commit lost its epoch to a foreign one before
|
|
1088
|
+
// it was ever confirmed, so there is no confirmed change to re-drive from; a later rewind that abandons
|
|
1089
|
+
// its node only has to know it is an own commit (never announced as a withdrawn message).
|
|
1090
|
+
...(declaresOwnChange(record) && record.pending !== undefined ? { node: tagOf(deserializeClientState(record.pending.newState)), ownCommitRoot: reportedIntentIdOf(record) } : {}),
|
|
1091
|
+
});
|
|
1092
|
+
}
|
|
1093
|
+
await deleteEndingRecord(tx, record, 'concurrent-change');
|
|
1094
|
+
}));
|
|
1095
|
+
if (retained !== undefined)
|
|
1096
|
+
await adoptRetainedWindow(rt);
|
|
1097
|
+
await terminalFail(rt, record, opts.hasCaller, new Error('a foreign commit at this epoch has already been confirmed'));
|
|
1098
|
+
return;
|
|
1099
|
+
}
|
|
1100
|
+
const envelope = record.envelope;
|
|
1101
|
+
const relays = relayPlaneFor(rt);
|
|
1102
|
+
let alreadyThere = false;
|
|
1103
|
+
if (opts.probeFirst) {
|
|
1104
|
+
// A probe that throws is treated like one that timed out (`own-commit-probe-retry-unbounded`): the record goes to
|
|
1105
|
+
// its publish retry, which spends `maxPublishAttempts` and drops it as before once exhausted. Letting it reject
|
|
1106
|
+
// the resume run instead re-ran the whole resume on the backoff forever, never spending an attempt.
|
|
1107
|
+
let result;
|
|
1108
|
+
try {
|
|
1109
|
+
result = await rt.guard.callSync(rt.groupId, 'transport', () => rt.ctx.transport.requestWithDeadline(relays, { ids: [envelope.id] }, { timeoutMs: rt.ctx.historyTimeoutMs }));
|
|
1110
|
+
}
|
|
1111
|
+
catch (error) {
|
|
1112
|
+
if (rt.disposed || rt.ctx.isDisposed() || isMarmotError(error, 'disposed'))
|
|
1113
|
+
throw error;
|
|
1114
|
+
}
|
|
1115
|
+
assertNotDisposed(rt);
|
|
1116
|
+
if (result === undefined || result.timedOut) {
|
|
1117
|
+
schedulePublishRetry(rt, record.intentId, record.attempts);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
alreadyThere = result.events.some((e) => e.id === envelope.id);
|
|
1121
|
+
}
|
|
1122
|
+
if (!alreadyThere) {
|
|
1123
|
+
// AC-HOOK-2: after the staging transaction (and the probe), before the first publish of this envelope.
|
|
1124
|
+
if (!(await runBeforePublishHooks(rt, record, opts)))
|
|
1125
|
+
return;
|
|
1126
|
+
let responses;
|
|
1127
|
+
try {
|
|
1128
|
+
// Inside the try: a transport whose publish() throws synchronously takes the same recovery path.
|
|
1129
|
+
responses = await rt.guard.callSync(rt.groupId, 'transport', () => rt.ctx.transport.publish(relays, envelope));
|
|
1130
|
+
}
|
|
1131
|
+
catch (error) {
|
|
1132
|
+
// 'stored': the retry ceiling was reached but a relay holds the envelope — confirm it below.
|
|
1133
|
+
if ((await onPublishFailed(rt, record, opts, error)) === 'handled')
|
|
1134
|
+
return;
|
|
1135
|
+
}
|
|
1136
|
+
assertNotDisposed(rt);
|
|
1137
|
+
if (responses !== undefined) {
|
|
1138
|
+
const { outcome } = classifyPublish(relays, responses);
|
|
1139
|
+
if (outcome === 'terminal') {
|
|
1140
|
+
await onPublishTerminal(rt, record, opts, new Error('every relay refused the publish'), { reason: 'publish-refused', responses, attempts: record.attempts + 1 });
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
1143
|
+
if (outcome === 'retryable' && (await onPublishFailed(rt, record, opts, new Error('no relay accepted the publish'))) === 'handled')
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
const confirming = { ...record, state: 'pending-confirm' };
|
|
1148
|
+
try {
|
|
1149
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1150
|
+
await writePendingIntentRecord(tx.artifact('pendingIntents'), confirming);
|
|
1151
|
+
// AC-RATCH-1: a relay holds an own application message — a pending post-load skip at or before it is delivered.
|
|
1152
|
+
if (record.kind === 'application')
|
|
1153
|
+
await markRatchetSkipDelivered(tx.extension(RATCHET_SKIP_EXTENSION), record.ordering);
|
|
1154
|
+
}));
|
|
1155
|
+
}
|
|
1156
|
+
catch (error) {
|
|
1157
|
+
// A relay accepted the envelope, but the record is still durably
|
|
1158
|
+
// pending-publish: keep it, put a staged commit back at the durable
|
|
1159
|
+
// (parent) state, and settle in the background — the retry re-publishes
|
|
1160
|
+
// (a duplicate) and confirms from the record. The caller still sees the failure.
|
|
1161
|
+
if (opts.live !== undefined && record.kind !== 'application')
|
|
1162
|
+
rt.engine.publishFailed(opts.live);
|
|
1163
|
+
if (!backgroundRetryEnds(rt, error))
|
|
1164
|
+
schedulePublishRetry(rt, record.intentId, record.attempts + 1);
|
|
1165
|
+
throw error;
|
|
1166
|
+
}
|
|
1167
|
+
assertNotDisposed(rt);
|
|
1168
|
+
await confirmRecord(rt, confirming, 0, opts.live, opts.hasCaller);
|
|
1169
|
+
}
|
|
1170
|
+
// ---------------------------------------------------------------------------
|
|
1171
|
+
// Drive a record through the engine (fresh send, and envelope-less resume)
|
|
1172
|
+
// ---------------------------------------------------------------------------
|
|
1173
|
+
/**
|
|
1174
|
+
* A signer call made on behalf of a group op: framed on the account scope AND
|
|
1175
|
+
* the group's scope, so a signer that awaits a mutating call on this group
|
|
1176
|
+
* (which would queue behind the op awaiting the signer) is rejected
|
|
1177
|
+
* `reentrant` instead of deadlocking; other groups stay reachable.
|
|
1178
|
+
*/
|
|
1179
|
+
function callSignerForGroup(rt, fn) {
|
|
1180
|
+
return rt.guard.call(rt.groupId, 'signer', () => rt.guard.call(ACCOUNT_SCOPE, 'signer', fn));
|
|
1181
|
+
}
|
|
1182
|
+
/**
|
|
1183
|
+
* A commit intent that carries ONLY `removeAccounts` (and its leaf-key snapshot): what `removeMember` records. The predicate is
|
|
1184
|
+
* deliberately narrow: a record that also carries an invite, a forget, an admin list, a profile or components keeps the generic stale-parent
|
|
1185
|
+
* rule below, because those parts are bound to the epoch they were recorded at.
|
|
1186
|
+
*/
|
|
1187
|
+
function isAccountRemovalOnly(intent) {
|
|
1188
|
+
return intent.type === 'commit' && intent.removeAccounts !== undefined && intent.removeAccountLeafKeys !== undefined && intent.invite === undefined && intent.removeLeafKeys === undefined && intent.admins === undefined && intent.profile === undefined && intent.components === undefined;
|
|
1189
|
+
}
|
|
1190
|
+
/**
|
|
1191
|
+
* A `removeMember` record is about an ACCOUNT as it stood when the call was made, not about an epoch. So, before the generic
|
|
1192
|
+
* stale-parent rule, an envelope-less record of it
|
|
1193
|
+
* - none of whose recorded leaves (`removeAccountLeafKeys`, the snapshot taken at record time) is still in the tree — as the
|
|
1194
|
+
* recorded key or as the key the target rotated it to in place, an edge of the retained window — is dropped
|
|
1195
|
+
* WITHOUT a commit: whatever the account holds now was added AFTER the call (a removed member that rejoined stays), and no
|
|
1196
|
+
* empty epoch bump is made. A waiting caller resolves as a no-op; an unattended record reports `intentFailed`
|
|
1197
|
+
* (`send-rejected`, `details.reason: 'absent'`) once;
|
|
1198
|
+
* - one of whose recorded leaves is still held and whose parent epoch moved is re-parented to the current epoch and built on
|
|
1199
|
+
* the current tree: ALL the account's current leaves are removed (one it added meanwhile too), and a caller demoted
|
|
1200
|
+
* meanwhile is refused by the engine's `authorizeCommit` backstop.
|
|
1201
|
+
* Returns the record to drive, or `undefined` when it was dropped.
|
|
1202
|
+
*/
|
|
1203
|
+
async function prepareAccountRemoval(rt, record, hasCaller) {
|
|
1204
|
+
const intent = record.intent;
|
|
1205
|
+
// Assertion: only a record carrying nothing but removeAccounts (plus its snapshot) and no envelope is treated here.
|
|
1206
|
+
if (!isAccountRemovalOnly(intent) || record.envelope !== undefined)
|
|
1207
|
+
return record;
|
|
1208
|
+
// AC-RM-1/2: a recorded leaf also counts when the target rotated it in place (a succession edge of the retained window,
|
|
1209
|
+
// on the canonical path or, as its predecessor, on an abandoned branch). A leaf got by a Remove plus a fresh Add has no
|
|
1210
|
+
// such edge and stays "added later"; a same-device re-add under a key the lineage reaches counts (residual 5).
|
|
1211
|
+
if (!holdsSnapshotLeaf(rt.engine.state, rt.window, intent.removeAccountLeafKeys, intent.removeAccounts, rt.exposedState)) {
|
|
1212
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deletePendingIntentRecord(tx.artifact('pendingIntents'), record.intentId)));
|
|
1213
|
+
assertNotDisposed(rt);
|
|
1214
|
+
if (hasCaller)
|
|
1215
|
+
notifySettled(rt, record, undefined);
|
|
1216
|
+
else {
|
|
1217
|
+
const reportedIntentId = reportedIntentIdOf(record);
|
|
1218
|
+
reportIntentFailed(rt, record, new MarmotError('send-rejected', `send: intent ${reportedIntentId} on group ${rt.groupId} was dropped: the account holds none of the leaves recorded for the call any more`, {
|
|
1219
|
+
details: { groupId: rt.groupId, intentId: reportedIntentId, reason: 'absent' },
|
|
1220
|
+
}));
|
|
1221
|
+
}
|
|
1222
|
+
scheduleDeferredReenqueueRetry(rt, record.intentId);
|
|
1223
|
+
return undefined;
|
|
1224
|
+
}
|
|
1225
|
+
const epoch = rt.engine.state.groupContext.epoch;
|
|
1226
|
+
if (record.parentEpoch === epoch)
|
|
1227
|
+
return record;
|
|
1228
|
+
const ready = { ...record, parentEpoch: epoch };
|
|
1229
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), ready)));
|
|
1230
|
+
assertNotDisposed(rt);
|
|
1231
|
+
return ready;
|
|
1232
|
+
}
|
|
1233
|
+
async function driveIntent(rt, given, hasCaller, redrive) {
|
|
1234
|
+
// AC-PIPE-15: the one place an own envelope is minted. Every caller checks the gate first; this keeps it so (the
|
|
1235
|
+
// record stays durable and is driven once the gate opens).
|
|
1236
|
+
assertSendGateOpen(rt, 'send');
|
|
1237
|
+
// AC-RACE-1: a re-enqueued self-update is never stale — it follows the tip (see reparentReenqueued).
|
|
1238
|
+
const prepared = await prepareAccountRemoval(rt, await reparentReenqueued(rt, given), hasCaller);
|
|
1239
|
+
if (prepared === undefined)
|
|
1240
|
+
return;
|
|
1241
|
+
const record = prepared;
|
|
1242
|
+
// A refusal before the staging transaction ends the record terminally: ONE transaction deletes it and writes the terminals that end it
|
|
1243
|
+
// (S11) -- the verdicts this run decided and never staged (each with its own reason) first, then the changes still owed -- and
|
|
1244
|
+
// `terminalFail` hands them over and fires `intentFailed`.
|
|
1245
|
+
const failBuild = async (cause, details) => {
|
|
1246
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deleteEndingRecord(tx, record, lostReasonOf(details), redrive?.newlyLost)));
|
|
1247
|
+
await terminalFail(rt, record, hasCaller, cause, details);
|
|
1248
|
+
};
|
|
1249
|
+
// What the commit is built from: the stored intent, or (a commit re-drive) the payload rebuilt in memory after the DD11 checks.
|
|
1250
|
+
const build = redrive?.intent ?? record.intent;
|
|
1251
|
+
// A resumed commit-kind intent whose epoch has moved on must be dropped before staging, or the engine
|
|
1252
|
+
// enters PendingPublish for a commit this path then discards.
|
|
1253
|
+
if (isCommitKind(record.kind) && record.parentEpoch !== rt.engine.state.groupContext.epoch) {
|
|
1254
|
+
await failBuild(new Error('a foreign commit at this epoch has already been confirmed'));
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
// AC-REKEY-6: a rekey that has nothing left to do (the leaf already carries the target, or the device key moved on)
|
|
1258
|
+
// is dropped before anything is staged — also before the staged-Adds commit it would otherwise cause.
|
|
1259
|
+
if (record.intent.type === 'self-update' && record.intent.rekey !== undefined) {
|
|
1260
|
+
const plan = await planRekey(rt, record.intent.rekey.target);
|
|
1261
|
+
assertNotDisposed(rt);
|
|
1262
|
+
if (plan.kind !== 'build') {
|
|
1263
|
+
await dropRekey(rt, record, hasCaller, plan.kind);
|
|
1264
|
+
return;
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
// S13: a self-update over staged Adds — also a resumed one whose staging failed before a peer's Add arrived —
|
|
1268
|
+
// is the two-commit path: the Add commit first, then a real self-update.
|
|
1269
|
+
if (record.intent.type === 'self-update' && hasStagedAdds(rt.engine.state)) {
|
|
1270
|
+
await driveSelfUpdateOverStagedAdds(rt, record, hasCaller);
|
|
1271
|
+
return;
|
|
1272
|
+
}
|
|
1273
|
+
// A staged Add admitting a leaf of an account / a leaf key this commit removes would keep it: refused at build time as well.
|
|
1274
|
+
if (build.type === 'commit' && (build.removeAccounts !== undefined || build.removeLeafKeys !== undefined)) {
|
|
1275
|
+
const conflict = stagedAddConflict(build.removeAccounts !== undefined ? 'removeMember' : 'forget', rt.groupId, rt.engine.state, { accounts: build.removeAccounts, leafKeys: build.removeLeafKeys });
|
|
1276
|
+
if (conflict !== undefined) {
|
|
1277
|
+
await failBuild(conflict);
|
|
1278
|
+
return;
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1281
|
+
// A rekey makes no signer call at all (AC-REKEY-11): its patch comes from the account store, its proof is the stored one,
|
|
1282
|
+
// and the account pubkey is only ever used by the commit kinds that build Welcomes.
|
|
1283
|
+
const selfPubkey = isRekeyRecord(record) ? '' : await callSignerForGroup(rt, () => rt.ctx.signer.getPublicKey());
|
|
1284
|
+
// S5/AC-INVITE-4: every Add an own commit carries admits a member, so staged Adds (a non-admin's
|
|
1285
|
+
// accepted proposal, an admin's) get Welcomes exactly like an invite's listed invitees. Looked up
|
|
1286
|
+
// before the engine stages anything (a staged commit must not wait on relays).
|
|
1287
|
+
let stagedAdds = [];
|
|
1288
|
+
if (record.kind === 'commit' && hasStagedAdds(rt.engine.state)) {
|
|
1289
|
+
const verifier = rt.ctx.verifier;
|
|
1290
|
+
stagedAdds = await stagedAddRecipients(rt.engine.state, {
|
|
1291
|
+
relays: [...new Set([...rt.ctx.relays.defaults, ...relayPlaneFor(rt)])],
|
|
1292
|
+
lookupTimeoutMs: Math.min(STAGED_ADD_LOOKUP_TIMEOUT_MS, rt.ctx.historyTimeoutMs),
|
|
1293
|
+
request: (relays, filter, opts) => rt.guard.callSync(rt.groupId, 'transport', () => rt.ctx.transport.requestWithDeadline(relays, filter, opts)),
|
|
1294
|
+
verify: verifier === undefined ? undefined : (event) => rt.guard.call(rt.groupId, 'verifier', () => verifier(event)),
|
|
1295
|
+
});
|
|
1296
|
+
assertNotDisposed(rt);
|
|
1297
|
+
}
|
|
1298
|
+
// AC-REKEY-2: a rekey's patch (device key pair, proof extension, duplicate-leaf Removes) is built from the account
|
|
1299
|
+
// store (outside any group transaction) against the state the commit is made on; the record names only the target.
|
|
1300
|
+
// Re-planned, not reused from the AC-REKEY-6 check above: `planRekey` re-reads `deviceRegistry/self`, which a rotation
|
|
1301
|
+
// prepare (device queue, not this group's queue) may have moved on in between — the drop below is reachable.
|
|
1302
|
+
let engineIntent;
|
|
1303
|
+
if (record.intent.type === 'self-update' && record.intent.rekey !== undefined) {
|
|
1304
|
+
const plan = await planRekey(rt, record.intent.rekey.target);
|
|
1305
|
+
assertNotDisposed(rt);
|
|
1306
|
+
if (plan.kind !== 'build') {
|
|
1307
|
+
await dropRekey(rt, record, hasCaller, plan.kind);
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
1310
|
+
engineIntent = plan.intent;
|
|
1311
|
+
}
|
|
1312
|
+
else
|
|
1313
|
+
engineIntent = toEngineIntent(build, selfPubkey, rt.engine.state, rt.groupId);
|
|
1314
|
+
// AC-RATCH-1: the first own application message after a load skips `ratchetSkipMargin` generations of the own
|
|
1315
|
+
// sender ratchet — past any own message the loaded (possibly restored) state never saw. The engine applies the
|
|
1316
|
+
// skip to the state it encrypts under and adopts it only with this message; the flag is cleared only once the
|
|
1317
|
+
// staging transaction below made the advanced state durable together with the envelope.
|
|
1318
|
+
const ratchetSkip = engineIntent.kind === 'applicationMessage' && rt.ratchetSkipPending && !rt.ctx.disableRatchetSkipForTest ? rt.ctx.ratchetSkipMargin : 0;
|
|
1319
|
+
let result;
|
|
1320
|
+
try {
|
|
1321
|
+
result = await rt.engine.send(engineIntent.kind === 'applicationMessage' && ratchetSkip > 0 ? { ...engineIntent, ratchetSkip } : engineIntent);
|
|
1322
|
+
}
|
|
1323
|
+
catch (error) {
|
|
1324
|
+
// AC-HIST-4: the device lost its own path secrets (a restored snapshot) — fenced for good, never retried.
|
|
1325
|
+
// Defensive: ts-mls throws this error only while decrypting an inbound commit's UpdatePath
|
|
1326
|
+
// (`processCommit`), which `engine.send()` never does (it only creates messages and commits), so
|
|
1327
|
+
// fencing is discovered on ingest/replay. Kept so a future engine send path that does process a
|
|
1328
|
+
// commit still fences instead of surfacing as an ordinary send failure.
|
|
1329
|
+
if (isFencingError(error)) {
|
|
1330
|
+
// S11: a fenced group strands a commit re-drive (DD11): its changes are lost `stranded`, written with the delete.
|
|
1331
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deleteEndingRecord(tx, record, 'stranded', redrive?.newlyLost)));
|
|
1332
|
+
await fenceGroup(rt, error);
|
|
1333
|
+
await failAwaitingIntents(rt);
|
|
1334
|
+
await announceTerminals(rt);
|
|
1335
|
+
const fenced = fencedError(rt.groupId, 'send', error);
|
|
1336
|
+
// A re-drive has no caller to reject: its one `intentFailed` under the root id (AC-RM-1), after its `own-change-lost`.
|
|
1337
|
+
if (!hasCaller)
|
|
1338
|
+
reportIntentFailed(rt, record, fenced);
|
|
1339
|
+
throw fenced;
|
|
1340
|
+
}
|
|
1341
|
+
// A removal that fails the admin-leaf coupling check: a staged admin-policy update (committed together with the removal)
|
|
1342
|
+
// still names a removed admin.
|
|
1343
|
+
const couplingFailure = isAccountRemovalOnly(build) && error?.violation?.reason === 'admin-leaf-coupling';
|
|
1344
|
+
await failBuild(error, couplingFailure ? { reason: 'staged-admin-policy-update' } : {});
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
assertNotDisposed(rt);
|
|
1348
|
+
if (result.kind === 'groupEvolution' || result.kind === 'selfUpdate') {
|
|
1349
|
+
const failures = enforceStateTransition(rt.engine.state, result.pending.newState);
|
|
1350
|
+
if (failures.length > 0) {
|
|
1351
|
+
rt.engine.publishFailed(result.pending);
|
|
1352
|
+
// AC-REKEY-4: a rekey whose leaf proof is absent or binds another key is refused here, before anything is published.
|
|
1353
|
+
await failBuild(new Error('own commit state failed proof enforcement'), { proofFailures: failures, ...(isRekeyRecord(record) ? { reason: 'proof-invalid' } : {}) });
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
// S5: an invite commit's Welcome rumors, built from the engine's one MLS Welcome now —
|
|
1358
|
+
// it exists only in this process — and staged below with the envelope.
|
|
1359
|
+
let welcomes;
|
|
1360
|
+
const listed = build.type === 'commit' && build.invite !== undefined
|
|
1361
|
+
? build.invite.invitees.map((i) => {
|
|
1362
|
+
const event = i.event;
|
|
1363
|
+
return { inviteeRef: i.inviteeRef, pubkey: event.pubkey, keyPackageEventId: event.id };
|
|
1364
|
+
})
|
|
1365
|
+
: [];
|
|
1366
|
+
if (listed.length > 0 || stagedAdds.length > 0) {
|
|
1367
|
+
try {
|
|
1368
|
+
if (result.kind !== 'groupEvolution' || result.welcome === undefined)
|
|
1369
|
+
throw new Error('a commit adding members produced no Welcome');
|
|
1370
|
+
// The Welcomes' `commit` tag digests the MLS bytes this commit is published and confirmed with.
|
|
1371
|
+
const commitMessage = result.pending.commitMessage;
|
|
1372
|
+
if (commitMessage === undefined)
|
|
1373
|
+
throw new Error('a commit adding members staged no commit message');
|
|
1374
|
+
const recipients = recipientsOfWelcome(result.welcome.welcome, [...listed, ...stagedAdds]);
|
|
1375
|
+
if (listed.some((l) => !recipients.some((r) => r.inviteeRef === l.inviteeRef)))
|
|
1376
|
+
throw new Error('the Welcome holds no secret for an invitee');
|
|
1377
|
+
welcomes = buildWelcomeRumors({
|
|
1378
|
+
welcome: result.welcome.welcome,
|
|
1379
|
+
commit: encode(mlsMessageEncoder, commitMessage),
|
|
1380
|
+
author: selfPubkey,
|
|
1381
|
+
groupRelays: groupRelayPlane(getMarmotGroupView(result.pending.newState)?.relays ?? [], rt.ctx.relays.defaults),
|
|
1382
|
+
invitees: recipients,
|
|
1383
|
+
});
|
|
1384
|
+
}
|
|
1385
|
+
catch (error) {
|
|
1386
|
+
if (result.kind !== 'applicationMessage')
|
|
1387
|
+
rt.engine.publishFailed(result.pending);
|
|
1388
|
+
await failBuild(error, { reason: 'welcome-build-failed' });
|
|
1389
|
+
return;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
const envelope = result.envelope;
|
|
1393
|
+
const storedPending = result.kind === 'applicationMessage' ? undefined : encodeStoredPending(result.pending);
|
|
1394
|
+
// S10: the message's ordering key, read now under the state it is sent in (its confirmation may come later).
|
|
1395
|
+
const ordering = result.kind === 'applicationMessage' ? (await ownMessage(rt, record, envelope))?.orderingKey : undefined;
|
|
1396
|
+
const updatedRecord = {
|
|
1397
|
+
...record,
|
|
1398
|
+
envelope: envelope,
|
|
1399
|
+
...(ordering === undefined ? {} : { ordering }),
|
|
1400
|
+
...(storedPending === undefined ? {} : { pending: storedPending }),
|
|
1401
|
+
...(welcomes === undefined ? {} : { welcomes }),
|
|
1402
|
+
};
|
|
1403
|
+
// Staging: the envelope-bearing record (and, for an application message,
|
|
1404
|
+
// the state with the advanced sending ratchet) is durable BEFORE anything
|
|
1405
|
+
// is published. If it fails, nothing was published: the engine — which
|
|
1406
|
+
// already advanced the ratchet or entered PendingPublish — is put back at
|
|
1407
|
+
// the durable state, so engine == durable at the op boundary and the
|
|
1408
|
+
// envelope-less record resumes like after a crash here.
|
|
1409
|
+
try {
|
|
1410
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1411
|
+
await writePendingIntentRecord(tx.artifact('pendingIntents'), updatedRecord);
|
|
1412
|
+
// S11: the verdicts this run decided are durable with the record that carries them, as outbox terminals (signalled once staged).
|
|
1413
|
+
if (redrive !== undefined)
|
|
1414
|
+
await enqueueTerminals(tx.extension(OWN_CHANGE_OUTBOX_EXTENSION), lostTerminals(reportedIntentIdOf(record), redrive.newlyLost));
|
|
1415
|
+
if (result.kind === 'applicationMessage')
|
|
1416
|
+
await writeGroupState(tx.artifact('clientState'), result.newState);
|
|
1417
|
+
// AC-RATCH-1: the skip this message carries, durable with it; a later load does not skip again on top of
|
|
1418
|
+
// it until a relay accepted an own message at or after it (`markRatchetSkipDelivered` below).
|
|
1419
|
+
if (result.kind === 'applicationMessage' && ratchetSkip > 0) {
|
|
1420
|
+
await writeRatchetSkipMarker(tx.extension(RATCHET_SKIP_EXTENSION), {
|
|
1421
|
+
epoch: result.newState.groupContext.epoch,
|
|
1422
|
+
tag: tagOf(result.newState),
|
|
1423
|
+
generation: nextOwnApplicationGeneration(result.newState) - 1,
|
|
1424
|
+
delivered: false,
|
|
1425
|
+
});
|
|
1426
|
+
}
|
|
1427
|
+
}));
|
|
1428
|
+
}
|
|
1429
|
+
catch (error) {
|
|
1430
|
+
await restoreDurableEngine(rt, result.kind === 'applicationMessage' ? undefined : result.pending);
|
|
1431
|
+
throw error;
|
|
1432
|
+
}
|
|
1433
|
+
if (ratchetSkip > 0)
|
|
1434
|
+
rt.ratchetSkipPending = false;
|
|
1435
|
+
assertNotDisposed(rt);
|
|
1436
|
+
if (result.kind === 'applicationMessage')
|
|
1437
|
+
rt.exposedState = rt.engine.state;
|
|
1438
|
+
// A commit re-drive's verdicts are durable now (with the envelope): they may be signalled (at least once, S11).
|
|
1439
|
+
await redrive?.staged();
|
|
1440
|
+
const live = result.kind === 'applicationMessage' ? undefined : result.pending;
|
|
1441
|
+
await settlePublishedRecord(rt, updatedRecord, { probeFirst: false, live, hasCaller });
|
|
1442
|
+
}
|
|
1443
|
+
/**
|
|
1444
|
+
* Puts the engine back at the persisted state after a failed staging write:
|
|
1445
|
+
* releases a staged commit lifecycle, then rebuilds from the durable
|
|
1446
|
+
* `clientState`. When that read fails too, the next op reloads first.
|
|
1447
|
+
*/
|
|
1448
|
+
async function restoreDurableEngine(rt, pending) {
|
|
1449
|
+
if (pending !== undefined)
|
|
1450
|
+
rt.engine.publishFailed(pending);
|
|
1451
|
+
try {
|
|
1452
|
+
await rt.reloadFromDurable();
|
|
1453
|
+
}
|
|
1454
|
+
catch {
|
|
1455
|
+
rt.needsReload = true;
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
// ---------------------------------------------------------------------------
|
|
1459
|
+
// Public entry points
|
|
1460
|
+
// ---------------------------------------------------------------------------
|
|
1461
|
+
/**
|
|
1462
|
+
* The shared start of every own-intent operation: resumes durable records,
|
|
1463
|
+
* then refuses while a confirmation is blocked. Returns the remaining records.
|
|
1464
|
+
*/
|
|
1465
|
+
async function preflight(rt, operation) {
|
|
1466
|
+
// AC-HIST-4: a fenced group accepts no further sends (and resumes nothing it recorded before).
|
|
1467
|
+
assertNotFenced(rt, operation);
|
|
1468
|
+
// Nor does a disbanded one (client/group-disband.ts).
|
|
1469
|
+
assertNotDisbanded(rt, operation);
|
|
1470
|
+
await resumePending(rt);
|
|
1471
|
+
assertNotDisposed(rt);
|
|
1472
|
+
const view = rt.groupStore.artifact('pendingIntents');
|
|
1473
|
+
const remaining = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(view));
|
|
1474
|
+
const stillBlocked = remaining.find((r) => r.state === 'pending-confirm');
|
|
1475
|
+
if (stillBlocked !== undefined) {
|
|
1476
|
+
throw new MarmotError('confirm-blocked', `${operation}: group ${rt.groupId} has a blocked confirmation (intent ${stillBlocked.intentId})`, {
|
|
1477
|
+
details: { groupId: rt.groupId, intentId: stillBlocked.intentId },
|
|
1478
|
+
});
|
|
1479
|
+
}
|
|
1480
|
+
return remaining;
|
|
1481
|
+
}
|
|
1482
|
+
/** AC-INVITE-7: a removed local leaf sends nothing — refused before any record or transport call. */
|
|
1483
|
+
function assertMember(rt, operation) {
|
|
1484
|
+
if (rt.engine.state.groupActiveState.kind === 'removedFromGroup') {
|
|
1485
|
+
throw new MarmotError('not-a-member', `${operation}: the local leaf has been removed from group ${rt.groupId}`, { details: { groupId: rt.groupId } });
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
function assertNoOwnCommitPending(rt, remaining, operation) {
|
|
1489
|
+
// An own commit still pending-publish may already sit on a relay (the
|
|
1490
|
+
// OK was lost, or the resume probe timed out): a second own commit
|
|
1491
|
+
// built now would be a sibling at the same epoch — a fork of our own
|
|
1492
|
+
// making. Refused until the pending one confirms or goes terminal.
|
|
1493
|
+
const pendingCommit = remaining.find((r) => isCommitKind(r.kind));
|
|
1494
|
+
if (pendingCommit !== undefined) {
|
|
1495
|
+
throw new MarmotError('commit-pending', `${operation}: group ${rt.groupId} already has an own commit pending publish (intent ${pendingCommit.intentId})`, {
|
|
1496
|
+
details: { groupId: rt.groupId, pendingIntentId: pendingCommit.intentId, retryable: true },
|
|
1497
|
+
});
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
async function recordIntent(rt, kind, intent, state = 'pending-publish',
|
|
1501
|
+
// Evaluated when the call is made, before the seq lookup below awaits: a caller that decided on a state snapshot passes that snapshot's epoch.
|
|
1502
|
+
parentEpoch = rt.engine.state.groupContext.epoch) {
|
|
1503
|
+
const view = rt.groupStore.artifact('pendingIntents');
|
|
1504
|
+
const seq = await rt.guard.call(rt.groupId, 'store', () => nextSeq(view));
|
|
1505
|
+
const record = {
|
|
1506
|
+
v: 1,
|
|
1507
|
+
intentId: randomIntentId(),
|
|
1508
|
+
seq,
|
|
1509
|
+
kind,
|
|
1510
|
+
state,
|
|
1511
|
+
parentEpoch,
|
|
1512
|
+
intent,
|
|
1513
|
+
attempts: 0,
|
|
1514
|
+
};
|
|
1515
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), record)));
|
|
1516
|
+
assertNotDisposed(rt);
|
|
1517
|
+
return record;
|
|
1518
|
+
}
|
|
1519
|
+
export async function sendIntent(rt, intent, observer) {
|
|
1520
|
+
const remaining = await preflight(rt, 'send');
|
|
1521
|
+
validateIntent(intent);
|
|
1522
|
+
assertMember(rt, 'send');
|
|
1523
|
+
// AC-PIPE-15: before the first post-load sync the intent is accepted durably and nothing is encrypted.
|
|
1524
|
+
if (!rt.sendGateOpen) {
|
|
1525
|
+
const accepted = await acceptAwaitingSync(rt, intent, remaining);
|
|
1526
|
+
observer?.onRecorded?.(accepted.intentId);
|
|
1527
|
+
return {};
|
|
1528
|
+
}
|
|
1529
|
+
if (intent.type === 'application') {
|
|
1530
|
+
const queued = await rt.guard.call(rt.groupId, 'store', () => listQueuedRumorRecords(rt.groupStore.artifact('queuedRumors')));
|
|
1531
|
+
const applicationPending = remaining.some((r) => r.kind === 'application');
|
|
1532
|
+
if (mustQueue(rt.engine.state, queued.length, applicationPending))
|
|
1533
|
+
return enqueueRumor(rt, intent.rumor, queued);
|
|
1534
|
+
}
|
|
1535
|
+
if (isCommitKind(intent.type)) {
|
|
1536
|
+
assertNoOwnCommitPending(rt, remaining, 'send');
|
|
1537
|
+
if (intent.type !== 'proposal')
|
|
1538
|
+
await applyProposalPolicy(rt, 'send');
|
|
1539
|
+
}
|
|
1540
|
+
if (intent.type === 'self-update' && hasStagedAdds(rt.engine.state)) {
|
|
1541
|
+
await driveSelfUpdateOverStagedAdds(rt, undefined, true, observer);
|
|
1542
|
+
return {};
|
|
1543
|
+
}
|
|
1544
|
+
const record = await recordIntent(rt, intent.type, encodeGroupIntent(intent));
|
|
1545
|
+
observer?.onRecorded?.(record.intentId);
|
|
1546
|
+
await driveIntent(rt, record, true);
|
|
1547
|
+
return {};
|
|
1548
|
+
}
|
|
1549
|
+
/**
|
|
1550
|
+
* A self-update over staged Add proposals is two own commits: an Add-only
|
|
1551
|
+
* commit carries no UpdatePath (the committer's leaf would not rotate), and the
|
|
1552
|
+
* engine's self-update path discards the MLS Welcome the Adds need. So the
|
|
1553
|
+
* staged proposals are committed first — Welcomes and all — and once that
|
|
1554
|
+
* commit is confirmed the self-update runs at the next epoch as a real engine
|
|
1555
|
+
* `selfUpdate` (one own commit in flight).
|
|
1556
|
+
*
|
|
1557
|
+
* Both intents are durable before anything is published (S13, AC-PIPE-1): ONE
|
|
1558
|
+
* transaction writes the Add commit and the self-update, the latter carrying
|
|
1559
|
+
* `after: <commit intentId>` (see {@link resolveDependency}). This one routine
|
|
1560
|
+
* serves the live send (`update` undefined: the self-update record is created
|
|
1561
|
+
* here) and every self-update record driven while Adds are staged (a resume
|
|
1562
|
+
* after a failed staging write, a peer's Add having arrived meanwhile).
|
|
1563
|
+
*
|
|
1564
|
+
* With a caller: if no relay accepted the Add commit (it stays a durable
|
|
1565
|
+
* intent and is retried) the call is refused `commit-pending` (reason
|
|
1566
|
+
* `staged-adds-commit-unconfirmed`); if its confirmation is blocked,
|
|
1567
|
+
* `confirm-blocked` — either way `details.selfUpdateIntentId` names the
|
|
1568
|
+
* self-update, which stays durable and follows the commit once it confirms.
|
|
1569
|
+
* Those refusals mean "the self-update has not run yet". If the Add commit
|
|
1570
|
+
* goes terminal with the epoch unchanged, the self-update is dropped with it
|
|
1571
|
+
* and the caller gets the commit's `send-rejected` (the self-update's own
|
|
1572
|
+
* settlement is reported to the settled-intent listeners too). If it goes terminal because
|
|
1573
|
+
* a foreign commit confirmed at its epoch (`stale-parent`), nothing holds the
|
|
1574
|
+
* self-update back: it is re-parented and driven in the same call, which
|
|
1575
|
+
* resolves once it is published (or rejects with the self-update's own error)
|
|
1576
|
+
* — one rotation, nothing left for the caller to retry. If the self-update
|
|
1577
|
+
* itself fails, the caller gets that error while the first commit stays confirmed.
|
|
1578
|
+
*/
|
|
1579
|
+
async function driveSelfUpdateOverStagedAdds(rt, update, hasCaller, observer) {
|
|
1580
|
+
const { first, linked } = await linkStagedAddsCommit(rt, update);
|
|
1581
|
+
observer?.onRecorded?.(linked.intentId);
|
|
1582
|
+
try {
|
|
1583
|
+
await driveIntent(rt, first, hasCaller);
|
|
1584
|
+
}
|
|
1585
|
+
catch (error) {
|
|
1586
|
+
if (isMarmotError(error, 'publish-failed')) {
|
|
1587
|
+
throw new MarmotError('commit-pending', `send: the commit of group ${rt.groupId}'s staged proposals was not accepted by any relay; it will be retried, and the self-update follows once it is confirmed`, {
|
|
1588
|
+
cause: error,
|
|
1589
|
+
details: { groupId: rt.groupId, reason: 'staged-adds-commit-unconfirmed', pendingIntentId: first.intentId, selfUpdateIntentId: linked.intentId, retryable: true },
|
|
1590
|
+
});
|
|
1591
|
+
}
|
|
1592
|
+
if (!isMarmotError(error, 'send-rejected'))
|
|
1593
|
+
throw error;
|
|
1594
|
+
// A terminal commit (its record is gone): the self-update goes with it (epoch unchanged) — or, a foreign
|
|
1595
|
+
// commit having moved the epoch, it is re-parented and runs now, in place of the commit's failure.
|
|
1596
|
+
const after = await resolveDependency(rt, linked);
|
|
1597
|
+
// Dropped with the commit: the caller gets the commit's failure, and the self-update's own settlement is
|
|
1598
|
+
// reported too — its id was handed out (`onRecorded`), and a waiter on it (`groups.restore`) must never be
|
|
1599
|
+
// left waiting on a record that is gone. Never `intentFailed`: this call's caller is told.
|
|
1600
|
+
if (after === 'terminal')
|
|
1601
|
+
notifySettled(rt, linked, dependencyTerminalError(rt, linked, first.intentId, error));
|
|
1602
|
+
if (after === 'waiting' || after === 'terminal' || after === 'requeued')
|
|
1603
|
+
throw error;
|
|
1604
|
+
await driveIntent(rt, after, hasCaller);
|
|
1605
|
+
return;
|
|
1606
|
+
}
|
|
1607
|
+
const next = await resolveDependency(rt, linked);
|
|
1608
|
+
if (next === 'requeued')
|
|
1609
|
+
return; // a re-enqueue: kept for its next drive (no caller ever waits on one)
|
|
1610
|
+
if (next === 'waiting') {
|
|
1611
|
+
if (!hasCaller)
|
|
1612
|
+
return;
|
|
1613
|
+
throw new MarmotError('confirm-blocked', `send: the commit of group ${rt.groupId}'s staged proposals is published but its confirmation is blocked; the self-update follows once it is confirmed`, {
|
|
1614
|
+
details: { groupId: rt.groupId, intentId: first.intentId, selfUpdateIntentId: linked.intentId },
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
if (next === 'terminal') {
|
|
1618
|
+
await terminalFail(rt, linked, hasCaller, new Error('the commit of the staged proposals it follows went terminal'), { reason: 'dependency-terminal', after: first.intentId });
|
|
1619
|
+
return;
|
|
1620
|
+
}
|
|
1621
|
+
await driveIntent(rt, next, hasCaller);
|
|
1622
|
+
}
|
|
1623
|
+
/**
|
|
1624
|
+
* Writes, in one transaction, the own commit of the staged proposals and the
|
|
1625
|
+
* self-update that must follow it (`update`, re-sequenced behind it, or a new
|
|
1626
|
+
* record): resume drives records in `seq` order, so the commit comes first.
|
|
1627
|
+
*/
|
|
1628
|
+
async function linkStagedAddsCommit(rt, update) {
|
|
1629
|
+
const parentEpoch = rt.engine.state.groupContext.epoch;
|
|
1630
|
+
const linkedRecords = await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1631
|
+
const view = tx.artifact('pendingIntents');
|
|
1632
|
+
const seq = await nextSeq(view);
|
|
1633
|
+
const first = { v: 1, intentId: randomIntentId(), seq, kind: 'commit', state: 'pending-publish', parentEpoch, intent: { type: 'commit' }, attempts: 0 };
|
|
1634
|
+
const linked = {
|
|
1635
|
+
...(update ?? { v: 1, intentId: randomIntentId(), kind: 'self-update', state: 'pending-publish', attempts: 0 }),
|
|
1636
|
+
seq: seq + 1,
|
|
1637
|
+
parentEpoch,
|
|
1638
|
+
intent: { type: 'self-update', after: first.intentId, ...(update?.intent.type === 'self-update' && update.intent.rekey !== undefined ? { rekey: update.intent.rekey } : {}) },
|
|
1639
|
+
};
|
|
1640
|
+
await writePendingIntentRecord(view, first);
|
|
1641
|
+
await writePendingIntentRecord(view, linked);
|
|
1642
|
+
return { first, linked };
|
|
1643
|
+
}));
|
|
1644
|
+
assertNotDisposed(rt);
|
|
1645
|
+
return linkedRecords;
|
|
1646
|
+
}
|
|
1647
|
+
/**
|
|
1648
|
+
* Where a self-update's `after` dependency stands. `waiting` while the commit
|
|
1649
|
+
* it follows is still a record. `terminal` when that record is gone and the
|
|
1650
|
+
* group is still at the self-update's parent epoch — the commit was dropped
|
|
1651
|
+
* without moving the epoch — and the self-update record is deleted here;
|
|
1652
|
+
* except that a re-enqueued self-update (AC-RACE-1) with budget left is
|
|
1653
|
+
* `requeued` instead: kept without `after`, one attempt spent, its drive
|
|
1654
|
+
* scheduled. Otherwise the commit (or a foreign one) moved the epoch on: the self-update
|
|
1655
|
+
* is re-parented to the current epoch without `after`, durably, and returned
|
|
1656
|
+
* to be driven like any self-update. A record without `after` is returned as is.
|
|
1657
|
+
*/
|
|
1658
|
+
async function resolveDependency(rt, record) {
|
|
1659
|
+
if (record.intent.type !== 'self-update' || record.intent.after === undefined)
|
|
1660
|
+
return record;
|
|
1661
|
+
if (await stillPending(rt, record.intent.after))
|
|
1662
|
+
return 'waiting';
|
|
1663
|
+
// The epoch below must be the durable one: an engine whose rebuild failed still holds an older state.
|
|
1664
|
+
if (rt.needsReload) {
|
|
1665
|
+
await rt.reloadFromDurable();
|
|
1666
|
+
rt.needsReload = false;
|
|
1667
|
+
}
|
|
1668
|
+
const epoch = rt.engine.state.groupContext.epoch;
|
|
1669
|
+
if (record.parentEpoch === epoch) {
|
|
1670
|
+
// AC-RACE-1: a re-enqueued self-update owes its rotation until the ORIGINAL's budget is spent — the Add commit
|
|
1671
|
+
// it followed going terminal costs it one attempt, never the rotation itself. It is kept, `after` dropped, for
|
|
1672
|
+
// the next drive (the two-commit path again while Adds stay staged, else standalone).
|
|
1673
|
+
if (record.reenqueuedFrom !== undefined && record.attempts + 1 < rt.ctx.maxPublishAttempts) {
|
|
1674
|
+
const attempts = record.attempts + 1;
|
|
1675
|
+
const kept = { ...record, intent: withoutAfter(record.intent), attempts, retryAt: rt.ctx.clock.now() + backoffMs(attempts) };
|
|
1676
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), kept)));
|
|
1677
|
+
assertNotDisposed(rt);
|
|
1678
|
+
armReenqueueRetry(rt, record.reenqueuedFrom, backoffMs(attempts));
|
|
1679
|
+
return 'requeued';
|
|
1680
|
+
}
|
|
1681
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deletePendingIntentRecord(tx.artifact('pendingIntents'), record.intentId)));
|
|
1682
|
+
return 'terminal';
|
|
1683
|
+
}
|
|
1684
|
+
const ready = { ...record, parentEpoch: epoch, intent: withoutAfter(record.intent) };
|
|
1685
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), ready)));
|
|
1686
|
+
assertNotDisposed(rt);
|
|
1687
|
+
return ready;
|
|
1688
|
+
}
|
|
1689
|
+
/** A self-update intent without its `after` dependency (resolved), keeping a rekey's target. */
|
|
1690
|
+
function withoutAfter(intent) {
|
|
1691
|
+
return intent.type === 'self-update' && intent.rekey !== undefined ? { type: 'self-update', rekey: intent.rekey } : { type: 'self-update' };
|
|
1692
|
+
}
|
|
1693
|
+
/**
|
|
1694
|
+
* After a background confirmation of own commit `intentId` (a publish or
|
|
1695
|
+
* confirm retry): a self-update waiting on it follows now rather than at the
|
|
1696
|
+
* group's next op. A failure here leaves the self-update to a background resume.
|
|
1697
|
+
*/
|
|
1698
|
+
async function followDependents(rt, intentId) {
|
|
1699
|
+
try {
|
|
1700
|
+
const records = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(rt.groupStore.artifact('pendingIntents')));
|
|
1701
|
+
if (records.some((r) => r.intentId === intentId))
|
|
1702
|
+
return;
|
|
1703
|
+
if (!records.some((r) => r.intent.type === 'self-update' && r.intent.after === intentId))
|
|
1704
|
+
return;
|
|
1705
|
+
await resumePending(rt, { unpublishedOnly: true });
|
|
1706
|
+
}
|
|
1707
|
+
catch (error) {
|
|
1708
|
+
if (!backgroundRetryEnds(rt, error))
|
|
1709
|
+
resumePendingInBackground(rt, 1);
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
// ---------------------------------------------------------------------------
|
|
1713
|
+
// S5 — re-enqueue of an own self-update withdrawn by a rewind (AC-RACE-1)
|
|
1714
|
+
// ---------------------------------------------------------------------------
|
|
1715
|
+
/**
|
|
1716
|
+
* A commit-kind record just confirmed (or went terminal), which is exactly the change a re-enqueued
|
|
1717
|
+
* self-update held back as "another own commit pending" (or as a later sibling) waits on. The queue drain only
|
|
1718
|
+
* resumes pending records while `queuedHint` says something may be queued, so each still-envelope-less
|
|
1719
|
+
* `reenqueuedFrom` record other than the one that just settled is re-evaluated directly here.
|
|
1720
|
+
*/
|
|
1721
|
+
async function retryDeferredSelfUpdateReenqueues(rt, justSettledIntentId) {
|
|
1722
|
+
const view = rt.groupStore.artifact('pendingIntents');
|
|
1723
|
+
const records = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(view));
|
|
1724
|
+
for (const r of records) {
|
|
1725
|
+
if (r.intentId === justSettledIntentId || r.reenqueuedFrom === undefined || r.envelope !== undefined)
|
|
1726
|
+
continue;
|
|
1727
|
+
// Both families wait for each other (DD1): a settled self-update frees the next commit re-drive and vice versa.
|
|
1728
|
+
if (isCommitRedrive(r))
|
|
1729
|
+
scheduleCommitRedrive(rt, r.reenqueuedFrom);
|
|
1730
|
+
else
|
|
1731
|
+
scheduleSelfUpdateReenqueue(rt, r.reenqueuedFrom);
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
/**
|
|
1735
|
+
* Runs {@link retryDeferredSelfUpdateReenqueues} as its own detached group op (S3 `detached-dispatch-rule`) —
|
|
1736
|
+
* never awaited by the settlement that triggers it: a confirmation or terminal failure is already durable, and
|
|
1737
|
+
* its own follow-up work (routing, resync, Welcome delivery, the queue drain, the caller's result) must not
|
|
1738
|
+
* depend on this discovery's store read. A failed discovery leaves the deferred records durable; the queue
|
|
1739
|
+
* drain is re-armed on its capped backoff (`queuedHint` cleared), which resumes them through the same rules.
|
|
1740
|
+
*/
|
|
1741
|
+
export function scheduleDeferredReenqueueRetry(rt, justSettledIntentId) {
|
|
1742
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
1743
|
+
return;
|
|
1744
|
+
void Promise.resolve()
|
|
1745
|
+
.then(() => rt.guard.detached(() => rt.run('selfUpdateReenqueueRetry', (rtc) => retryDeferredSelfUpdateReenqueues(rtc, justSettledIntentId))))
|
|
1746
|
+
.catch((error) => {
|
|
1747
|
+
if (backgroundRetryEnds(rt, error))
|
|
1748
|
+
return;
|
|
1749
|
+
rt.queuedHint = undefined;
|
|
1750
|
+
rt.scheduleQueueDrainRetry();
|
|
1751
|
+
});
|
|
1752
|
+
}
|
|
1753
|
+
/**
|
|
1754
|
+
* A re-enqueued self-update (AC-RACE-1) is re-parented to the current epoch rather than failed as stale: it
|
|
1755
|
+
* carries no epoch-specific payload, and the epoch it was recorded at routinely moves before it is driven (a
|
|
1756
|
+
* sibling re-enqueue confirming first, a foreign commit ingested meanwhile) — the same move `resolveDependency`
|
|
1757
|
+
* makes for the S13 self-update-after-a-commit chain. Only an envelope-less record without an `after`
|
|
1758
|
+
* dependency qualifies (a minted envelope is bound to its epoch; `after` is `resolveDependency`'s to resolve).
|
|
1759
|
+
*/
|
|
1760
|
+
async function reparentReenqueued(rt, record) {
|
|
1761
|
+
const epoch = rt.engine.state.groupContext.epoch;
|
|
1762
|
+
if (record.reenqueuedFrom === undefined || record.envelope !== undefined || record.intent.type !== 'self-update' || record.intent.after !== undefined)
|
|
1763
|
+
return record;
|
|
1764
|
+
if (record.parentEpoch === epoch)
|
|
1765
|
+
return record;
|
|
1766
|
+
const ready = { ...record, parentEpoch: epoch };
|
|
1767
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writePendingIntentRecord(tx.artifact('pendingIntents'), ready)));
|
|
1768
|
+
assertNotDisposed(rt);
|
|
1769
|
+
return ready;
|
|
1770
|
+
}
|
|
1771
|
+
/**
|
|
1772
|
+
* A re-enqueued self-update (AC-RACE-1) whose envelope was minted at an epoch a foreign commit has since moved
|
|
1773
|
+
* the group past (its publish failed and was waiting on a retry). Its envelope can never be confirmed, but the
|
|
1774
|
+
* rotation it stands for is still owed — unlike an ordinary own commit, it is not failed as stale. While the
|
|
1775
|
+
* original's `maxPublishAttempts` budget lasts, the stale envelope is ledgered as a withdrawn rotation of the
|
|
1776
|
+
* original (so a later fork choice landing on it is recognised by `cancelRestoredReenqueues`), the record is put
|
|
1777
|
+
* back envelope-less with one attempt spent, and its drive is scheduled to re-mint it at the tip. (A rotation of
|
|
1778
|
+
* the same original that became canonical again never reaches here: the restoring transition already deleted this
|
|
1779
|
+
* record — see {@link driveReenqueuedSelfUpdate}.)
|
|
1780
|
+
*
|
|
1781
|
+
* Returns `'done'` when handled, `'fenced'` (fence / removal: the caller fails it at once, without probing) or
|
|
1782
|
+
* `'budget'` (the budget is spent: the caller fails it terminally, once, under the original id, after one more stale
|
|
1783
|
+
* probe that may still retain a stored commit). Only ever reached without a caller — a re-enqueue is driven in the
|
|
1784
|
+
* background.
|
|
1785
|
+
*
|
|
1786
|
+
* The stale envelope's commit is retained as a fork node of the rewind window (staleOwnCommitWindow), so if a relay
|
|
1787
|
+
* stored it despite the failed publish (a probe finds it; a timeout spends an attempt and retries, bounded by the
|
|
1788
|
+
* budget — see staleProbe) and other members extend it, this client can follow that branch, and the withdrawn ledger
|
|
1789
|
+
* entry written here names that node. If the probe found the envelope absent the entry is written as before but
|
|
1790
|
+
* names no node of the window.
|
|
1791
|
+
*/
|
|
1792
|
+
async function remintStaleReenqueue(rt, record, fromRetryTimer) {
|
|
1793
|
+
const root = record.reenqueuedFrom;
|
|
1794
|
+
// Fenced, or removed for good (no longer revisable): nothing can ever be minted again — fail it now, no probe.
|
|
1795
|
+
if (rt.fence !== undefined || (rt.engine.state.groupActiveState.kind === 'removedFromGroup' && rt.removal === undefined))
|
|
1796
|
+
return 'fenced';
|
|
1797
|
+
const attempts = record.attempts + 1;
|
|
1798
|
+
if (attempts >= rt.ctx.maxPublishAttempts)
|
|
1799
|
+
return 'budget';
|
|
1800
|
+
const envelope = record.envelope;
|
|
1801
|
+
const node = record.pending === undefined ? undefined : tagOf(deserializeClientState(record.pending.newState));
|
|
1802
|
+
let retained = staleOwnCommitWindow(rt, record);
|
|
1803
|
+
if (retained !== undefined) {
|
|
1804
|
+
const found = await staleProbe(rt, record, envelope, fromRetryTimer);
|
|
1805
|
+
if (found === 'retry')
|
|
1806
|
+
return 'done';
|
|
1807
|
+
// The probe budget cannot run out here: the pre-check above fails first at the same ceiling.
|
|
1808
|
+
if (found === 'absent')
|
|
1809
|
+
retained = undefined;
|
|
1810
|
+
}
|
|
1811
|
+
const { envelope: _stale, pending: _pending, ...rest } = record;
|
|
1812
|
+
const remint = { ...rest, state: 'pending-publish', parentEpoch: rt.engine.state.groupContext.epoch, attempts, retryAt: rt.ctx.clock.now() + backoffMs(attempts) };
|
|
1813
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1814
|
+
// Stored case only (`retained` set): the ledger handle below names a node the engine can build. When the probe
|
|
1815
|
+
// found the envelope absent, the handle is the pre-existing one and names no window node.
|
|
1816
|
+
if (retained !== undefined)
|
|
1817
|
+
await writeWindow(tx.artifact('rewindTree'), retained);
|
|
1818
|
+
if (node !== undefined) {
|
|
1819
|
+
await writeTerminalLedgerEntry(tx.artifact('terminalLedger'), envelope.id, {
|
|
1820
|
+
disposition: WITHDRAWN_DISPOSITION,
|
|
1821
|
+
at: rt.ctx.clock.now(),
|
|
1822
|
+
createdAt: envelope.created_at,
|
|
1823
|
+
node,
|
|
1824
|
+
selfUpdateIntentId: root,
|
|
1825
|
+
attempts: record.attempts,
|
|
1826
|
+
...rekeyOf(record),
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
await writePendingIntentRecord(tx.artifact('pendingIntents'), remint);
|
|
1830
|
+
}));
|
|
1831
|
+
assertNotDisposed(rt);
|
|
1832
|
+
if (retained !== undefined)
|
|
1833
|
+
await adoptRetainedWindow(rt);
|
|
1834
|
+
armReenqueueRetry(rt, root, backoffMs(attempts));
|
|
1835
|
+
return 'done';
|
|
1836
|
+
}
|
|
1837
|
+
/**
|
|
1838
|
+
* Ends the commit re-drive `record` (already deleted by the caller) because the publish budget carried across the re-drives of its
|
|
1839
|
+
* root is spent (DD9, AC-DRIVE-4): every change that has no verdict is lost `attempts-exhausted`, and `intentFailed` fires once under
|
|
1840
|
+
* the root id. The verdicts this run decided before it saw the budget are signalled first; all of them are outbox terminals written in the transaction that deleted the record.
|
|
1841
|
+
*/
|
|
1842
|
+
async function endRedriveExhausted(rt, record) {
|
|
1843
|
+
// The deleting transaction wrote the terminals (`deleteEndingRecord`: `newlyLost` with their reasons, then `attempts-exhausted` for the changes
|
|
1844
|
+
// still owed); terminalFail hands them over, releases the queue's commit hold and fires the one intentFailed.
|
|
1845
|
+
await terminalFail(rt, record, false, new Error('the publish budget carried across the re-drives of this intent is spent'), { reason: 'max-attempts', attempts: record.attempts });
|
|
1846
|
+
}
|
|
1847
|
+
/**
|
|
1848
|
+
* A commit re-drive that was published (pending-confirm) or staged and awaits its publish retry (an envelope), and whose parent
|
|
1849
|
+
* epoch has since left the canonical path (AC-DRIVE-8, DD1). No marked node of its root is canonical (a restoring transition would
|
|
1850
|
+
* have retired it, cancelRestoredReenqueues), so its changes are still owed: the envelope is discarded and the record goes back to
|
|
1851
|
+
* an envelope-less `pending-publish` one at the tip, one attempt of the carried budget spent, and its drive is scheduled at once
|
|
1852
|
+
* -- which re-runs every build-time check (a change that no longer passes is lost with its reason). The stale-settle path
|
|
1853
|
+
* that fails the record `intentFailed` never runs for it.
|
|
1854
|
+
*
|
|
1855
|
+
* The discarded envelope may still be held by a relay (a lost ACK; a pending-confirm one was ACKed), and peers may extend it. Like
|
|
1856
|
+
* {@link remintStaleReenqueue} it is therefore probed (`confirmed`: no probe, a relay accepted it) and, when stored, retained in the SAME
|
|
1857
|
+
* transaction as the re-mint: the commit as a fork node of the rewind window (so a redelivery is a duplicate, not a foreign replay of
|
|
1858
|
+
* our own leaf, and fork choice can follow a branch built on it), a `withdrawn` ledger entry for its event id, and a marker on that node
|
|
1859
|
+
* carrying the root and the surviving changes. If that branch wins, the DD14 restore finds the marker: the re-mint record is retired
|
|
1860
|
+
* and each change gets one `own-change-reapplied` naming this commit. An absent envelope is dropped. A probe that cannot tell keeps
|
|
1861
|
+
* the record and retries (`'retry'`); one that stays silent past the budget counts as absent.
|
|
1862
|
+
*/
|
|
1863
|
+
async function remintStaleRedrive(rt, record, fromRetryTimer = false, confirmed = false) {
|
|
1864
|
+
const root = record.reenqueuedFrom;
|
|
1865
|
+
// Fenced, or removed for good: nothing can ever be minted again (DD11 stranded rung).
|
|
1866
|
+
if (rt.fence !== undefined || (rt.engine.state.groupActiveState.kind === 'removedFromGroup' && rt.removal === undefined)) {
|
|
1867
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deleteEndingRecord(tx, record, 'stranded')));
|
|
1868
|
+
assertNotDisposed(rt);
|
|
1869
|
+
await reportStrandedSelfUpdateReenqueues(rt, [record], rt.fence ?? new Error(`group ${rt.groupId}'s removal of the local leaf settled for good`));
|
|
1870
|
+
return;
|
|
1871
|
+
}
|
|
1872
|
+
const attempts = record.attempts + 1;
|
|
1873
|
+
const exhausted = attempts >= rt.ctx.maxPublishAttempts;
|
|
1874
|
+
const envelope = record.envelope;
|
|
1875
|
+
let retained = envelope === undefined ? undefined : staleOwnCommitWindow(rt, record);
|
|
1876
|
+
if (retained !== undefined && !confirmed) {
|
|
1877
|
+
const found = await staleProbe(rt, record, envelope, fromRetryTimer);
|
|
1878
|
+
if (found === 'retry')
|
|
1879
|
+
return;
|
|
1880
|
+
if (found !== 'stored')
|
|
1881
|
+
retained = undefined; // absent, or unverified after the probe budget
|
|
1882
|
+
}
|
|
1883
|
+
const node = record.pending === undefined ? undefined : tagOf(deserializeClientState(record.pending.newState));
|
|
1884
|
+
// The staged commit's own pieces (envelope, pending state, Welcomes, ordering) belong to the discarded commit.
|
|
1885
|
+
// A retained commit can still win (peers extend it), and its Welcomes live only on this record until its confirmation: parked as an
|
|
1886
|
+
// already-withdrawn invite so a restore of that node delivers them (parkWithdrawnWelcomes). Before the delete, so a failure keeps the record.
|
|
1887
|
+
// Residual (S10's AC-REC-4, now with an invite consequence): a re-drive still only awaiting its publish retry that a DD14 restore retires is
|
|
1888
|
+
// neither probed, retained nor parked -- if a relay stored it after all and peers extend it, the invitee has no Welcome for it.
|
|
1889
|
+
if (retained !== undefined && envelope !== undefined && node !== undefined)
|
|
1890
|
+
await parkWithdrawnWelcomes(rt, record);
|
|
1891
|
+
const { envelope: _envelope, pending: _pending, welcomes: _welcomes, ordering: _ordering, ...rest } = record;
|
|
1892
|
+
const remint = { ...rest, state: 'pending-publish', parentEpoch: rt.engine.state.groupContext.epoch, attempts };
|
|
1893
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
1894
|
+
if (retained !== undefined && envelope !== undefined && node !== undefined) {
|
|
1895
|
+
await writeWindow(tx.artifact('rewindTree'), retained);
|
|
1896
|
+
await writeTerminalLedgerEntry(tx.artifact('terminalLedger'), envelope.id, { disposition: WITHDRAWN_DISPOSITION, at: rt.ctx.clock.now(), createdAt: envelope.created_at, node, ownCommitRoot: root });
|
|
1897
|
+
await writeOwnCommitMarker(tx.extension(OWN_COMMIT_EXTENSION), envelope.id, ownCommitMarkerOf(record, node));
|
|
1898
|
+
}
|
|
1899
|
+
if (exhausted)
|
|
1900
|
+
await deleteEndingRecord(tx, { ...record, attempts }, 'attempts-exhausted');
|
|
1901
|
+
else
|
|
1902
|
+
await writePendingIntentRecord(tx.artifact('pendingIntents'), remint);
|
|
1903
|
+
}));
|
|
1904
|
+
assertNotDisposed(rt);
|
|
1905
|
+
if (retained !== undefined)
|
|
1906
|
+
await adoptRetainedWindow(rt);
|
|
1907
|
+
if (exhausted)
|
|
1908
|
+
await endRedriveExhausted(rt, { ...record, attempts });
|
|
1909
|
+
else
|
|
1910
|
+
scheduleCommitRedrive(rt, root);
|
|
1911
|
+
}
|
|
1912
|
+
/**
|
|
1913
|
+
* The rules both re-enqueue families share (a self-update re-enqueue, AC-RACE-1, and a commit re-drive, epic
|
|
1914
|
+
* marmot-rewound-own-commit-no-signal DD1): whether the record of `root` may be built NOW. Either the disposition the
|
|
1915
|
+
* evaluation ended with, or the record to build.
|
|
1916
|
+
*
|
|
1917
|
+
* 1. the AC-RACE-3 test seam stops here; a fenced or finally removed group fails its stranded records of both families; a
|
|
1918
|
+
* revisable removal holds them (a winning sibling may restore the leaf);
|
|
1919
|
+
* 2. nothing is minted before the S18 send gate opens;
|
|
1920
|
+
* 3. a record that already has an envelope (or is pending-confirm) belongs to its own retry/confirmation path;
|
|
1921
|
+
* 4. a kept attempt waits out its backoff;
|
|
1922
|
+
* 5. another own commit-kind record that has actually started blocks it, and among the envelope-less records of BOTH
|
|
1923
|
+
* families only the lowest `seq` (withdrawal order) drives; the rest wait for `retryDeferredSelfUpdateReenqueues`.
|
|
1924
|
+
*/
|
|
1925
|
+
async function reenqueueTurn(rt, root, redrive) {
|
|
1926
|
+
if (rt.ctx.disableSelfUpdateReenqueueForTest)
|
|
1927
|
+
return 'absent';
|
|
1928
|
+
if (rt.fence !== undefined) {
|
|
1929
|
+
await failStrandedSelfUpdateReenqueues(rt, rt.fence);
|
|
1930
|
+
return 'absent';
|
|
1931
|
+
}
|
|
1932
|
+
if (rt.engine.state.groupActiveState.kind === 'removedFromGroup') {
|
|
1933
|
+
if (rt.removal === undefined) {
|
|
1934
|
+
await failStrandedSelfUpdateReenqueues(rt, new Error(`group ${rt.groupId}'s removal of the local leaf settled for good`));
|
|
1935
|
+
return 'absent';
|
|
1936
|
+
}
|
|
1937
|
+
// Revisable (`rt.removal` held): a winning sibling may still bring the leaf back. Held like the election wait —
|
|
1938
|
+
// the ingest that restores the membership (or settles the removal) drains the queue, which must not find a
|
|
1939
|
+
// stale "nothing queued" hint.
|
|
1940
|
+
rt.queuedHint = undefined;
|
|
1941
|
+
return 'held';
|
|
1942
|
+
}
|
|
1943
|
+
if (!rt.sendGateOpen)
|
|
1944
|
+
return 'waiting';
|
|
1945
|
+
const pendingView = rt.groupStore.artifact('pendingIntents');
|
|
1946
|
+
const remaining = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(pendingView));
|
|
1947
|
+
// A drive answers for the records of its own family only.
|
|
1948
|
+
const tagged = remaining.find((r) => r.reenqueuedFrom === root && isCommitRedrive(r) === redrive);
|
|
1949
|
+
if (tagged === undefined)
|
|
1950
|
+
return 'absent';
|
|
1951
|
+
if (tagged.envelope !== undefined || tagged.state === 'pending-confirm')
|
|
1952
|
+
return 'started';
|
|
1953
|
+
// A kept attempt waits out its backoff whatever triggered this evaluation (a sibling settling, a drain, a load):
|
|
1954
|
+
// the timer re-arms for what is left (`early-timer-rearm-pattern`; also after a reload, which lost the timer).
|
|
1955
|
+
const now = rt.ctx.clock.now();
|
|
1956
|
+
if (tagged.retryAt !== undefined && tagged.retryAt > now) {
|
|
1957
|
+
armReenqueueRetry(rt, root, tagged.retryAt - now, redrive);
|
|
1958
|
+
return 'waiting';
|
|
1959
|
+
}
|
|
1960
|
+
const started = (r) => isCommitKind(r.kind) && (r.envelope !== undefined || r.state === 'pending-confirm');
|
|
1961
|
+
if (remaining.some((r) => r.intentId !== tagged.intentId && started(r)))
|
|
1962
|
+
return 'waiting';
|
|
1963
|
+
const siblings = remaining.filter((r) => r.reenqueuedFrom !== undefined && r.envelope === undefined && r.state !== 'pending-confirm');
|
|
1964
|
+
const leader = siblings.reduce((lowest, r) => (r.seq < lowest.seq ? r : lowest), tagged);
|
|
1965
|
+
if (leader.intentId !== tagged.intentId)
|
|
1966
|
+
return 'waiting';
|
|
1967
|
+
return tagged;
|
|
1968
|
+
}
|
|
1969
|
+
/**
|
|
1970
|
+
* The one set of rules for driving an envelope-less self-update a rewind re-enqueued (AC-RACE-1,
|
|
1971
|
+
* `own-self-update-reenqueue-on-rewind`). Rules 1-4 are {@link reenqueueTurn}'s, shared with the commit re-drive
|
|
1972
|
+
* ({@link driveCommitRedrive}); 5-7 are this drive's own. Every path that reaches such a record — the detached drive scheduled at
|
|
1973
|
+
* the withdrawal, a confirmation or terminal failure re-evaluating deferred siblings, the queue drain, the load-time
|
|
1974
|
+
* and post-sync resumes (`resumePending`) — goes through here, so the rules cannot diverge between them:
|
|
1975
|
+
*
|
|
1976
|
+
* 1. the AC-RACE-3 test seam (`disableSelfUpdateReenqueueForTest`) stops here; a fenced or finally removed group
|
|
1977
|
+
* fails its stranded re-enqueues; a revisable removal holds them (a winning sibling may restore the leaf);
|
|
1978
|
+
* 2. nothing is minted before the S18 send gate opens (`releaseAwaitingSync` resumes it);
|
|
1979
|
+
* 3. a record that already has an envelope (or is pending-confirm) belongs to its own retry/confirmation path —
|
|
1980
|
+
* driving it again would mint a second commit at the same epoch;
|
|
1981
|
+
* 4. another own commit-kind record that has actually started (an envelope, or pending-confirm) blocks it, and
|
|
1982
|
+
* among envelope-less re-enqueue siblings only the lowest `seq` (withdrawal order) drives — the rest wait for
|
|
1983
|
+
* `retryDeferredSelfUpdateReenqueues` after it settles (siblings never block each other otherwise);
|
|
1984
|
+
* 5. an S13 `after` dependency is resolved first (`resolveDependency`);
|
|
1985
|
+
* 6. the AC-RACE-2 election wait and a deferred proposal hold it, clearing `queuedHint` so the release (the wait's
|
|
1986
|
+
* timer, a staged-set change) drains for real (`queue-drain-queuedHint-gate`);
|
|
1987
|
+
* 7. otherwise it is driven, re-parented to the current epoch (`reparentReenqueued`, inside `driveIntent`), which
|
|
1988
|
+
* routes a staged-Adds tip through the S13 two-commit path on its own.
|
|
1989
|
+
*
|
|
1990
|
+
* There is no drive-time "is the original rotation canonical again?" check: the restoring transition already
|
|
1991
|
+
* cancelled such a record atomically (group-history.ts `cancelRestoredReenqueues`, over the transition's whole
|
|
1992
|
+
* canonical path), because (a) the canonical path (`rt.window`'s tip) only ever changes through `recordTransition` —
|
|
1993
|
+
* every writer that moves the tip writes the window in the same transaction as `recordTransition`, the other window
|
|
1994
|
+
* writers only add fork nodes or re-snapshot nodes with the tip unchanged (pinned by
|
|
1995
|
+
* test/client/write-window-sites.test.ts), and groups.restore/create start from a cleared store; and (b) every other
|
|
1996
|
+
* writer of a ledger entry's `selfUpdateIntentId` names a non-canonical node or deletes the root's only record in the
|
|
1997
|
+
* same transaction. A pending-confirm record the first tier keeps is resolved by `confirmRecord`, never driven here.
|
|
1998
|
+
*/
|
|
1999
|
+
async function driveReenqueuedSelfUpdate(rt, originalIntentId) {
|
|
2000
|
+
const turn = await reenqueueTurn(rt, originalIntentId, false);
|
|
2001
|
+
if (typeof turn === 'string')
|
|
2002
|
+
return turn;
|
|
2003
|
+
const tagged = turn;
|
|
2004
|
+
const resolved = await resolveDependency(rt, tagged);
|
|
2005
|
+
if (resolved === 'waiting' || resolved === 'requeued')
|
|
2006
|
+
return 'waiting';
|
|
2007
|
+
if (resolved === 'terminal') {
|
|
2008
|
+
const after = tagged.intent.type === 'self-update' ? tagged.intent.after : undefined;
|
|
2009
|
+
await terminalFail(rt, tagged, false, new Error('the commit of the staged proposals it follows went terminal'), { reason: 'dependency-terminal', after });
|
|
2010
|
+
return 'driven';
|
|
2011
|
+
}
|
|
2012
|
+
if (awaitingElectedSelfRemove(rt)) {
|
|
2013
|
+
rt.queuedHint = undefined;
|
|
2014
|
+
return 'held';
|
|
2015
|
+
}
|
|
2016
|
+
const deferred = await decideStagedProposals(rt);
|
|
2017
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
2018
|
+
return 'absent';
|
|
2019
|
+
if (deferred > 0) {
|
|
2020
|
+
rt.queuedHint = undefined;
|
|
2021
|
+
return 'held';
|
|
2022
|
+
}
|
|
2023
|
+
await driveIntent(rt, resolved, false);
|
|
2024
|
+
return 'driven';
|
|
2025
|
+
}
|
|
2026
|
+
/**
|
|
2027
|
+
* Drives the re-enqueue of `originalIntentId` again after `delayMs` (clamped to the platform timer limit): a
|
|
2028
|
+
* pipeline timer (tracked in `rt.pipelineTimers`, cancelled on dispose, its body detached) that calls
|
|
2029
|
+
* {@link scheduleSelfUpdateReenqueue}. At most one is pending per original; the record itself stays durable, so a
|
|
2030
|
+
* load resumes it (and re-arms for whatever is left of its `retryAt`).
|
|
2031
|
+
*/
|
|
2032
|
+
function armReenqueueRetry(rt, originalIntentId, delayMs, redrive = false) {
|
|
2033
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
2034
|
+
return;
|
|
2035
|
+
const armed = rt.reenqueueRetryArmed;
|
|
2036
|
+
if (armed.has(originalIntentId))
|
|
2037
|
+
return;
|
|
2038
|
+
armed.add(originalIntentId);
|
|
2039
|
+
setPipelineTimer(rt, () => {
|
|
2040
|
+
armed.delete(originalIntentId);
|
|
2041
|
+
if (redrive)
|
|
2042
|
+
scheduleCommitRedrive(rt, originalIntentId);
|
|
2043
|
+
else
|
|
2044
|
+
scheduleSelfUpdateReenqueue(rt, originalIntentId);
|
|
2045
|
+
}, Math.min(MAX_TIMER_DELAY_MS, Math.max(1, delayMs)));
|
|
2046
|
+
}
|
|
2047
|
+
/**
|
|
2048
|
+
* Schedules {@link driveReenqueuedSelfUpdate} detached (S3 `detached-dispatch-rule`) — outside the ingest
|
|
2049
|
+
* batch or confirmation frame that discovered the withdrawal (reachable from inside a device op's own commit,
|
|
2050
|
+
* via `confirmRecord`), as its own group op. Never throws. A failure before anything was minted (a store or
|
|
2051
|
+
* signer fault) leaves the record durable and envelope-less; nothing else may come along to drive it, so the
|
|
2052
|
+
* queue drain is re-armed (`queuedHint` cleared) on its capped backoff, which resumes it through the same rules.
|
|
2053
|
+
*/
|
|
2054
|
+
export function scheduleSelfUpdateReenqueue(rt, originalIntentId) {
|
|
2055
|
+
if (rt.disposed || rt.ctx.isDisposed() || rt.ctx.disableSelfUpdateReenqueueForTest)
|
|
2056
|
+
return;
|
|
2057
|
+
void Promise.resolve()
|
|
2058
|
+
.then(() => rt.guard.detached(() => rt.run('selfUpdateReenqueue', (rtc) => driveReenqueuedSelfUpdate(rtc, originalIntentId))))
|
|
2059
|
+
.catch((error) => {
|
|
2060
|
+
if (backgroundRetryEnds(rt, error))
|
|
2061
|
+
return;
|
|
2062
|
+
rt.queuedHint = undefined;
|
|
2063
|
+
rt.scheduleQueueDrainRetry();
|
|
2064
|
+
});
|
|
2065
|
+
}
|
|
2066
|
+
/**
|
|
2067
|
+
* Schedules the drives of everything a transition withdrew: each withdrawn self-update (AC-RACE-1) and each withdrawn own
|
|
2068
|
+
* commit (epic marmot-rewound-own-commit-no-signal, DD1) — detached, after the transition's transaction committed. The one
|
|
2069
|
+
* call site for the transition producers (ingest, a confirmation, a branch switch).
|
|
2070
|
+
*/
|
|
2071
|
+
export function scheduleWithdrawnRedrives(rt, plan) {
|
|
2072
|
+
for (const w of plan?.withdrawnSelfUpdates ?? [])
|
|
2073
|
+
scheduleSelfUpdateReenqueue(rt, w.originalIntentId);
|
|
2074
|
+
for (const w of plan?.withdrawnOwnCommits ?? [])
|
|
2075
|
+
scheduleCommitRedrive(rt, w.rootIntentId);
|
|
2076
|
+
}
|
|
2077
|
+
/**
|
|
2078
|
+
* Schedules {@link driveCommitRedrive} detached, exactly as {@link scheduleSelfUpdateReenqueue} does for a self-update
|
|
2079
|
+
* (never throws; a failure before anything was minted leaves the record durable and re-arms the queue drain).
|
|
2080
|
+
*/
|
|
2081
|
+
export function scheduleCommitRedrive(rt, rootIntentId) {
|
|
2082
|
+
if (rt.disposed || rt.ctx.isDisposed() || rt.ctx.disableSelfUpdateReenqueueForTest)
|
|
2083
|
+
return;
|
|
2084
|
+
void Promise.resolve()
|
|
2085
|
+
.then(() => rt.guard.detached(() => rt.run('commitRedrive', (rtc) => driveCommitRedrive(rtc, rootIntentId))))
|
|
2086
|
+
.catch((error) => {
|
|
2087
|
+
if (backgroundRetryEnds(rt, error))
|
|
2088
|
+
return;
|
|
2089
|
+
rt.queuedHint = undefined;
|
|
2090
|
+
rt.scheduleQueueDrainRetry();
|
|
2091
|
+
});
|
|
2092
|
+
}
|
|
2093
|
+
/** How long a re-drive whose KeyPackage check met a failing consumer verifier waits before it is driven again (ms). */
|
|
2094
|
+
const VERIFIER_RETRY_MS = 5_000;
|
|
2095
|
+
/** Holds a throwing consumer verifier caused, per runtime and root intent id (in memory: a reload starts the count again, see decideRedrive). */
|
|
2096
|
+
const verifierHolds = new WeakMap();
|
|
2097
|
+
/**
|
|
2098
|
+
* The refs of the KeyPackages of `checks` (the invite changes the ladder would reach the validity rung for, see
|
|
2099
|
+
* `changesNeedingKeyPackageCheck`) that the invitations path would no longer accept (AC-INV-4, DD11): each event goes through the
|
|
2100
|
+
* very check an `invite()` call makes (`validateInvitees`: the consumer verifier, `selectKeyPackage` on the client clock -- so an
|
|
2101
|
+
* expired one -- and the group's own requirements), alone, so one bad KeyPackage never hides another. A stored event the check
|
|
2102
|
+
* cannot even read counts as invalid: it is classified, never trusted.
|
|
2103
|
+
*
|
|
2104
|
+
* A verifier that THROWS is not a verdict (`validateInvitees` would fold it into "rejected"): it is a transient failure of the
|
|
2105
|
+
* consumer's callback, reported as `'transient'` so the drive holds with no verdict and no publish spent. A verifier answering
|
|
2106
|
+
* `false` is a rejection and loses the change.
|
|
2107
|
+
*/
|
|
2108
|
+
async function invalidRetainedKeyPackages(rt, checks, state) {
|
|
2109
|
+
const verifier = rt.ctx.verifier;
|
|
2110
|
+
const invalid = new Set();
|
|
2111
|
+
for (const stored of checks) {
|
|
2112
|
+
const { event, inviteeRef } = stored.keyPackage;
|
|
2113
|
+
let verifierThrew = false;
|
|
2114
|
+
try {
|
|
2115
|
+
await validateInvitees([event], {
|
|
2116
|
+
verify: verifier === undefined
|
|
2117
|
+
? undefined
|
|
2118
|
+
: (e) => rt.guard.call(rt.groupId, 'verifier', () => verifier(e)).catch((error) => {
|
|
2119
|
+
if (rt.disposed || rt.ctx.isDisposed() || isMarmotError(error, 'disposed'))
|
|
2120
|
+
throw error;
|
|
2121
|
+
verifierThrew = true;
|
|
2122
|
+
return false;
|
|
2123
|
+
}),
|
|
2124
|
+
nowSeconds: Math.floor(rt.ctx.clock.now() / 1000),
|
|
2125
|
+
state,
|
|
2126
|
+
});
|
|
2127
|
+
}
|
|
2128
|
+
catch (error) {
|
|
2129
|
+
if (rt.disposed || rt.ctx.isDisposed() || isMarmotError(error, 'disposed'))
|
|
2130
|
+
throw error;
|
|
2131
|
+
if (verifierThrew)
|
|
2132
|
+
return 'transient';
|
|
2133
|
+
invalid.add(inviteeRef);
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
return invalid;
|
|
2137
|
+
}
|
|
2138
|
+
/**
|
|
2139
|
+
* The DD11 ladder over one commit re-drive record, on the current tip (see commit-redrive.ts for the rungs): the stranded
|
|
2140
|
+
* rung is {@link reenqueueTurn}'s; not-admin is decided here on the author's ACCOUNT (a rotated leaf key does not matter,
|
|
2141
|
+
* AC-DRIVE-9) and the rest per change by kind. With no survivor nothing is published: the record is deleted and the lost
|
|
2142
|
+
* verdicts are signalled (AC-COMP-5). Otherwise the record to mint is returned; its verdicts become durable in the
|
|
2143
|
+
* staging transaction `driveIntent` runs, and are signalled after it.
|
|
2144
|
+
*/
|
|
2145
|
+
async function decideRedrive(rt, record) {
|
|
2146
|
+
// The state below must be the durable one: an engine whose rebuild failed still holds an older state.
|
|
2147
|
+
if (rt.needsReload) {
|
|
2148
|
+
await rt.reloadFromDurable();
|
|
2149
|
+
rt.needsReload = false;
|
|
2150
|
+
}
|
|
2151
|
+
const author = await callSignerForGroup(rt, () => rt.ctx.signer.getPublicKey());
|
|
2152
|
+
assertNotDisposed(rt);
|
|
2153
|
+
const state = rt.engine.state;
|
|
2154
|
+
const adminView = getMarmotGroupView(state);
|
|
2155
|
+
const authorIsAdmin = adminView !== null && isActiveAdmin(state.ratchetTree, adminView.adminPubkeys, author);
|
|
2156
|
+
const changes = record.changes ?? [];
|
|
2157
|
+
const base = { state, window: rt.window, windowTip: rt.exposedState, authorIsAdmin, forkEpoch: record.forkEpoch ?? 0 };
|
|
2158
|
+
// Only the invites the ladder would reach the validity rung for are checked (a consumer verifier is never called for a change lost anyway).
|
|
2159
|
+
const checks = changesNeedingKeyPackageCheck(changes, base);
|
|
2160
|
+
let invalidKeyPackages = await invalidRetainedKeyPackages(rt, checks, state);
|
|
2161
|
+
assertNotDisposed(rt);
|
|
2162
|
+
const root = record.reenqueuedFrom;
|
|
2163
|
+
if (invalidKeyPackages === 'transient') {
|
|
2164
|
+
// A failing consumer verifier decided nothing: no verdict, no publish spent, the record stays durable and is driven again later. A held
|
|
2165
|
+
// commit-kind record blocks every own commit of the group (commit-pending), so the hold is bounded: after `maxPublishAttempts` holds of
|
|
2166
|
+
// this root the checked KeyPackages count as not accepted (one `invalid-keypackage` verdict each, through the normal outbox path).
|
|
2167
|
+
// The count lives in memory, per runtime: a reload starts it again (a persistently failing verifier is then held another bounded round).
|
|
2168
|
+
const holds = (verifierHolds.get(rt)?.get(root) ?? 0) + 1;
|
|
2169
|
+
if (holds < rt.ctx.maxPublishAttempts) {
|
|
2170
|
+
const perRoot = verifierHolds.get(rt) ?? new Map();
|
|
2171
|
+
perRoot.set(root, holds);
|
|
2172
|
+
verifierHolds.set(rt, perRoot);
|
|
2173
|
+
rt.queuedHint = undefined;
|
|
2174
|
+
armReenqueueRetry(rt, root, VERIFIER_RETRY_MS, true);
|
|
2175
|
+
return 'held';
|
|
2176
|
+
}
|
|
2177
|
+
invalidKeyPackages = new Set(checks.map((c) => c.keyPackage.inviteeRef));
|
|
2178
|
+
}
|
|
2179
|
+
verifierHolds.get(rt)?.delete(root);
|
|
2180
|
+
const outcome = decideChanges(changes, { ...base, invalidKeyPackages });
|
|
2181
|
+
if (outcome.survivors.length === 0) {
|
|
2182
|
+
// Every change has a verdict now: this run's (`newlyLost`) are the terminals, written with the delete (S11).
|
|
2183
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deleteEndingRecord(tx, { ...record, changes: outcome.changes }, 'concurrent-change', outcome.newlyLost)));
|
|
2184
|
+
assertNotDisposed(rt);
|
|
2185
|
+
await announceTerminals(rt);
|
|
2186
|
+
scheduleDeferredReenqueueRetry(rt, record.intentId);
|
|
2187
|
+
return undefined;
|
|
2188
|
+
}
|
|
2189
|
+
return {
|
|
2190
|
+
record: { ...record, parentEpoch: state.groupContext.epoch, changes: outcome.changes },
|
|
2191
|
+
intent: rt.ctx.redriveFaultsForTest?.dropPayload === true ? { type: 'commit' } : { type: 'commit', ...outcome.contribution },
|
|
2192
|
+
newlyLost: outcome.newlyLost,
|
|
2193
|
+
};
|
|
2194
|
+
}
|
|
2195
|
+
/**
|
|
2196
|
+
* The drive of the commit re-drive record of root `rootIntentId` (epic marmot-rewound-own-commit-no-signal, DD1/DD11): the
|
|
2197
|
+
* record a withdrawal wrote, checked against the current tip and built as ONE fresh commit. Reached from the detached
|
|
2198
|
+
* schedule after a transition, a settling sibling's retry, the queue drain and the load-time resume (`resumePending`).
|
|
2199
|
+
* The shared rules (stranded, gate, started, leadership) are {@link reenqueueTurn}'s; the election wait and a deferred
|
|
2200
|
+
* proposal hold it like any own commit. No timer is involved: the first attempt is made as soon as the group queue allows.
|
|
2201
|
+
* Publishing, confirmation and retry are the ordinary pipeline's (`driveIntent`), which spend `attempts` like any commit.
|
|
2202
|
+
*/
|
|
2203
|
+
async function driveCommitRedrive(rt, rootIntentId) {
|
|
2204
|
+
const turn = await reenqueueTurn(rt, rootIntentId, true);
|
|
2205
|
+
if (typeof turn === 'string')
|
|
2206
|
+
return turn;
|
|
2207
|
+
if (awaitingElectedSelfRemove(rt)) {
|
|
2208
|
+
rt.queuedHint = undefined;
|
|
2209
|
+
return 'held';
|
|
2210
|
+
}
|
|
2211
|
+
const deferred = await decideStagedProposals(rt);
|
|
2212
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
2213
|
+
return 'absent';
|
|
2214
|
+
if (deferred > 0) {
|
|
2215
|
+
rt.queuedHint = undefined;
|
|
2216
|
+
return 'held';
|
|
2217
|
+
}
|
|
2218
|
+
const decision = await decideRedrive(rt, turn);
|
|
2219
|
+
if (decision === 'held')
|
|
2220
|
+
return 'held';
|
|
2221
|
+
if (decision === undefined)
|
|
2222
|
+
return 'driven';
|
|
2223
|
+
// DD9 (AC-DRIVE-4): the budget is carried across the re-drives of the root (the marker's `attempts`, +1 per confirmed re-drive). A
|
|
2224
|
+
// change the checks just found lost needs no publish and keeps its own reason; the survivors need one, which the budget no longer allows.
|
|
2225
|
+
if (decision.record.attempts >= rt.ctx.maxPublishAttempts) {
|
|
2226
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deleteEndingRecord(tx, decision.record, 'attempts-exhausted', decision.newlyLost)));
|
|
2227
|
+
assertNotDisposed(rt);
|
|
2228
|
+
await endRedriveExhausted(rt, decision.record);
|
|
2229
|
+
return 'driven';
|
|
2230
|
+
}
|
|
2231
|
+
// `newlyLost` are written with the staging transaction (or, a terminal refusal before it, with the one that ends the record); `staged` hands the
|
|
2232
|
+
// outbox over once the staging transaction committed (at least once, S11).
|
|
2233
|
+
if (rt.ctx.redriveFaultsForTest?.dispatchLostBeforeMint === true)
|
|
2234
|
+
dispatch(rt, lostTerminals(reportedIntentIdOf(decision.record), decision.newlyLost));
|
|
2235
|
+
await driveIntent(rt, decision.record, false, { intent: decision.intent, newlyLost: decision.newlyLost, staged: () => announceTerminals(rt) });
|
|
2236
|
+
return 'driven';
|
|
2237
|
+
}
|
|
2238
|
+
/**
|
|
2239
|
+
* A re-enqueued self-update left envelope-less (never published) when the group is fenced, or when its
|
|
2240
|
+
* removal settles for good (`removedFromGroup`, no longer revisable) can never be driven. Unlike an
|
|
2241
|
+
* `awaiting-sync` intent, `failAwaitingIntents` never looks at a `reenqueuedFrom` record, so each such record is
|
|
2242
|
+
* deleted and failed `intentFailed` once, under the ORIGINAL intent id (`reportedIntentIdOf`) a caller actually
|
|
2243
|
+
* saw — never the fresh id this particular re-enqueue attempt was given.
|
|
2244
|
+
*
|
|
2245
|
+
* Crash safety: the removal's settlement deletes them in the SAME transaction that clears the removal marker
|
|
2246
|
+
* ({@link takeStrandedSelfUpdateReenqueues}, ingest.ts `settleDueRemoval`), then reports them; a fence's cleanup
|
|
2247
|
+
* is repeated by every load of the fenced group (`scheduleAwaitingFenceCleanup`). And any drive of such a record
|
|
2248
|
+
* in a group already fenced or finally removed fails it here too (`driveReenqueuedSelfUpdate`), so a record a
|
|
2249
|
+
* failed or interrupted cleanup left durable is still reported by the next load or trigger.
|
|
2250
|
+
*/
|
|
2251
|
+
export async function takeStrandedSelfUpdateReenqueues(tx) {
|
|
2252
|
+
const view = tx.artifact('pendingIntents');
|
|
2253
|
+
// Both families (DD11 stranded rung): a commit re-drive is stranded exactly like a self-update re-enqueue.
|
|
2254
|
+
const records = (await listPendingIntentRecords(view)).filter((r) => r.reenqueuedFrom !== undefined && r.envelope === undefined);
|
|
2255
|
+
// S11: a stranded commit re-drive's `own-change-lost` terminals commit with its delete.
|
|
2256
|
+
for (const r of records)
|
|
2257
|
+
await deleteEndingRecord(tx, r, 'stranded');
|
|
2258
|
+
return records;
|
|
2259
|
+
}
|
|
2260
|
+
/** Reports records {@link takeStrandedSelfUpdateReenqueues} deleted (after their transaction committed). */
|
|
2261
|
+
export async function reportStrandedSelfUpdateReenqueues(rt, stranded, cause) {
|
|
2262
|
+
// A commit re-drive owes each change a terminal: `stranded` (written with the delete), handed over once, then the `intentFailed`s (DD11).
|
|
2263
|
+
if (stranded.some(isCommitRedrive))
|
|
2264
|
+
await announceTerminals(rt);
|
|
2265
|
+
for (const record of stranded) {
|
|
2266
|
+
const reportedIntentId = reportedIntentIdOf(record);
|
|
2267
|
+
const error = new MarmotError('send-rejected', `send: re-enqueued intent ${reportedIntentId} on group ${rt.groupId} can never be driven`, {
|
|
2268
|
+
cause,
|
|
2269
|
+
details: { groupId: rt.groupId, intentId: reportedIntentId },
|
|
2270
|
+
});
|
|
2271
|
+
reportIntentFailed(rt, record, error);
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
export async function failStrandedSelfUpdateReenqueues(rt, cause) {
|
|
2275
|
+
const stranded = await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => takeStrandedSelfUpdateReenqueues(tx)));
|
|
2276
|
+
await reportStrandedSelfUpdateReenqueues(rt, stranded, cause);
|
|
2277
|
+
}
|
|
2278
|
+
// ---------------------------------------------------------------------------
|
|
2279
|
+
// S18 — the pre-sync send gate (AC-PIPE-15)
|
|
2280
|
+
// ---------------------------------------------------------------------------
|
|
2281
|
+
//
|
|
2282
|
+
// A runtime built by a load starts with `sendGateOpen === false` (group-runtime.ts): the stored state may be a
|
|
2283
|
+
// restored snapshot whose sender ratchet was rolled back, and encrypting under it reuses keys and nonces. The gate
|
|
2284
|
+
// opens when the group's first post-load history pass completes (ingest.ts `runHistoryPass`, where it sets
|
|
2285
|
+
// `caughtUp`: every requested relay returned EOSE — a relay excluded as silent, AC-PIPE-14, is not requested; a
|
|
2286
|
+
// timed-out pass never completes). That sync either fences the group (AC-HIST-4: the restore forgot an own commit)
|
|
2287
|
+
// or brings the state up to what the relays hold. Until then:
|
|
2288
|
+
// - `send()` (every intent kind) records the intent `awaiting-sync` — envelope-less, nothing ratchet-derived —
|
|
2289
|
+
// and resolves (queue rather than block: offline-first consumers keep working);
|
|
2290
|
+
// - `invite`, `setAdmins`/`departAsAdmin`, `rename`, `devices.forget` and `leave` are refused `commit-pending` (reason
|
|
2291
|
+
// `awaiting-sync`) before recording anything: their results need the publish;
|
|
2292
|
+
// - background minting (the rumor queue's drain, the SelfRemove auto-commit, envelope-less records a crash left
|
|
2293
|
+
// behind) waits; re-publishing an envelope minted before the load and confirming a published record mint
|
|
2294
|
+
// nothing and go ahead.
|
|
2295
|
+
// On opening, `releaseAwaitingSync` drives the awaiting records in `seq` order through the ordinary resume path.
|
|
2296
|
+
// When the group is fenced instead, `failAwaitingIntents` fails each one `fenced` (`intentFailed`), unpublished.
|
|
2297
|
+
/** Refuses an own envelope while the gate is closed: `commit-pending`, reason `awaiting-sync` (retryable). */
|
|
2298
|
+
function assertSendGateOpen(rt, operation) {
|
|
2299
|
+
if (rt.sendGateOpen)
|
|
2300
|
+
return;
|
|
2301
|
+
throw new MarmotError('commit-pending', `${operation}: group ${rt.groupId} has not completed its first sync since it was loaded; nothing is encrypted before it`, {
|
|
2302
|
+
details: { groupId: rt.groupId, reason: 'awaiting-sync', retryable: true },
|
|
2303
|
+
});
|
|
2304
|
+
}
|
|
2305
|
+
/**
|
|
2306
|
+
* `send()` before the first post-load sync: the intent is recorded `awaiting-sync` and nothing else happens. One
|
|
2307
|
+
* own commit at a time still holds (`commit-pending`); the proposal policy and the AC-INVITE-3 queue decision are
|
|
2308
|
+
* taken at release, over the synced state.
|
|
2309
|
+
*/
|
|
2310
|
+
async function acceptAwaitingSync(rt, intent, remaining) {
|
|
2311
|
+
if (isCommitKind(intent.type))
|
|
2312
|
+
assertNoOwnCommitPending(rt, remaining, 'send');
|
|
2313
|
+
return recordIntent(rt, intent.type, encodeGroupIntent(intent), 'awaiting-sync');
|
|
2314
|
+
}
|
|
2315
|
+
/**
|
|
2316
|
+
* Releases one awaiting-sync record (gate open), in one transaction: an application rumor that must wait for a
|
|
2317
|
+
* commit (AC-INVITE-3: proposals staged, rumors queued, an application intent still pending) moves to the rumor
|
|
2318
|
+
* queue — `undefined` is returned and the queue drains it; anything else becomes an ordinary envelope-less
|
|
2319
|
+
* `pending-publish` record re-parented to the synced epoch (nothing was built at the epoch it was accepted at) and
|
|
2320
|
+
* is returned for the resume path to drive. `undefined` too when the record is gone.
|
|
2321
|
+
*/
|
|
2322
|
+
async function releaseAwaitingRecord(rt, record) {
|
|
2323
|
+
const state = rt.engine.state;
|
|
2324
|
+
const outcome = await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
2325
|
+
const pendingView = tx.artifact('pendingIntents');
|
|
2326
|
+
const current = await readPendingIntentRecord(pendingView, record.intentId);
|
|
2327
|
+
if (current?.state !== 'awaiting-sync')
|
|
2328
|
+
return undefined;
|
|
2329
|
+
if (current.intent.type === 'application') {
|
|
2330
|
+
const queuedView = tx.artifact('queuedRumors');
|
|
2331
|
+
const queued = await listQueuedRumorRecords(queuedView);
|
|
2332
|
+
const applicationPending = (await listPendingIntentRecords(pendingView)).some((r) => r.kind === 'application' && r.state !== 'awaiting-sync');
|
|
2333
|
+
if (mustQueue(state, queued.length, applicationPending)) {
|
|
2334
|
+
const last = queued.at(-1);
|
|
2335
|
+
const barrier = queueBarrier(state, last?.barrier);
|
|
2336
|
+
await deletePendingIntentRecord(pendingView, current.intentId);
|
|
2337
|
+
await writeQueuedRumorRecord(queuedView, {
|
|
2338
|
+
v: 1,
|
|
2339
|
+
queueId: current.intentId,
|
|
2340
|
+
seq: (last?.seq ?? 0) + 1,
|
|
2341
|
+
rumor: current.intent.rumor,
|
|
2342
|
+
...(barrier === undefined ? {} : { barrier }),
|
|
2343
|
+
queuedAt: rt.ctx.clock.now(),
|
|
2344
|
+
});
|
|
2345
|
+
return 'queued';
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
const ready = { ...current, state: 'pending-publish', parentEpoch: state.groupContext.epoch };
|
|
2349
|
+
await writePendingIntentRecord(pendingView, ready);
|
|
2350
|
+
return ready;
|
|
2351
|
+
}));
|
|
2352
|
+
assertNotDisposed(rt);
|
|
2353
|
+
if (outcome === 'queued') {
|
|
2354
|
+
rt.queuedHint = undefined;
|
|
2355
|
+
rt.scheduleQueueDrain();
|
|
2356
|
+
return undefined;
|
|
2357
|
+
}
|
|
2358
|
+
return outcome;
|
|
2359
|
+
}
|
|
2360
|
+
/**
|
|
2361
|
+
* The gate just opened (ingest.ts, the first complete post-load history pass): the awaiting records are driven in
|
|
2362
|
+
* the background, in `seq` order (the resume path), the rumor queue drains, and a SelfRemove election at rest that
|
|
2363
|
+
* the load deferred is re-run. Never throws.
|
|
2364
|
+
*/
|
|
2365
|
+
export function releaseAwaitingSync(rt) {
|
|
2366
|
+
for (const waiter of rt.syncWaiters)
|
|
2367
|
+
waiter.resolve();
|
|
2368
|
+
rt.syncWaiters.clear();
|
|
2369
|
+
if (rt.disposed || rt.ctx.isDisposed() || rt.fence !== undefined)
|
|
2370
|
+
return;
|
|
2371
|
+
resumePendingInBackground(rt);
|
|
2372
|
+
rt.scheduleQueueDrain();
|
|
2373
|
+
const election = electionAtRest(rt.engine.state, rt.leaverPredicate);
|
|
2374
|
+
if (election !== undefined)
|
|
2375
|
+
scheduleSelfRemoveCommit(rt, election);
|
|
2376
|
+
}
|
|
2377
|
+
/**
|
|
2378
|
+
* AC-PIPE-15 with AC-HIST-4: the group is fenced (the marker is already durable). Every awaiting-sync intent —
|
|
2379
|
+
* never encrypted — is dropped and fails `fenced` via `intentFailed`, and so is every queued rumor (to its waiting
|
|
2380
|
+
* `send()` caller when it has one); nothing is published for them. Records are
|
|
2381
|
+
* deleted first, then announced, as for every terminal intent; a load of a fenced group repeats this for records
|
|
2382
|
+
* a crash left between the fence write and this cleanup. AC-RACE-1: a re-enqueued self-update left envelope-less
|
|
2383
|
+
* is ALSO stranded by a fence (every drive path refuses a fenced group) and is failed the same way, under its
|
|
2384
|
+
* original intent id — it is not an `awaiting-sync` record, so it would otherwise go unreported.
|
|
2385
|
+
*/
|
|
2386
|
+
export async function failAwaitingIntents(rt) {
|
|
2387
|
+
const fence = rt.fence;
|
|
2388
|
+
if (fence === undefined)
|
|
2389
|
+
return;
|
|
2390
|
+
for (const waiter of rt.syncWaiters)
|
|
2391
|
+
waiter.reject(fencedError(rt.groupId, 'synced', fence));
|
|
2392
|
+
rt.syncWaiters.clear();
|
|
2393
|
+
const failed = await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
2394
|
+
const view = tx.artifact('pendingIntents');
|
|
2395
|
+
const awaiting = (await listPendingIntentRecords(view)).filter((r) => r.state === 'awaiting-sync');
|
|
2396
|
+
for (const r of awaiting)
|
|
2397
|
+
await deletePendingIntentRecord(view, r.intentId);
|
|
2398
|
+
return awaiting;
|
|
2399
|
+
}));
|
|
2400
|
+
for (const record of failed) {
|
|
2401
|
+
const error = new MarmotError('fenced', `send: intent ${record.intentId} on group ${rt.groupId} was never encrypted: the group's first sync after its load fenced it`, {
|
|
2402
|
+
cause: fence,
|
|
2403
|
+
details: { groupId: rt.groupId, intentId: record.intentId },
|
|
2404
|
+
});
|
|
2405
|
+
reportIntentFailed(rt, record, error);
|
|
2406
|
+
}
|
|
2407
|
+
await failStrandedSelfUpdateReenqueues(rt, fence);
|
|
2408
|
+
// Queued rumors (AC-INVITE-3; also awaiting-sync ones released into the queue) are unencrypted too, and a fenced
|
|
2409
|
+
// group never drains its queue: each fails `fenced` — to its waiting caller, else via intentFailed.
|
|
2410
|
+
const queued = await rt.guard.call(rt.groupId, 'store', () => listQueuedRumorRecords(rt.groupStore.artifact('queuedRumors')));
|
|
2411
|
+
await dropQueuedRumors(rt, queued, (record) => new MarmotError('fenced', `send: queued rumor ${record.queueId} on group ${rt.groupId} was never encrypted and can no longer be: the group is fenced`, {
|
|
2412
|
+
cause: fence,
|
|
2413
|
+
details: { groupId: rt.groupId, intentId: record.queueId },
|
|
2414
|
+
}));
|
|
2415
|
+
}
|
|
2416
|
+
/** {@link failAwaitingIntents} on the group's queue, for a group loaded already fenced. A failure is dropped (the next load retries). */
|
|
2417
|
+
export function scheduleAwaitingFenceCleanup(rt) {
|
|
2418
|
+
void Promise.resolve()
|
|
2419
|
+
.then(() => rt.run('fenceCleanup', (rtc) => failAwaitingIntents(rtc)))
|
|
2420
|
+
.catch(() => undefined);
|
|
2421
|
+
}
|
|
2422
|
+
function leafSignatureKeyHex(bytes) {
|
|
2423
|
+
return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
|
|
2424
|
+
}
|
|
2425
|
+
/**
|
|
2426
|
+
* Whether `account` holds a leaf carrying `leafKey` (hex) — or any key of the set `leafKey` (a device chain, AC-RM-3) —
|
|
2427
|
+
* in `state`: the per-state half of `group-runtime.ts`'s `leafOnBranches` walk.
|
|
2428
|
+
*/
|
|
2429
|
+
function stateHoldsLeaf(state, account, leafKey) {
|
|
2430
|
+
const wanted = typeof leafKey === 'string' ? [leafKey] : leafKey;
|
|
2431
|
+
return getPubkeyLeafNodes(state, account).some((node) => wanted.includes(leafSignatureKeyHex(node.signaturePublicKey)));
|
|
2432
|
+
}
|
|
2433
|
+
/**
|
|
2434
|
+
* AC-RECHK-1 (amended 2026-10-04): whether `account`'s leaf `leafKey`
|
|
2435
|
+
* appears, while the exposed TIP does not hold it, on a state of the group's CANONICAL path retained in
|
|
2436
|
+
* `rt.window` whose epoch is at or after `approvalEpoch`. Modelled on `group-runtime.ts`'s `leafOnBranches`, but
|
|
2437
|
+
* evaluated synchronously here, over `rt.window`/`rt.exposedState`, from inside the invite group op itself rather
|
|
2438
|
+
* than a separate queued read: the same TOCTOU gap the S10 settlement-recheck closed for the rewind pass
|
|
2439
|
+
* (`DeviceGroupView.recheck`) would reopen here between a standalone check and this commit's own
|
|
2440
|
+
* `recordIntent`/`driveIntent`.
|
|
2441
|
+
*
|
|
2442
|
+
* Two deliberate narrowings versus a literal "any canonical window state" reading:
|
|
2443
|
+
*
|
|
2444
|
+
* 1. Canonical-path presence only, never a merely abandoned fork (`leafOnBranches`'s `canonical || abandoned`) —
|
|
2445
|
+
* a leaf whose ONLY window presence is on an abandoned fork is exactly the leaf a rewind legitimately
|
|
2446
|
+
* withdrew (AC-RECHK-2's reopen exists to re-add it) and must not be refused here.
|
|
2447
|
+
* 2. Epoch-bound by `approvalEpoch`, the group's tip epoch recorded when the approval was taken
|
|
2448
|
+
* (`ApprovalGroupRecord.approvalEpoch`). Without this, a `devices.forget` Remove — itself a CANONICAL
|
|
2449
|
+
* state — that settled strictly BEFORE the device re-enrolled would be seen here too (it is retained inside
|
|
2450
|
+
* `REWIND_HORIZON`) and would wrongly refuse the legitimate AC-RECHK-2 redrive forever. Only a canonical
|
|
2451
|
+
* departure at or after the approval's own epoch — e.g. an ordinary Remove racing the reopen decision itself,
|
|
2452
|
+
* between it and this re-drive — is grounds to refuse. `approvalEpoch === undefined` (a pre-remediation
|
|
2453
|
+
* approval record with no recorded epoch) falls back to the whole window, unbounded: the original, strictly
|
|
2454
|
+
* conservative behaviour, mirroring `approvedAt ?? 0` elsewhere in this file's enrolment counterpart. The bound
|
|
2455
|
+
* may have been advanced by `driveApproval` past a departure the approval supersedes (a forget started before
|
|
2456
|
+
* the approval whose Remove landed after it), to the tip epoch the approval's Add was first built on.
|
|
2457
|
+
*/
|
|
2458
|
+
function leafHeldOnCanonicalPath(rt, account, leafKey, approvalEpoch) {
|
|
2459
|
+
if (stateHoldsLeaf(rt.exposedState, account, leafKey))
|
|
2460
|
+
return false; // the tip holds it: this refusal does not apply
|
|
2461
|
+
const window = rt.window;
|
|
2462
|
+
const path = new Set(canonicalPath(window).map((node) => node.tag));
|
|
2463
|
+
for (const node of window.nodes.values()) {
|
|
2464
|
+
if (node.tag === window.tipTag || node.snapshot === undefined || !path.has(node.tag))
|
|
2465
|
+
continue;
|
|
2466
|
+
if (approvalEpoch !== undefined && node.epoch < approvalEpoch)
|
|
2467
|
+
continue; // older than the approval: never grounds to refuse
|
|
2468
|
+
if (stateHoldsLeaf(deserializeClientState(node.snapshot), account, leafKey))
|
|
2469
|
+
return true;
|
|
2470
|
+
}
|
|
2471
|
+
return false;
|
|
2472
|
+
}
|
|
2473
|
+
/**
|
|
2474
|
+
* `MarmotGroup.invite`'s group-queue part: validates the invitees, applies
|
|
2475
|
+
* the pending-proposal policy, and drives ONE commit adding all of them
|
|
2476
|
+
* through the durable pipeline. Resolves once the commit is published and
|
|
2477
|
+
* confirmed (its Welcomes then belong to the invitations service); rejects
|
|
2478
|
+
* like `send` for a commit that is refused or not (yet) published.
|
|
2479
|
+
*
|
|
2480
|
+
* `reopenedLeafKey`/`approvalEpoch` (S4 AC-RECHK-1): threaded only from `DeviceGroupPort.addDevice` when this Add
|
|
2481
|
+
* re-drives a group whose approval record was reopened (never set by the public `MarmotGroup.invite`). See
|
|
2482
|
+
* {@link leafHeldOnCanonicalPath}. `origin: 'approval'` (S9, DD10) is likewise threaded
|
|
2483
|
+
* only from the enrolment approval (`DeviceGroupPort.addDevice`): it stores the Add as an approval's, which keeps it
|
|
2484
|
+
* unmarked (its owner is `reopenApprovalsAfterRewind`).
|
|
2485
|
+
*/
|
|
2486
|
+
export async function inviteIntent(rt, keyPackages, reopenedLeafKey, approvalEpoch, origin) {
|
|
2487
|
+
const remaining = await preflight(rt, 'invite');
|
|
2488
|
+
assertMember(rt, 'invite');
|
|
2489
|
+
assertSendGateOpen(rt, 'invite');
|
|
2490
|
+
assertNoOwnCommitPending(rt, remaining, 'invite');
|
|
2491
|
+
const verifier = rt.ctx.verifier;
|
|
2492
|
+
const invitees = await validateInvitees(keyPackages, {
|
|
2493
|
+
verify: verifier === undefined ? undefined : (event) => rt.guard.call(rt.groupId, 'verifier', () => verifier(event)),
|
|
2494
|
+
nowSeconds: Math.floor(rt.ctx.clock.now() / 1000),
|
|
2495
|
+
state: rt.engine.state,
|
|
2496
|
+
});
|
|
2497
|
+
assertNotDisposed(rt);
|
|
2498
|
+
await refuseOwnChainAdd(rt, 'invite', invitees);
|
|
2499
|
+
if (relayPlaneFor(rt).length === 0) {
|
|
2500
|
+
throw new MarmotError('invalid-options', `invite: group ${rt.groupId} has no relays to name in its Welcomes`, { details: { groupId: rt.groupId } });
|
|
2501
|
+
}
|
|
2502
|
+
// S5 seam (architecture.md "S4 <-> S5"): this check stays AFTER applyProposalPolicy and as plain synchronous
|
|
2503
|
+
// code over the window — S5's elected-SelfRemove wait lands in or before applyProposalPolicy, never after it.
|
|
2504
|
+
await applyProposalPolicy(rt, 'invite');
|
|
2505
|
+
if (reopenedLeafKey !== undefined) {
|
|
2506
|
+
const self = await callSignerForGroup(rt, () => rt.ctx.signer.getPublicKey());
|
|
2507
|
+
if (leafHeldOnCanonicalPath(rt, self, reopenedLeafKey, approvalEpoch)) {
|
|
2508
|
+
throw new MarmotError('invalid-options', `invite: group ${rt.groupId} still holds leaf ${String(reopenedLeafKey)} on its canonical rewind-window path at or after epoch ${String(approvalEpoch)}; refusing to re-add it`, {
|
|
2509
|
+
details: { groupId: rt.groupId, leafKey: typeof reopenedLeafKey === 'string' ? reopenedLeafKey : reopenedLeafKey[0], ...(typeof reopenedLeafKey === 'string' ? {} : { leafKeys: [...reopenedLeafKey] }), reason: REOPENED_LEAF_STILL_CANONICAL },
|
|
2510
|
+
});
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
const record = await recordIntent(rt, 'commit', {
|
|
2514
|
+
type: 'commit',
|
|
2515
|
+
invite: { invitees: invitees.map((i) => ({ event: i.event, inviteeRef: i.inviteeRef })) },
|
|
2516
|
+
...(origin === undefined ? {} : { origin }),
|
|
2517
|
+
});
|
|
2518
|
+
await driveIntent(rt, record, true);
|
|
2519
|
+
return {
|
|
2520
|
+
inviteId: record.intentId,
|
|
2521
|
+
invitees: invitees.map(({ inviteeRef, pubkey, keyPackageEventId }) => ({ inviteeRef, pubkey, keyPackageEventId })),
|
|
2522
|
+
};
|
|
2523
|
+
}
|
|
2524
|
+
// ---------------------------------------------------------------------------
|
|
2525
|
+
// S8 — same-account Remove (devices.forget, AC-DEV-7)
|
|
2526
|
+
// ---------------------------------------------------------------------------
|
|
2527
|
+
/**
|
|
2528
|
+
* `devices.forget`'s group-queue part: ONE durable commit removing every leaf
|
|
2529
|
+
* carrying `leafKey` (the same-account seams let a non-admin leaf commit it),
|
|
2530
|
+
* through the same pipeline as every own commit — proposal policy first,
|
|
2531
|
+
* refused while an own commit is pending. `absent` when the tree holds no
|
|
2532
|
+
* such leaf any more (nothing is recorded). Rejects like `send`.
|
|
2533
|
+
*/
|
|
2534
|
+
export async function removeLeafIntent(rt, leafKey) {
|
|
2535
|
+
// A device chain (`DeviceGroupPort.removeLeaves`, Design Decision 8): ONE commit removing every leaf carrying any key.
|
|
2536
|
+
const leafKeys = typeof leafKey === 'string' ? [leafKey] : [...new Set(leafKey)];
|
|
2537
|
+
const remaining = await preflight(rt, 'forget');
|
|
2538
|
+
assertMember(rt, 'forget');
|
|
2539
|
+
if (!leafKeys.some((key) => holdsLeafKey(rt.engine.state, key)))
|
|
2540
|
+
return 'absent';
|
|
2541
|
+
assertSendGateOpen(rt, 'forget');
|
|
2542
|
+
assertNoOwnCommitPending(rt, remaining, 'forget');
|
|
2543
|
+
await applyProposalPolicy(rt, 'forget');
|
|
2544
|
+
const conflict = stagedAddConflict('forget', rt.groupId, rt.engine.state, { leafKeys });
|
|
2545
|
+
if (conflict !== undefined)
|
|
2546
|
+
throw conflict;
|
|
2547
|
+
const record = await recordIntent(rt, 'commit', { type: 'commit', removeLeafKeys: leafKeys });
|
|
2548
|
+
await driveIntent(rt, record, true);
|
|
2549
|
+
return 'committed';
|
|
2550
|
+
}
|
|
2551
|
+
// ---------------------------------------------------------------------------
|
|
2552
|
+
// Epic marmot-device-key-rotation (S6) — the `rekey` self-update (AC-REKEY-1..13)
|
|
2553
|
+
// ---------------------------------------------------------------------------
|
|
2554
|
+
/** Whether `record` is a self-update that replaces the local leaf's signature key. */
|
|
2555
|
+
function isRekeyRecord(record) {
|
|
2556
|
+
return record.intent.type === 'self-update' && record.intent.rekey !== undefined;
|
|
2557
|
+
}
|
|
2558
|
+
/**
|
|
2559
|
+
* AC-REKEY-6: a rekey with nothing left to do is dropped without a commit (no empty epoch bump) — `already-rekeyed`
|
|
2560
|
+
* (the leaf carries the target) or `superseded` (`deviceRegistry/self` moved on: the newer rotation's re-key set
|
|
2561
|
+
* covers the group). The record is deleted first, then reported: a waiting caller gets the `send-rejected`, an
|
|
2562
|
+
* unattended record reports `intentFailed` once. No queue-commit hold is taken: no commit was refused.
|
|
2563
|
+
*/
|
|
2564
|
+
async function dropRekey(rt, record, hasCaller, reason) {
|
|
2565
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => deletePendingIntentRecord(tx.artifact('pendingIntents'), record.intentId)));
|
|
2566
|
+
assertNotDisposed(rt);
|
|
2567
|
+
const reportedIntentId = reportedIntentIdOf(record);
|
|
2568
|
+
const error = new MarmotError('send-rejected', `send: rekey intent ${reportedIntentId} on group ${rt.groupId} was dropped: ${reason}`, {
|
|
2569
|
+
details: { groupId: rt.groupId, intentId: reportedIntentId, reason },
|
|
2570
|
+
});
|
|
2571
|
+
scheduleDeferredReenqueueRetry(rt, record.intentId);
|
|
2572
|
+
if (hasCaller) {
|
|
2573
|
+
notifySettled(rt, record, error);
|
|
2574
|
+
throw error;
|
|
2575
|
+
}
|
|
2576
|
+
reportIntentFailed(rt, record, error);
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* `DeviceGroupPort.rekey`'s group-queue part: ONE durable self-update replacing the local leaf's signature key with
|
|
2580
|
+
* `target`. `satisfied` (no commit) when the leaf already carries it, `not-member` when the local leaf was removed;
|
|
2581
|
+
* otherwise gated and recorded like every own commit (send gate, one own commit at a time, proposal policy) and driven
|
|
2582
|
+
* as a caller (`driveIntent` with a caller): it resolves `committed` once the commit is published and confirmed, and
|
|
2583
|
+
* rejects like `send` (`superseded` / `already-rekeyed` / `proof-invalid` as `send-rejected` `details.reason`).
|
|
2584
|
+
*/
|
|
2585
|
+
export async function rekeyIntent(rt, target) {
|
|
2586
|
+
if (!isHex64(target))
|
|
2587
|
+
throw new MarmotError('invalid-options', 'rekey: target must be a 64-char lower-case hex leaf key', { details: { reason: 'invalid-target' } });
|
|
2588
|
+
const remaining = await preflight(rt, 'rekey');
|
|
2589
|
+
if (rt.engine.state.groupActiveState.kind === 'removedFromGroup')
|
|
2590
|
+
return 'not-member';
|
|
2591
|
+
if (ownLeafKeyOf(rt.engine.state) === target)
|
|
2592
|
+
return 'satisfied';
|
|
2593
|
+
assertSendGateOpen(rt, 'rekey');
|
|
2594
|
+
assertNoOwnCommitPending(rt, remaining, 'rekey');
|
|
2595
|
+
await applyProposalPolicy(rt, 'rekey');
|
|
2596
|
+
const record = await recordIntent(rt, 'self-update', { type: 'self-update', rekey: { target } });
|
|
2597
|
+
await driveIntent(rt, record, true);
|
|
2598
|
+
return 'committed';
|
|
2599
|
+
}
|
|
2600
|
+
/**
|
|
2601
|
+
* AC-REKEY-11 (Design Decision 10): after a Welcome adoption (join, enrolment approval, a competing Welcome's
|
|
2602
|
+
* switch) or a restore, a local leaf whose key differs from `deviceRegistry/self` AND has a `rotated:` link is a
|
|
2603
|
+
* leaf of this device's own retired key (a Welcome built on a retired-key KeyPackage, a backup older than the
|
|
2604
|
+
* rotation): a `rekey` toward `self` is recorded, with no new key, no proof and no signer call. A leaf whose key
|
|
2605
|
+
* has no link is a foreign key and is left alone. Before the first post-load sync the record is accepted
|
|
2606
|
+
* `awaiting-sync` (the pipeline releases it); another own commit pending, a group left or fenced, and an owed rekey
|
|
2607
|
+
* already recorded all end it quietly (the resume-time sweep of the device service covers a miss).
|
|
2608
|
+
*/
|
|
2609
|
+
async function rekeyFollowUp(rt) {
|
|
2610
|
+
if (rt.disposed || rt.ctx.isDisposed() || rt.leaseLost !== undefined)
|
|
2611
|
+
return; // queued before a dispose/reload/lease loss: it belongs to the old holder
|
|
2612
|
+
if (rt.fence !== undefined || rt.disbanded !== undefined || rt.abandoned || rt.engine.state.groupActiveState.kind === 'removedFromGroup')
|
|
2613
|
+
return;
|
|
2614
|
+
const target = await retiredLeafTarget(rt, rt.engine.state);
|
|
2615
|
+
if (target === undefined)
|
|
2616
|
+
return;
|
|
2617
|
+
const remaining = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(rt.groupStore.artifact('pendingIntents')));
|
|
2618
|
+
if (remaining.some((r) => r.intent.type === 'self-update' && r.intent.rekey?.target === target))
|
|
2619
|
+
return;
|
|
2620
|
+
if (remaining.some((r) => isCommitKind(r.kind)))
|
|
2621
|
+
return;
|
|
2622
|
+
const intent = { type: 'self-update', rekey: { target } };
|
|
2623
|
+
if (!rt.sendGateOpen) {
|
|
2624
|
+
await recordIntent(rt, 'self-update', intent, 'awaiting-sync');
|
|
2625
|
+
return;
|
|
2626
|
+
}
|
|
2627
|
+
// Durable first, then the ordinary resume loop drives it: it runs the elected-SelfRemove wait and the staged-proposal
|
|
2628
|
+
// policy (`onProposal` deferrals) before any commit is built, and leaves the record in place while either holds.
|
|
2629
|
+
await recordIntent(rt, 'self-update', intent);
|
|
2630
|
+
await resumePending(rt);
|
|
2631
|
+
}
|
|
2632
|
+
/**
|
|
2633
|
+
* The target (`deviceRegistry/self`'s key) a rekey owes when the local leaf of `state` sits on a retired key of this
|
|
2634
|
+
* device — its key differs from `self` AND has a `rotated:` link — else `undefined`. Shared by the off-queue pre-check
|
|
2635
|
+
* and the queued follow-up.
|
|
2636
|
+
*/
|
|
2637
|
+
async function retiredLeafTarget(rt, state) {
|
|
2638
|
+
const self = await readSelfKey(rt);
|
|
2639
|
+
if (self === undefined || rt.disposed || rt.ctx.isDisposed())
|
|
2640
|
+
return undefined;
|
|
2641
|
+
const own = ownLeafKeyOf(state);
|
|
2642
|
+
if (own === undefined || own === self.leafKey)
|
|
2643
|
+
return undefined;
|
|
2644
|
+
const link = await accountRead(rt, () => rt.ctx.stores.account.transaction((tx) => readDeviceKeyLink(tx.artifact('deviceRegistry'), own)));
|
|
2645
|
+
return link === undefined ? undefined : self.leafKey;
|
|
2646
|
+
}
|
|
2647
|
+
/** {@link rekeyFollowUp} as its own detached group operation (queued only when the leaf is on a retired key, decided OFF the queue so the ordinary case queues nothing); a failure leaves the durable record (if any) to the next resume. A lost lease ends the runtime like every background retry. Never throws. */
|
|
2648
|
+
export function scheduleRekeyFollowUp(rt) {
|
|
2649
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
2650
|
+
return;
|
|
2651
|
+
void Promise.resolve()
|
|
2652
|
+
.then(() => rt.guard.detached(async () => ((await retiredLeafTarget(rt, rt.exposedState)) !== undefined ? rt.run('rekeyFollowUp', (rtc) => rekeyFollowUp(rtc)) : undefined)))
|
|
2653
|
+
.catch((error) => {
|
|
2654
|
+
backgroundRetryEnds(rt, error); // records a lost lease (`no-lease`) and fires the terminal watchers; anything else leaves the durable record to the next resume
|
|
2655
|
+
});
|
|
2656
|
+
}
|
|
2657
|
+
/**
|
|
2658
|
+
* AC-REKEY-13 (Design Decision 10): the invite and enrolment Add-builders refuse, sender-side only, a KeyPackage whose
|
|
2659
|
+
* leaf key is in the local device's own chain (`deviceRegistry/self` and the keys its lineage links to it) when any
|
|
2660
|
+
* chain key is already in the group tree: the device would hold two leaves of one device, and a later rekey would have
|
|
2661
|
+
* to Remove one. `send-rejected`, `details.reason: 'device-already-member'`, before anything is recorded. No
|
|
2662
|
+
* receive-side rule exists: a peer's commit adding it is still processed (and repaired by the rekey build).
|
|
2663
|
+
*/
|
|
2664
|
+
async function refuseOwnChainAdd(rt, operation, invitees) {
|
|
2665
|
+
const self = await readSelfKey(rt);
|
|
2666
|
+
if (self === undefined)
|
|
2667
|
+
return;
|
|
2668
|
+
// This device's DIRECTED ancestry: `self` and every key from which `self` is reached along the recorded links. Not the
|
|
2669
|
+
// lineage component: two live installations that share an origin key are one component but two devices, and the other
|
|
2670
|
+
// installation's KeyPackage must stay invitable.
|
|
2671
|
+
const chain = new Set([self.leafKey]);
|
|
2672
|
+
try {
|
|
2673
|
+
// The registry's recorded edges (primary `rotated:` links, then the `rotated-alt:` alternate evidence) through the ONE
|
|
2674
|
+
// strict decoder: a malformed row rejects and the chain falls back to the current key alone.
|
|
2675
|
+
const edges = await accountRead(rt, () => rt.ctx.stores.account.transaction((tx) => readRegistryEdges(tx.artifact('deviceRegistry'))));
|
|
2676
|
+
for (const key of directedAncestors(self.leafKey, edges))
|
|
2677
|
+
chain.add(key);
|
|
2678
|
+
}
|
|
2679
|
+
catch {
|
|
2680
|
+
// Without the links only the current device key counts: still refuses the shape that needs no registry.
|
|
2681
|
+
}
|
|
2682
|
+
assertNotDisposed(rt);
|
|
2683
|
+
const tree = commitViewOf(rt.engine.state);
|
|
2684
|
+
if (![...chain].some((key) => holdsLeafKeyOfView(tree, key)))
|
|
2685
|
+
return;
|
|
2686
|
+
for (const invitee of invitees) {
|
|
2687
|
+
const key = toHex(getKeyPackage(invitee.event).leafNode.signaturePublicKey);
|
|
2688
|
+
if (!chain.has(key))
|
|
2689
|
+
continue;
|
|
2690
|
+
throw new MarmotError('send-rejected', `${operation}: KeyPackage ${invitee.keyPackageEventId} carries a key of this device, which already has a leaf in group ${rt.groupId}`, {
|
|
2691
|
+
details: { groupId: rt.groupId, reason: 'device-already-member', keyPackageEventId: invitee.keyPackageEventId },
|
|
2692
|
+
});
|
|
2693
|
+
}
|
|
2694
|
+
}
|
|
2695
|
+
// ---------------------------------------------------------------------------
|
|
2696
|
+
// S9 — admin list, leave, SelfRemove auto-commit (AC-ADMIN-1, -3, -6, -9)
|
|
2697
|
+
// ---------------------------------------------------------------------------
|
|
2698
|
+
/**
|
|
2699
|
+
* `setAdmins`' group-queue part (and `departAsAdmin`'s demotion): the list is
|
|
2700
|
+
* validated against the current tree (AC-ADMIN-1: `invalid-admin-policy`,
|
|
2701
|
+
* nothing recorded), then ONE durable commit sets it — refused
|
|
2702
|
+
* `commit-pending` while an own commit is pending, like every own commit.
|
|
2703
|
+
* The committer must be an admin (the engine's `authorizeCommit` refuses a
|
|
2704
|
+
* non-admin's admin-list change: `send-rejected`).
|
|
2705
|
+
*/
|
|
2706
|
+
export async function setAdminsIntent(rt, accounts, operation = 'setAdmins') {
|
|
2707
|
+
const remaining = await preflight(rt, operation);
|
|
2708
|
+
assertMember(rt, operation);
|
|
2709
|
+
const admins = validateAdminList(rt.engine.state.ratchetTree, accounts);
|
|
2710
|
+
assertSendGateOpen(rt, operation);
|
|
2711
|
+
assertNoOwnCommitPending(rt, remaining, operation);
|
|
2712
|
+
await applyProposalPolicy(rt, operation);
|
|
2713
|
+
const record = await recordIntent(rt, 'commit', { type: 'commit', admins: [...admins] });
|
|
2714
|
+
await driveIntent(rt, record, true);
|
|
2715
|
+
}
|
|
2716
|
+
/** The admin-commit pre-check shared by `rename` and `removeMember`: `invalid-options` / `not-admin` unless `selfPubkey` is an active admin, before anything is recorded. */
|
|
2717
|
+
function assertLocalAdmin(rt, operation, selfPubkey) {
|
|
2718
|
+
const adminView = getMarmotGroupView(rt.engine.state);
|
|
2719
|
+
if (adminView === null || !isActiveAdmin(rt.engine.state.ratchetTree, adminView.adminPubkeys, selfPubkey)) {
|
|
2720
|
+
throw new MarmotError('invalid-options', `${operation}: the local account is not an admin of group ${rt.groupId}`, { details: { groupId: rt.groupId, reason: 'not-admin' } });
|
|
2721
|
+
}
|
|
2722
|
+
}
|
|
2723
|
+
/**
|
|
2724
|
+
* `removeMember`'s group-queue part: ONE durable admin commit removing every leaf `account` holds when it is built. Gate order
|
|
2725
|
+
* (Design Decision 6): preflight, member, send gate, malformed (`invalid-options`, `malformed-account`), self (`invalid-options`,
|
|
2726
|
+
* `self`: leaving is `departAsAdmin()` / `leave()`), not-admin (`invalid-options`, `not-admin`), absent (no leaf: resolves, nothing
|
|
2727
|
+
* recorded), own commit pending (`commit-pending`), proposal policy. The client adds no admin-policy update: the engine's
|
|
2728
|
+
* coupling splice drops a removed admin's entry in the same commit.
|
|
2729
|
+
*/
|
|
2730
|
+
export async function removeMemberIntent(rt, account) {
|
|
2731
|
+
const operation = 'removeMember';
|
|
2732
|
+
const remaining = await preflight(rt, operation);
|
|
2733
|
+
assertMember(rt, operation);
|
|
2734
|
+
assertSendGateOpen(rt, operation);
|
|
2735
|
+
assertAccountPubkey(operation, account);
|
|
2736
|
+
const selfPubkey = await callSignerForGroup(rt, () => rt.ctx.signer.getPublicKey());
|
|
2737
|
+
assertNotDisposed(rt);
|
|
2738
|
+
if (account === selfPubkey) {
|
|
2739
|
+
throw new MarmotError('invalid-options', `${operation}: the local account cannot remove itself; use departAsAdmin() or leave()`, { details: { groupId: rt.groupId, reason: 'self' } });
|
|
2740
|
+
}
|
|
2741
|
+
assertLocalAdmin(rt, operation, selfPubkey);
|
|
2742
|
+
if (!holdsLeafOfAny(rt.engine.state, [account]))
|
|
2743
|
+
return;
|
|
2744
|
+
assertNoOwnCommitPending(rt, remaining, operation);
|
|
2745
|
+
await applyProposalPolicy(rt, operation);
|
|
2746
|
+
const conflict = stagedAddConflict(operation, rt.groupId, rt.engine.state, { accounts: [account] });
|
|
2747
|
+
if (conflict !== undefined)
|
|
2748
|
+
throw conflict;
|
|
2749
|
+
const record = await recordIntent(rt, 'commit', { type: 'commit', removeAccounts: [account], removeAccountLeafKeys: leafKeysOfAccount(rt.engine.state, account) });
|
|
2750
|
+
await driveIntent(rt, record, true);
|
|
2751
|
+
}
|
|
2752
|
+
/**
|
|
2753
|
+
* `rename`'s group-queue part: ONE durable admin commit carrying the `marmot.group.profile.v1` update by value.
|
|
2754
|
+
* Gate order (Design Decision 6): preflight, member, send gate, malformed (`invalid-options`), not-admin
|
|
2755
|
+
* (`invalid-options`, `details.reason: 'not-admin'`, nothing recorded), own commit pending (`commit-pending`), proposal
|
|
2756
|
+
* policy. An omitted description is resolved HERE, right before the record is written, from the group view, and stored by
|
|
2757
|
+
* value. A concurrent change is never merged: if another admin's profile commit is applied before this commit is built,
|
|
2758
|
+
* the stored record is dropped as stale (`send-rejected`) and that profile stands; in a same-epoch race the fork rule
|
|
2759
|
+
* keeps one whole profile.
|
|
2760
|
+
*/
|
|
2761
|
+
export async function renameIntent(rt, name, options) {
|
|
2762
|
+
const operation = 'rename';
|
|
2763
|
+
const remaining = await preflight(rt, operation);
|
|
2764
|
+
assertMember(rt, operation);
|
|
2765
|
+
assertSendGateOpen(rt, operation);
|
|
2766
|
+
if (options !== undefined && (typeof options !== 'object' || options === null)) {
|
|
2767
|
+
throw new MarmotError('invalid-options', `${operation}: options must be an object`, { details: { reason: 'malformed-options' } });
|
|
2768
|
+
}
|
|
2769
|
+
const description = options?.description;
|
|
2770
|
+
assertGroupProfile(operation, name, description);
|
|
2771
|
+
const selfPubkey = await callSignerForGroup(rt, () => rt.ctx.signer.getPublicKey());
|
|
2772
|
+
assertNotDisposed(rt);
|
|
2773
|
+
assertLocalAdmin(rt, operation, selfPubkey);
|
|
2774
|
+
assertNoOwnCommitPending(rt, remaining, operation);
|
|
2775
|
+
await applyProposalPolicy(rt, operation);
|
|
2776
|
+
assertNoStagedComponentUpdate(rt.engine.state, profileUpdateGuard(rt.groupId, operation));
|
|
2777
|
+
// One snapshot, taken after every await above: the call-time description and the record's parent epoch come from the same state.
|
|
2778
|
+
const snapshot = rt.engine.state;
|
|
2779
|
+
const profile = { name: name, description: description ?? getMarmotGroupView(snapshot)?.description ?? '' };
|
|
2780
|
+
const record = await recordIntent(rt, 'commit', { type: 'commit', profile }, 'pending-publish', snapshot.groupContext.epoch);
|
|
2781
|
+
await driveIntent(rt, record, true);
|
|
2782
|
+
}
|
|
2783
|
+
/** `leave()`'s proposal: the local leaf's SelfRemove, a durable proposal intent (a peer commits it). */
|
|
2784
|
+
export async function proposeSelfRemoveIntent(rt) {
|
|
2785
|
+
const remaining = await preflight(rt, 'leave');
|
|
2786
|
+
assertMember(rt, 'leave');
|
|
2787
|
+
assertSendGateOpen(rt, 'leave');
|
|
2788
|
+
assertNoOwnCommitPending(rt, remaining, 'leave');
|
|
2789
|
+
const record = await recordIntent(rt, 'proposal', { type: 'proposal', proposal: { proposalType: selfRemoveProposalType } });
|
|
2790
|
+
await driveIntent(rt, record, true);
|
|
2791
|
+
}
|
|
2792
|
+
export function electionOf(state) {
|
|
2793
|
+
return { epoch: state.groupContext.epoch, refs: Object.keys(state.unappliedProposals).sort().join(',') };
|
|
2794
|
+
}
|
|
2795
|
+
/**
|
|
2796
|
+
* The staged-proposal SelfRemove election, independent of who is elected (S5 AC-RACE-2): `undefined` unless the
|
|
2797
|
+
* staged set is a non-empty batch of ONLY SelfRemove proposals with a readable sender leaf and no active-admin
|
|
2798
|
+
* leaver. `committerLeafIndex` is the eligible member `decideAutoCommit` would pick (lowest leaf index among
|
|
2799
|
+
* every current member but the leavers) — reusing the engine's own rule rather than re-deriving the
|
|
2800
|
+
* lowest-index pick locally, so the two can never drift apart.
|
|
2801
|
+
*/
|
|
2802
|
+
function stagedSelfRemoveElection(state, isActiveAdminLeaver) {
|
|
2803
|
+
if (state.groupActiveState.kind === 'removedFromGroup')
|
|
2804
|
+
return undefined;
|
|
2805
|
+
const staged = Object.values(state.unappliedProposals);
|
|
2806
|
+
if (staged.length === 0 || !staged.every((p) => p.proposal.proposalType === selfRemoveProposalType))
|
|
2807
|
+
return undefined;
|
|
2808
|
+
const leaverLeafIndices = [];
|
|
2809
|
+
for (const p of staged) {
|
|
2810
|
+
if (p.senderLeafIndex === undefined)
|
|
2811
|
+
return undefined;
|
|
2812
|
+
leaverLeafIndices.push(Number(p.senderLeafIndex));
|
|
2813
|
+
}
|
|
2814
|
+
const leavers = new Set(leaverLeafIndices);
|
|
2815
|
+
const adminPubkeys = getMarmotGroupView(state)?.adminPubkeys ?? [];
|
|
2816
|
+
const anyLeaverIsActiveAdmin = leaverLeafIndices.some((leaverLeafIndex) => {
|
|
2817
|
+
try {
|
|
2818
|
+
return isActiveAdminLeaver({ ratchetTree: state.ratchetTree, adminPubkeys, leaverLeafIndex, leaverLeafIndices });
|
|
2819
|
+
}
|
|
2820
|
+
catch {
|
|
2821
|
+
return true;
|
|
2822
|
+
}
|
|
2823
|
+
});
|
|
2824
|
+
if (anyLeaverIsActiveAdmin)
|
|
2825
|
+
return undefined;
|
|
2826
|
+
const memberLeafIndices = [];
|
|
2827
|
+
for (let i = 0; i < state.ratchetTree.length; i += 2)
|
|
2828
|
+
if (state.ratchetTree[i]?.nodeType === nodeTypes.leaf)
|
|
2829
|
+
memberLeafIndices.push(i / 2);
|
|
2830
|
+
const eligible = memberLeafIndices.filter((i) => !leavers.has(i));
|
|
2831
|
+
if (eligible.length === 0)
|
|
2832
|
+
return undefined;
|
|
2833
|
+
const committerLeafIndex = Math.min(...eligible);
|
|
2834
|
+
const decision = decideAutoCommit({ leaverLeafIndices, ownLeafIndex: committerLeafIndex, memberLeafIndices, anyLeaverIsActiveAdmin });
|
|
2835
|
+
return decision === 'commit' ? { ...electionOf(state), committerLeafIndex } : undefined;
|
|
2836
|
+
}
|
|
2837
|
+
/**
|
|
2838
|
+
* The engine's SelfRemove election re-run over a state at rest (S9 Z2): a group
|
|
2839
|
+
* loaded with SelfRemoves already staged — the ingest that staged them was
|
|
2840
|
+
* persisted, but the process ended before its commit was recorded — would
|
|
2841
|
+
* otherwise wait for unrelated traffic to re-elect. Same inputs and rule as
|
|
2842
|
+
* the engine's `#maybeAutoCommitSelfRemoves` (its exported `decideAutoCommit`,
|
|
2843
|
+
* the leaver predicate the engine was built with; a throwing predicate is
|
|
2844
|
+
* fail-closed); `undefined` unless this client is elected.
|
|
2845
|
+
*/
|
|
2846
|
+
export function electionAtRest(state, isActiveAdminLeaver) {
|
|
2847
|
+
const election = stagedSelfRemoveElection(state, isActiveAdminLeaver);
|
|
2848
|
+
if (election === undefined || election.committerLeafIndex !== Number(state.privatePath.leafIndex))
|
|
2849
|
+
return undefined;
|
|
2850
|
+
return { epoch: election.epoch, refs: election.refs };
|
|
2851
|
+
}
|
|
2852
|
+
/**
|
|
2853
|
+
* S5 (AC-RACE-2): while the staged proposals are exactly a SelfRemove batch another member is elected to
|
|
2854
|
+
* commit (`stagedSelfRemoveElection`'s `committerLeafIndex` differs from this client's own leaf), an own commit
|
|
2855
|
+
* built now — whatever intent drives it — would bundle those SelfRemoves into it, pre-empting the elected
|
|
2856
|
+
* committer. Waits up to `ctx.selfRemoveElectionWaitMs` (client-clock) from when this exact election ({epoch,
|
|
2857
|
+
* refs}) was first observed; `0` never waits. A genuinely different staged set (the election committed, or
|
|
2858
|
+
* changed) is never still "the same wait" — {@link releaseSelfRemoveElectionWait} clears it and this starts a
|
|
2859
|
+
* fresh one if one still applies.
|
|
2860
|
+
*/
|
|
2861
|
+
function awaitingElectedSelfRemove(rt) {
|
|
2862
|
+
const waitMs = rt.ctx.selfRemoveElectionWaitMs;
|
|
2863
|
+
if (waitMs <= 0)
|
|
2864
|
+
return false;
|
|
2865
|
+
const election = stagedSelfRemoveElection(rt.engine.state, rt.leaverPredicate);
|
|
2866
|
+
const hold = rt.selfRemoveElectionWait;
|
|
2867
|
+
if (election === undefined || election.committerLeafIndex === Number(rt.engine.state.privatePath.leafIndex)) {
|
|
2868
|
+
releaseSelfRemoveElectionWait(rt);
|
|
2869
|
+
return false;
|
|
2870
|
+
}
|
|
2871
|
+
const now = rt.ctx.clock.now();
|
|
2872
|
+
if (hold === undefined || hold.epoch !== election.epoch || hold.refs !== election.refs) {
|
|
2873
|
+
if (hold?.timer !== undefined) {
|
|
2874
|
+
rt.ctx.timers.clear(hold.timer);
|
|
2875
|
+
rt.pipelineTimers.delete(hold.timer);
|
|
2876
|
+
}
|
|
2877
|
+
rt.selfRemoveElectionWait = { epoch: election.epoch, refs: election.refs, startedAt: now, timer: undefined };
|
|
2878
|
+
armElectionWaitTimer(rt, waitMs);
|
|
2879
|
+
return true;
|
|
2880
|
+
}
|
|
2881
|
+
// AC-RACE-2: the timer already fired and cleared itself (`timer === undefined`) while the
|
|
2882
|
+
// injected `ClientClock` still shows time left — a lagging clock, an independent timers implementation,
|
|
2883
|
+
// or Node firing a touch early. Without re-arming, the gate would stay `true` forever with nothing left
|
|
2884
|
+
// to release it: the same early-fire risk `scheduleRemovalSettle` (ingest.ts) re-arms for.
|
|
2885
|
+
if (hold.timer === undefined && now - hold.startedAt < waitMs)
|
|
2886
|
+
armElectionWaitTimer(rt, waitMs);
|
|
2887
|
+
return now - hold.startedAt < waitMs;
|
|
2888
|
+
}
|
|
2889
|
+
/**
|
|
2890
|
+
* Arms (or re-arms) the release timer for `rt.selfRemoveElectionWait`'s CURRENT hold, for whatever time
|
|
2891
|
+
* remains of `waitMs` since it started. If the timer fires before the injected clock agrees `waitMs` has
|
|
2892
|
+
* actually elapsed, it re-arms itself for the remainder instead of releasing — `scheduleRemovalSettle`'s own
|
|
2893
|
+
* re-arm-on-early-fire pattern (ingest.ts), applied here to the election wait's hold.
|
|
2894
|
+
*/
|
|
2895
|
+
/** The largest delay a platform timer honours (2^31 - 1 ms); `setTimeout` fires a longer one after 1 ms. */
|
|
2896
|
+
export const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
2897
|
+
function armElectionWaitTimer(rt, waitMs) {
|
|
2898
|
+
const hold = rt.selfRemoveElectionWait;
|
|
2899
|
+
if (hold === undefined)
|
|
2900
|
+
return;
|
|
2901
|
+
// Clamped to the platform timer limit (a longer delay fires at once in Node): the early-fire branch below
|
|
2902
|
+
// re-arms at the limit until the clock catches up (`early-timer-rearm-pattern`).
|
|
2903
|
+
const remaining = Math.min(MAX_TIMER_DELAY_MS, Math.max(1, waitMs - (rt.ctx.clock.now() - hold.startedAt)));
|
|
2904
|
+
const timer = rt.ctx.timers.set(() => {
|
|
2905
|
+
rt.pipelineTimers.delete(timer);
|
|
2906
|
+
const fired = rt.selfRemoveElectionWait;
|
|
2907
|
+
if (fired === undefined || fired.timer !== timer)
|
|
2908
|
+
return; // superseded or released meanwhile
|
|
2909
|
+
if (rt.ctx.clock.now() - fired.startedAt < waitMs) {
|
|
2910
|
+
armElectionWaitTimer(rt, waitMs); // fired early: re-arm for what's actually left
|
|
2911
|
+
return;
|
|
2912
|
+
}
|
|
2913
|
+
rt.selfRemoveElectionWait = { epoch: fired.epoch, refs: fired.refs, startedAt: fired.startedAt, timer: undefined };
|
|
2914
|
+
rt.scheduleQueueDrain();
|
|
2915
|
+
}, remaining);
|
|
2916
|
+
rt.pipelineTimers.add(timer);
|
|
2917
|
+
rt.selfRemoveElectionWait = { ...hold, timer };
|
|
2918
|
+
}
|
|
2919
|
+
/** Cancels any pending {@link awaitingElectedSelfRemove} timer without blocking a fresh evaluation — a genuine staged-proposal change (ingest.ts). */
|
|
2920
|
+
export function releaseSelfRemoveElectionWait(rt) {
|
|
2921
|
+
const hold = rt.selfRemoveElectionWait;
|
|
2922
|
+
if (hold === undefined)
|
|
2923
|
+
return;
|
|
2924
|
+
if (hold.timer !== undefined) {
|
|
2925
|
+
rt.ctx.timers.clear(hold.timer);
|
|
2926
|
+
rt.pipelineTimers.delete(hold.timer);
|
|
2927
|
+
}
|
|
2928
|
+
rt.selfRemoveElectionWait = undefined;
|
|
2929
|
+
}
|
|
2930
|
+
/**
|
|
2931
|
+
* AC-ADMIN-9: the engine elected this client to commit the staged SelfRemoves
|
|
2932
|
+
* (`autoCommit`, with the admin-leaver predicate `marmot` injects). Ingest
|
|
2933
|
+
* rolls the engine's own staged commit back — it is not durable — and this
|
|
2934
|
+
* schedules, on the group's queue after the ingest, ONE durable commit of the
|
|
2935
|
+
* same staged set through the pipeline, provided that set is still exactly
|
|
2936
|
+
* what the engine decided on. A failure before anything was recorded is
|
|
2937
|
+
* retried on the capped backoff; afterwards the record retries itself.
|
|
2938
|
+
*/
|
|
2939
|
+
export function scheduleSelfRemoveCommit(rt, election, failures = 0) {
|
|
2940
|
+
const run = () => {
|
|
2941
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
2942
|
+
return;
|
|
2943
|
+
rt.run('selfRemoveCommit', (rtc) => commitElectedSelfRemoves(rtc, election)).catch((error) => {
|
|
2944
|
+
if (!backgroundRetryEnds(rt, error) && !isMarmotError(error, 'not-a-member'))
|
|
2945
|
+
scheduleSelfRemoveCommit(rt, election, failures + 1);
|
|
2946
|
+
});
|
|
2947
|
+
};
|
|
2948
|
+
if (failures === 0)
|
|
2949
|
+
run();
|
|
2950
|
+
else
|
|
2951
|
+
setPipelineTimer(rt, run, backoffMs(failures));
|
|
2952
|
+
}
|
|
2953
|
+
async function commitElectedSelfRemoves(rt, election) {
|
|
2954
|
+
// AC-PIPE-15: before the first post-load sync nothing is minted; releaseAwaitingSync re-runs the election at rest.
|
|
2955
|
+
if (rt.fence !== undefined || !rt.sendGateOpen)
|
|
2956
|
+
return;
|
|
2957
|
+
const current = electionOf(rt.engine.state);
|
|
2958
|
+
// Anything since (another commit, a new proposal) re-ran the engine's election on that ingest.
|
|
2959
|
+
if (current.epoch !== election.epoch || current.refs !== election.refs)
|
|
2960
|
+
return;
|
|
2961
|
+
if (rt.engine.state.groupActiveState.kind === 'removedFromGroup')
|
|
2962
|
+
return;
|
|
2963
|
+
const pending = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(rt.groupStore.artifact('pendingIntents')));
|
|
2964
|
+
if (pending.some((r) => r.state === 'pending-confirm' || isCommitKind(r.kind)))
|
|
2965
|
+
return;
|
|
2966
|
+
if ((await decideStagedProposals(rt)) > 0)
|
|
2967
|
+
return;
|
|
2968
|
+
// S9 C4: onProposal may have rejected part of the batch (dropped from the staged set): the election is re-run over
|
|
2969
|
+
// what is left — the leaver set changed, so the elected committer may have too.
|
|
2970
|
+
if (electionOf(rt.engine.state).refs !== election.refs && electionAtRest(rt.engine.state, rt.leaverPredicate) === undefined)
|
|
2971
|
+
return;
|
|
2972
|
+
await driveIntent(rt, await recordIntent(rt, 'commit', { type: 'commit' }), false);
|
|
2973
|
+
}
|
|
2974
|
+
// ---------------------------------------------------------------------------
|
|
2975
|
+
// S5 — pending-proposal policy (AC-INVITE-4)
|
|
2976
|
+
// ---------------------------------------------------------------------------
|
|
2977
|
+
/**
|
|
2978
|
+
* Asks `onProposal` about every staged foreign proposal and drops the rejected
|
|
2979
|
+
* ones from the group state (persisted, engine rebuilt), so the next own commit
|
|
2980
|
+
* omits them. Returns how many stay deferred.
|
|
2981
|
+
*
|
|
2982
|
+
* S15 (AC-INVITE-4 amended): the deferral deadline is evaluated here, against
|
|
2983
|
+
* the client clock, on every call — every own-commit attempt runs through
|
|
2984
|
+
* this. A proposal past its deadline is not asked about again: it joins the
|
|
2985
|
+
* rejected ones (the one drop path, {@link dropRejectedProposals}), its
|
|
2986
|
+
* record is marked expired in that same transaction, and `proposalExpired`
|
|
2987
|
+
* is emitted once the drop is durable (proposal-deadline.ts: record
|
|
2988
|
+
* lifecycle and emission semantics). `expireOnly` (the group load, history
|
|
2989
|
+
* and inbox passes) evaluates deadlines without asking `onProposal`.
|
|
2990
|
+
* Nothing expires unless `rt.caughtUp` (a complete history pass since load
|
|
2991
|
+
* and since the last connectivity gap); expiries already marked durably are
|
|
2992
|
+
* still announced.
|
|
2993
|
+
*/
|
|
2994
|
+
async function decideStagedProposals(rt, opts) {
|
|
2995
|
+
const { foreign } = stagedProposals(rt.engine.state, rt.groupId);
|
|
2996
|
+
const records = await rt.guard.call(rt.groupId, 'store', () => readDeferrals(rt.groupStore.extension(DEFERRED_PROPOSALS_EXTENSION)));
|
|
2997
|
+
const exclusions = await rt.guard.call(rt.groupId, 'store', () => readExclusions(rt.groupStore.extension(EXCLUDED_PROPOSALS_EXTENSION)));
|
|
2998
|
+
if (foreign.length === 0 && records.length === 0 && exclusions.records.length === 0 && exclusions.snapshots.size === 0)
|
|
2999
|
+
return 0;
|
|
3000
|
+
// Round 2: a proposal already dropped at this epoch (rejected or expired) and staged again — re-staged so a peer's
|
|
3001
|
+
// commit could resolve it, or by a rewind — is dropped again, silently: never asked about, deferred or announced anew.
|
|
3002
|
+
const stateTag = tagOf(rt.engine.state);
|
|
3003
|
+
const { current: excludedNow, outdated } = exclusionsAt(exclusions, rt.engine.state.groupContext.epoch, stateTag, REWIND_HORIZON);
|
|
3004
|
+
const excludedRefs = new Set(excludedNow.map((r) => r.proposalRef));
|
|
3005
|
+
const reExcluded = foreign.filter((p) => excludedRefs.has(p.proposalRef)).map((p) => p.proposalRef);
|
|
3006
|
+
const candidates = foreign.filter((p) => !excludedRefs.has(p.proposalRef));
|
|
3007
|
+
// Lead decision (VQ-S15-101, round 2): no deadline expires unless the group is caught up since the last
|
|
3008
|
+
// connectivity gap (GroupRuntime.caughtUp) — a peer admin's commit of the proposal, fetched by that catch-up,
|
|
3009
|
+
// must resolve it first. Until then a past-deadline proposal stays deferred: timeout 0 = never.
|
|
3010
|
+
const plan = planDeferrals(candidates.map((p) => p.proposalRef), records, rt.ctx.clock.now(), rt.caughtUp ? rt.ctx.proposalDeferralTimeoutMs : 0);
|
|
3011
|
+
const live = new Set(plan.live);
|
|
3012
|
+
const undecided = candidates.filter((p) => live.has(p.proposalRef));
|
|
3013
|
+
// Not asked (expireOnly): nothing is decided, so nothing starts a deferral either.
|
|
3014
|
+
let resolution;
|
|
3015
|
+
const asked = opts?.expireOnly !== true && undecided.length > 0;
|
|
3016
|
+
if (!asked) {
|
|
3017
|
+
resolution = { accepted: [], rejected: [], deferred: plan.live };
|
|
3018
|
+
}
|
|
3019
|
+
else {
|
|
3020
|
+
const onProposal = rt.ctx.onProposal;
|
|
3021
|
+
const ask = onProposal === undefined ? undefined : (proposal) => rt.guard.call(rt.groupId, 'onProposal', () => onProposal(proposal));
|
|
3022
|
+
resolution = await resolveProposals(undecided, ask);
|
|
3023
|
+
// ADR-013: the callback may have disposed the client; the state itself cannot have moved (this op holds the group queue).
|
|
3024
|
+
assertNotDisposed(rt);
|
|
3025
|
+
}
|
|
3026
|
+
const decidedAt = rt.ctx.clock.now();
|
|
3027
|
+
const expired = plan.expired.map((r) => (r.expiredAt === undefined ? { ...r, expiredAt: decidedAt } : r));
|
|
3028
|
+
// A re-deferral keeps its record (the deadline never slides); a first one starts it now.
|
|
3029
|
+
const firstDeferrals = !asked ? [] : resolution.deferred.filter((ref) => !plan.recorded.has(ref)).map((proposalRef) => ({ proposalRef, deferredAt: decidedAt }));
|
|
3030
|
+
// Resolved without expiring (accepted, rejected) or no longer staged: forgotten, never announced.
|
|
3031
|
+
const forgotten = [...plan.stale, ...[...resolution.accepted, ...resolution.rejected].filter((ref) => plan.recorded.has(ref))];
|
|
3032
|
+
const dropped = [...resolution.rejected, ...expired.map((r) => r.proposalRef), ...reExcluded];
|
|
3033
|
+
// Each new drop is recorded with its branch (the state's tag), and that node's snapshot (one per node) merged to
|
|
3034
|
+
// stage it (proposal-deadline.ts, Exclusions).
|
|
3035
|
+
const epoch = String(rt.engine.state.groupContext.epoch);
|
|
3036
|
+
const newExclusions = [
|
|
3037
|
+
...resolution.rejected.map((proposalRef) => ({ proposalRef, reason: 'rejected' })),
|
|
3038
|
+
...expired.map(({ proposalRef }) => ({ proposalRef, reason: 'expired' })),
|
|
3039
|
+
].map((e) => ({ ...e, epoch, tag: stateTag }));
|
|
3040
|
+
let snapshot;
|
|
3041
|
+
if (newExclusions.length > 0) {
|
|
3042
|
+
const previous = exclusions.snapshots.get(stateTag);
|
|
3043
|
+
const merged = previous === undefined ? {} : deserializeClientState(previous).unappliedProposals;
|
|
3044
|
+
snapshot = serializeClientState({ ...rt.engine.state, unappliedProposals: { ...merged, ...rt.engine.state.unappliedProposals } });
|
|
3045
|
+
}
|
|
3046
|
+
const recordWrites = async (tx) => {
|
|
3047
|
+
const view = tx.extension(DEFERRED_PROPOSALS_EXTENSION);
|
|
3048
|
+
for (const ref of forgotten)
|
|
3049
|
+
await deleteDeferral(view, ref);
|
|
3050
|
+
for (const record of [...firstDeferrals, ...expired])
|
|
3051
|
+
await writeDeferral(view, record);
|
|
3052
|
+
const excluded = tx.extension(EXCLUDED_PROPOSALS_EXTENSION);
|
|
3053
|
+
if (outdated.size > 0)
|
|
3054
|
+
await deleteExclusionEpochs(excluded, exclusions, outdated);
|
|
3055
|
+
for (const record of newExclusions)
|
|
3056
|
+
await writeExclusion(excluded, record);
|
|
3057
|
+
if (snapshot !== undefined)
|
|
3058
|
+
await writeExclusionSnapshot(excluded, stateTag, snapshot);
|
|
3059
|
+
};
|
|
3060
|
+
if (dropped.length > 0) {
|
|
3061
|
+
await dropRejectedProposals(rt, dropped, recordWrites);
|
|
3062
|
+
// The staged set changed durably: the queue's commit may go ahead now. Scheduled before the announcement,
|
|
3063
|
+
// whose marker cleanup may still fail — nothing else would change the staged set again to trigger a drain.
|
|
3064
|
+
releaseQueueCommitHold(rt);
|
|
3065
|
+
rt.scheduleQueueDrain();
|
|
3066
|
+
}
|
|
3067
|
+
else if (forgotten.length + firstDeferrals.length + outdated.size > 0) {
|
|
3068
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(recordWrites));
|
|
3069
|
+
assertNotDisposed(rt);
|
|
3070
|
+
}
|
|
3071
|
+
await announceExpired(rt, [...plan.owed, ...expired]);
|
|
3072
|
+
return resolution.deferred.length;
|
|
3073
|
+
}
|
|
3074
|
+
/**
|
|
3075
|
+
* Emits `proposalExpired` for each record (its drop is already durable), then
|
|
3076
|
+
* deletes the records: a crash before the deletion re-announces them on the
|
|
3077
|
+
* next evaluation (at-least-once; proposal-deadline.ts).
|
|
3078
|
+
*/
|
|
3079
|
+
async function announceExpired(rt, records) {
|
|
3080
|
+
if (records.length === 0)
|
|
3081
|
+
return;
|
|
3082
|
+
// A client disposed since the drop (a crash, in effect) emits nothing and keeps the markers for the reload.
|
|
3083
|
+
assertNotDisposed(rt);
|
|
3084
|
+
for (const { proposalRef, deferredAt } of records)
|
|
3085
|
+
rt.ctx.emit('proposalExpired', { groupId: rt.groupId, proposalRef, deferredAt });
|
|
3086
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
3087
|
+
const view = tx.extension(DEFERRED_PROPOSALS_EXTENSION);
|
|
3088
|
+
for (const { proposalRef } of records)
|
|
3089
|
+
await deleteDeferral(view, proposalRef);
|
|
3090
|
+
}));
|
|
3091
|
+
}
|
|
3092
|
+
/**
|
|
3093
|
+
* S15: evaluates the deferral deadlines of the group's staged proposals
|
|
3094
|
+
* without asking `onProposal` (an expired one is rejected, see
|
|
3095
|
+
* {@link decideStagedProposals}); when one expired, the queue drains — the
|
|
3096
|
+
* admin's commit that releases the queued rumors is no longer refused.
|
|
3097
|
+
* Runs inside a group op.
|
|
3098
|
+
*/
|
|
3099
|
+
export async function expireDeferredProposals(rt) {
|
|
3100
|
+
if (rt.fence !== undefined || rt.engine.state.groupActiveState.kind === 'removedFromGroup')
|
|
3101
|
+
return;
|
|
3102
|
+
const before = rt.engine.state.unappliedProposals;
|
|
3103
|
+
await decideStagedProposals(rt, { expireOnly: true });
|
|
3104
|
+
if (rt.engine.state.unappliedProposals !== before) {
|
|
3105
|
+
releaseQueueCommitHold(rt);
|
|
3106
|
+
rt.scheduleQueueDrain();
|
|
3107
|
+
}
|
|
3108
|
+
}
|
|
3109
|
+
/**
|
|
3110
|
+
* Schedules {@link expireDeferredProposals} on the group's queue (group load,
|
|
3111
|
+
* inbox pass). A failure is dropped: the deadline is durable, and the next
|
|
3112
|
+
* evaluation point evaluates it again. Never throws.
|
|
3113
|
+
*/
|
|
3114
|
+
export function scheduleProposalDeadlineCheck(rt) {
|
|
3115
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
3116
|
+
return;
|
|
3117
|
+
void Promise.resolve()
|
|
3118
|
+
.then(() => rt.run('proposalDeadline', (rtc) => expireDeferredProposals(rtc)))
|
|
3119
|
+
.catch(() => undefined);
|
|
3120
|
+
}
|
|
3121
|
+
/**
|
|
3122
|
+
* The policy in front of an own commit: a deferred proposal refuses it (`commit-pending`, reason
|
|
3123
|
+
* `proposal-deferred`); S5 (AC-RACE-2): a staged SelfRemove-only batch another member is elected to commit
|
|
3124
|
+
* refuses it too (reason `selfremove-election-wait`), checked first — never asking `onProposal` about a
|
|
3125
|
+
* foreign proposal while the commit is going to be refused anyway.
|
|
3126
|
+
*/
|
|
3127
|
+
async function applyProposalPolicy(rt, operation) {
|
|
3128
|
+
if (awaitingElectedSelfRemove(rt)) {
|
|
3129
|
+
throw new MarmotError('commit-pending', `${operation}: group ${rt.groupId}'s staged proposals are a SelfRemove batch another member is elected to commit; waiting up to ${String(rt.ctx.selfRemoveElectionWaitMs)}ms`, {
|
|
3130
|
+
details: { groupId: rt.groupId, reason: 'selfremove-election-wait', retryable: true },
|
|
3131
|
+
});
|
|
3132
|
+
}
|
|
3133
|
+
const deferred = await decideStagedProposals(rt);
|
|
3134
|
+
if (deferred > 0) {
|
|
3135
|
+
throw new MarmotError('commit-pending', `${operation}: ${deferred} staged proposal(s) of group ${rt.groupId} are deferred by onProposal`, {
|
|
3136
|
+
details: { groupId: rt.groupId, reason: 'proposal-deferred', deferred, retryable: true },
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
/**
|
|
3141
|
+
* Marks `refs` invalid under application policy: removed from the staged set, durably, before any commit is built.
|
|
3142
|
+
* `alongside` runs in the same transaction (S15: the deferral records of the drop).
|
|
3143
|
+
*/
|
|
3144
|
+
export async function dropRejectedProposals(rt, refs, alongside) {
|
|
3145
|
+
const next = withoutProposals(rt.engine.state, refs);
|
|
3146
|
+
const pooled = rt.engine.pendingEnvelopes();
|
|
3147
|
+
const evidence = [];
|
|
3148
|
+
let fence;
|
|
3149
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
3150
|
+
await writeGroupState(tx.artifact('clientState'), next);
|
|
3151
|
+
await alongside?.(tx);
|
|
3152
|
+
}));
|
|
3153
|
+
assertNotDisposed(rt);
|
|
3154
|
+
try {
|
|
3155
|
+
// A fresh copy: never hand one ClientState object to two engines (ts-mls mutates it in place).
|
|
3156
|
+
await rt.rebuildEngine(deserializeClientState(serializeClientState(next)));
|
|
3157
|
+
for await (const result of rt.engine.ingest([...pooled])) {
|
|
3158
|
+
if (result.kind === 'autoCommit')
|
|
3159
|
+
rt.engine.publishFailed(result.pending);
|
|
3160
|
+
else if (result.kind === 'leafEquivocation')
|
|
3161
|
+
evidence.push(result.evidence);
|
|
3162
|
+
// A held opaque candidate this pass decides to fence (ADR-016 D6): its engine record is released once the verdict
|
|
3163
|
+
// is yielded, so the verdict is persisted here, not dropped with the rest of the discarded results.
|
|
3164
|
+
else if (result.kind === 'unreadable')
|
|
3165
|
+
fence ??= fencingErrorIn(result.errors);
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
catch (error) {
|
|
3169
|
+
rt.needsReload = true;
|
|
3170
|
+
throw error;
|
|
3171
|
+
}
|
|
3172
|
+
rt.exposedState = rt.engine.state;
|
|
3173
|
+
// ADR-016: the re-ingest's selection pass reports equivocation evidence like any drain; recorded here, best-effort:
|
|
3174
|
+
// the drop is already durable, and a refused evidence write must not fail it — the next pass over the fork
|
|
3175
|
+
// re-reports the evidence (as restorePreBatchState does).
|
|
3176
|
+
await recordEquivocations(rt, evidence).catch(() => undefined);
|
|
3177
|
+
if (fence !== undefined)
|
|
3178
|
+
await fenceGroup(rt, fence);
|
|
3179
|
+
}
|
|
3180
|
+
function createQueueWaiter() {
|
|
3181
|
+
let resolve;
|
|
3182
|
+
let reject;
|
|
3183
|
+
const promise = new Promise((res, rej) => {
|
|
3184
|
+
resolve = res;
|
|
3185
|
+
reject = rej;
|
|
3186
|
+
});
|
|
3187
|
+
// Settled while no one awaits it yet (inside the op) must not count as unhandled.
|
|
3188
|
+
promise.catch(() => undefined);
|
|
3189
|
+
return { promise, resolve, reject };
|
|
3190
|
+
}
|
|
3191
|
+
function takeWaiter(rt, queueId) {
|
|
3192
|
+
const waiter = rt.queueWaiters.get(queueId);
|
|
3193
|
+
rt.queueWaiters.delete(queueId);
|
|
3194
|
+
return waiter;
|
|
3195
|
+
}
|
|
3196
|
+
/**
|
|
3197
|
+
* Queues `rumor` durably, then advances the queue (which may commit and publish
|
|
3198
|
+
* it right away). The outcome carries the caller's wait as a promise inside an
|
|
3199
|
+
* object: an async function returning the bare promise would adopt it, and the
|
|
3200
|
+
* group op would wait on its own queue.
|
|
3201
|
+
*/
|
|
3202
|
+
async function enqueueRumor(rt, rumor, queued) {
|
|
3203
|
+
const last = queued.at(-1);
|
|
3204
|
+
const barrier = queueBarrier(rt.engine.state, last?.barrier);
|
|
3205
|
+
const record = {
|
|
3206
|
+
v: 1,
|
|
3207
|
+
queueId: randomIntentId(),
|
|
3208
|
+
seq: (last?.seq ?? 0) + 1,
|
|
3209
|
+
rumor: rumor,
|
|
3210
|
+
...(barrier === undefined ? {} : { barrier }),
|
|
3211
|
+
queuedAt: rt.ctx.clock.now(),
|
|
3212
|
+
};
|
|
3213
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction((tx) => writeQueuedRumorRecord(tx.artifact('queuedRumors'), record)));
|
|
3214
|
+
assertNotDisposed(rt);
|
|
3215
|
+
const waiter = createQueueWaiter();
|
|
3216
|
+
rt.queueWaiters.set(record.queueId, waiter);
|
|
3217
|
+
rt.queuedHint = queued.length + 1;
|
|
3218
|
+
let commitError;
|
|
3219
|
+
try {
|
|
3220
|
+
({ commitError } = await advanceQueue(rt));
|
|
3221
|
+
}
|
|
3222
|
+
catch (error) {
|
|
3223
|
+
// The rumor stays queued durably; this caller learns why its own drain stopped, and — the
|
|
3224
|
+
// group may stay quiet — a background drain is scheduled on the backoff so it still goes out.
|
|
3225
|
+
takeWaiter(rt, record.queueId);
|
|
3226
|
+
if (!backgroundRetryEnds(rt, error))
|
|
3227
|
+
rt.scheduleQueueDrainRetry();
|
|
3228
|
+
throw error;
|
|
3229
|
+
}
|
|
3230
|
+
// A commit that could not even be built or recorded (a signer or store fault) left nothing to
|
|
3231
|
+
// retry it: the backoff drain does. (A terminal one is held; an unpublished one retries itself.)
|
|
3232
|
+
if (commitError !== undefined && !isMarmotError(commitError, 'send-rejected') && !isMarmotError(commitError, 'publish-failed'))
|
|
3233
|
+
rt.scheduleQueueDrainRetry();
|
|
3234
|
+
// The commit this call tried to make failed: the caller is told (the rumor stays queued
|
|
3235
|
+
// and still goes out after the next confirmed commit).
|
|
3236
|
+
if (commitError !== undefined && rt.queueWaiters.get(record.queueId) === waiter)
|
|
3237
|
+
takeWaiter(rt, record.queueId)?.reject(commitError);
|
|
3238
|
+
return { queued: waiter.promise };
|
|
3239
|
+
}
|
|
3240
|
+
/**
|
|
3241
|
+
* Publishes queued rumors, head first, as long as nothing is staged and the
|
|
3242
|
+
* epoch is past the head's barrier; builds the commit that clears the way
|
|
3243
|
+
* itself when the local account is an authorised committer and no proposal is
|
|
3244
|
+
* deferred. Runs inside a group op. `commitError` is the failure of the
|
|
3245
|
+
* commit it tried to make, if any (the queue then stays as it is).
|
|
3246
|
+
*/
|
|
3247
|
+
export async function advanceQueue(rt) {
|
|
3248
|
+
const queuedView = rt.groupStore.artifact('queuedRumors');
|
|
3249
|
+
const pendingView = rt.groupStore.artifact('pendingIntents');
|
|
3250
|
+
// A commit or head intent a failed earlier drain left behind (e.g. envelope-less after a signer or
|
|
3251
|
+
// store fault), or a blocked confirmation, is resumed through the ordinary path before any new
|
|
3252
|
+
// commit is considered. A published intent awaiting a publish retry keeps its own timer.
|
|
3253
|
+
// A fenced group sends nothing, queued rumors included (they stay queued; `send` callers learn `fenced`).
|
|
3254
|
+
// AC-PIPE-15: nor anything before the group's first post-load sync (releaseAwaitingSync drains it then).
|
|
3255
|
+
if (rt.fence !== undefined || !rt.sendGateOpen)
|
|
3256
|
+
return {};
|
|
3257
|
+
const heldByElectionOrProposal = await resumePending(rt, { unpublishedOnly: true });
|
|
3258
|
+
for (;;) {
|
|
3259
|
+
assertNotDisposed(rt);
|
|
3260
|
+
const queued = await rt.guard.call(rt.groupId, 'store', () => listQueuedRumorRecords(queuedView));
|
|
3261
|
+
// A record `resumePending` just held back for the election wait or a deferred proposal needs
|
|
3262
|
+
// the NEXT trigger to re-evaluate for real — never overwrite that with the rumor queue's own (possibly
|
|
3263
|
+
// zero) length, or the exact same `queuedHint === 0` short-circuit this fixes would just reappear here.
|
|
3264
|
+
rt.queuedHint = heldByElectionOrProposal ? undefined : queued.length;
|
|
3265
|
+
const head = queued[0];
|
|
3266
|
+
if (head === undefined)
|
|
3267
|
+
return {};
|
|
3268
|
+
if (rt.engine.state.groupActiveState.kind === 'removedFromGroup') {
|
|
3269
|
+
// S20: held, not dropped, while the removal is revisable — a winning sibling may still bring the leaf back
|
|
3270
|
+
// (nothing is sent either way); dropped by the drain that follows its settling (client/group-removal.ts).
|
|
3271
|
+
if (rt.removal !== undefined)
|
|
3272
|
+
return {};
|
|
3273
|
+
await dropQueueOfRemovedMember(rt, queued);
|
|
3274
|
+
return {};
|
|
3275
|
+
}
|
|
3276
|
+
const pending = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(pendingView));
|
|
3277
|
+
// A blocked confirmation's retry drains the queue once it clears; an unconfirmed application
|
|
3278
|
+
// intent (a released head still being retried) goes out before anything queued behind it —
|
|
3279
|
+
// its confirmation drains the queue again.
|
|
3280
|
+
if (pending.some((r) => r.state === 'pending-confirm' || r.kind === 'application'))
|
|
3281
|
+
return {};
|
|
3282
|
+
const selfPubkey = await callSignerForGroup(rt, () => rt.ctx.signer.getPublicKey());
|
|
3283
|
+
assertNotDisposed(rt);
|
|
3284
|
+
const step = nextQueueStep({
|
|
3285
|
+
state: rt.engine.state,
|
|
3286
|
+
barrier: head.barrier,
|
|
3287
|
+
authorised: isAuthorisedCommitter(rt.engine.state, selfPubkey),
|
|
3288
|
+
ownCommitPending: pending.some((r) => isCommitKind(r.kind)),
|
|
3289
|
+
});
|
|
3290
|
+
if (step === 'wait')
|
|
3291
|
+
return {};
|
|
3292
|
+
if (step === 'commit') {
|
|
3293
|
+
const hold = rt.queueCommitHold;
|
|
3294
|
+
if (hold?.timer !== undefined && hold.epoch === rt.engine.state.groupContext.epoch)
|
|
3295
|
+
return {};
|
|
3296
|
+
// S5 (AC-RACE-2): the queue's own commit must not bundle a staged SelfRemove-only batch another member
|
|
3297
|
+
// is elected to commit — wait like any other hold; `awaitingElectedSelfRemove`'s own timer / a genuine
|
|
3298
|
+
// staged-set change (ingest.ts) drains the queue again once it no longer applies.
|
|
3299
|
+
if (awaitingElectedSelfRemove(rt))
|
|
3300
|
+
return {};
|
|
3301
|
+
if ((await decideStagedProposals(rt)) > 0)
|
|
3302
|
+
return {};
|
|
3303
|
+
const epochBefore = rt.engine.state.groupContext.epoch;
|
|
3304
|
+
try {
|
|
3305
|
+
await driveIntent(rt, await recordIntent(rt, 'commit', { type: 'commit' }), true);
|
|
3306
|
+
}
|
|
3307
|
+
catch (error) {
|
|
3308
|
+
if (backgroundRetryEnds(rt, error))
|
|
3309
|
+
throw error;
|
|
3310
|
+
// The queue's own commit went terminal: surfaced like any intent that went terminal with no caller of its own.
|
|
3311
|
+
if (isMarmotError(error, 'send-rejected')) {
|
|
3312
|
+
rt.ctx.emit('intentFailed', { groupId: rt.groupId, intentId: String(error.details?.intentId ?? ''), error });
|
|
3313
|
+
}
|
|
3314
|
+
return { commitError: error };
|
|
3315
|
+
}
|
|
3316
|
+
// No epoch progress (the confirmation is blocked): its retry drains the queue.
|
|
3317
|
+
if (rt.engine.state.groupContext.epoch === epochBefore)
|
|
3318
|
+
return {};
|
|
3319
|
+
rt.queueCommitHold = undefined;
|
|
3320
|
+
continue;
|
|
3321
|
+
}
|
|
3322
|
+
// Queue order: a head that is not confirmed (its publish is being retried) holds the rest.
|
|
3323
|
+
if (!(await publishQueued(rt, head)))
|
|
3324
|
+
return {};
|
|
3325
|
+
}
|
|
3326
|
+
}
|
|
3327
|
+
/**
|
|
3328
|
+
* Moves the head rumor into an application intent record — one transaction,
|
|
3329
|
+
* re-reading the queued record first, so the rumor is never lost nor published
|
|
3330
|
+
* twice — and drives it; its `send()` caller, if still waiting, is settled.
|
|
3331
|
+
* Returns whether the drain may go on: `false` while the head's intent is
|
|
3332
|
+
* still pending (not published yet — the rest of the queue waits for it).
|
|
3333
|
+
*/
|
|
3334
|
+
async function publishQueued(rt, head) {
|
|
3335
|
+
const moved = await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
3336
|
+
const queuedView = tx.artifact('queuedRumors');
|
|
3337
|
+
if ((await readQueuedRumorRecord(queuedView, head.queueId)) === undefined)
|
|
3338
|
+
return undefined;
|
|
3339
|
+
const pendingView = tx.artifact('pendingIntents');
|
|
3340
|
+
const record = {
|
|
3341
|
+
v: 1,
|
|
3342
|
+
intentId: head.queueId,
|
|
3343
|
+
seq: await nextSeq(pendingView),
|
|
3344
|
+
kind: 'application',
|
|
3345
|
+
state: 'pending-publish',
|
|
3346
|
+
parentEpoch: rt.engine.state.groupContext.epoch,
|
|
3347
|
+
intent: { type: 'application', rumor: head.rumor },
|
|
3348
|
+
attempts: 0,
|
|
3349
|
+
};
|
|
3350
|
+
await queuedView.delete(head.queueId);
|
|
3351
|
+
await writePendingIntentRecord(pendingView, record);
|
|
3352
|
+
return record;
|
|
3353
|
+
}));
|
|
3354
|
+
if (moved === undefined)
|
|
3355
|
+
return true;
|
|
3356
|
+
assertNotDisposed(rt);
|
|
3357
|
+
const waiter = takeWaiter(rt, head.queueId);
|
|
3358
|
+
try {
|
|
3359
|
+
await driveIntent(rt, moved, waiter !== undefined);
|
|
3360
|
+
// With a caller, driveIntent returns only once a relay accepted the rumor.
|
|
3361
|
+
waiter?.resolve();
|
|
3362
|
+
}
|
|
3363
|
+
catch (error) {
|
|
3364
|
+
waiter?.reject(error);
|
|
3365
|
+
// A refused or unpublished rumor is its caller's failure (the record is retried or dropped as for any send); anything else stops the drain.
|
|
3366
|
+
if (waiter === undefined || !(isMarmotError(error, 'publish-failed') || isMarmotError(error, 'send-rejected')))
|
|
3367
|
+
throw error;
|
|
3368
|
+
}
|
|
3369
|
+
// Still recorded = not confirmed yet (a publish retry, or a blocked confirmation): the rest waits for its confirmation.
|
|
3370
|
+
return !(await stillPending(rt, moved.intentId));
|
|
3371
|
+
}
|
|
3372
|
+
async function stillPending(rt, intentId) {
|
|
3373
|
+
return (await rt.guard.call(rt.groupId, 'store', () => readPendingIntentRecord(rt.groupStore.artifact('pendingIntents'), intentId))) !== undefined;
|
|
3374
|
+
}
|
|
3375
|
+
/**
|
|
3376
|
+
* Queued rumors of a group that removed us can never be sent: dropped, and each reported `not-a-member` — to its
|
|
3377
|
+
* waiting `send()` caller, or, with none left (queued before a reload, or released into the queue from
|
|
3378
|
+
* awaiting-sync after its `send()` resolved, AC-PIPE-15), through `intentFailed`.
|
|
3379
|
+
*/
|
|
3380
|
+
async function dropQueueOfRemovedMember(rt, queued) {
|
|
3381
|
+
await dropQueuedRumors(rt, queued, (record) => new MarmotError('not-a-member', `send: the local leaf was removed from group ${rt.groupId} before the queued rumor could be sent`, {
|
|
3382
|
+
details: { groupId: rt.groupId, intentId: record.queueId },
|
|
3383
|
+
}));
|
|
3384
|
+
}
|
|
3385
|
+
/** Deletes `queued` in one transaction, then reports each one's `error` to its waiting caller, else via `intentFailed`. */
|
|
3386
|
+
async function dropQueuedRumors(rt, queued, errorOf) {
|
|
3387
|
+
if (queued.length === 0)
|
|
3388
|
+
return;
|
|
3389
|
+
await rt.guard.call(rt.groupId, 'store', () => rt.groupStore.transaction(async (tx) => {
|
|
3390
|
+
for (const record of queued)
|
|
3391
|
+
await tx.artifact('queuedRumors').delete(record.queueId);
|
|
3392
|
+
}));
|
|
3393
|
+
rt.queuedHint = 0;
|
|
3394
|
+
for (const record of queued) {
|
|
3395
|
+
const error = errorOf(record);
|
|
3396
|
+
const waiter = takeWaiter(rt, record.queueId);
|
|
3397
|
+
if (waiter !== undefined)
|
|
3398
|
+
waiter.reject(error);
|
|
3399
|
+
else
|
|
3400
|
+
rt.ctx.emit('intentFailed', { groupId: rt.groupId, intentId: record.queueId, error });
|
|
3401
|
+
}
|
|
3402
|
+
}
|
|
3403
|
+
/**
|
|
3404
|
+
* Runs {@link resumePending} on the group's queue in the background (group
|
|
3405
|
+
* load). A whole-run failure — a transient store read — is retried on the
|
|
3406
|
+
* capped backoff until it succeeds, the runtime or client is disposed, or the
|
|
3407
|
+
* stores lose their lease (the next holder resumes the durable records). A
|
|
3408
|
+
* throwing relay probe no longer fails the run: it goes to the record's
|
|
3409
|
+
* bounded publish retry ({@link settlePublishedRecord}).
|
|
3410
|
+
*/
|
|
3411
|
+
export function resumePendingInBackground(rt, failures = 0) {
|
|
3412
|
+
const run = () => {
|
|
3413
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
3414
|
+
return;
|
|
3415
|
+
rt.run('resumePending', (rtc) => resumePending(rtc)).catch((error) => {
|
|
3416
|
+
if (!backgroundRetryEnds(rt, error))
|
|
3417
|
+
resumePendingInBackground(rt, failures + 1);
|
|
3418
|
+
});
|
|
3419
|
+
};
|
|
3420
|
+
if (failures === 0)
|
|
3421
|
+
run();
|
|
3422
|
+
else
|
|
3423
|
+
setPipelineTimer(rt, run, backoffMs(failures));
|
|
3424
|
+
}
|
|
3425
|
+
/**
|
|
3426
|
+
* `unpublishedOnly`: resume only records nothing was published for yet
|
|
3427
|
+
* (envelope-less) and blocked confirmations — what a queue drain needs
|
|
3428
|
+
* before it may build a commit — and leave records awaiting a publish retry
|
|
3429
|
+
* to their own backoff timer (a busy group must not spend their
|
|
3430
|
+
* `maxPublishAttempts` faster).
|
|
3431
|
+
*/
|
|
3432
|
+
/**
|
|
3433
|
+
* Returns `true` when any record was left behind specifically because `awaitingElectedSelfRemove` (AC-RACE-2),
|
|
3434
|
+
* a deferred foreign proposal (AC-INVITE-4) or a revisable removal (a re-enqueued self-update) held it back — never for an ordinary "not open yet" / "waiting
|
|
3435
|
+
* on a dependency" skip. `advanceQueue` uses this to keep `queuedHint` from masking that record's own next
|
|
3436
|
+
* chance to be picked back up (`queue-drain-queuedHint-gate`; `driveReenqueuedSelfUpdate` reports its own holds).
|
|
3437
|
+
*/
|
|
3438
|
+
export async function resumePending(rt, opts) {
|
|
3439
|
+
if (rt.fence !== undefined || rt.disbanded !== undefined)
|
|
3440
|
+
return false;
|
|
3441
|
+
const view = rt.groupStore.artifact('pendingIntents');
|
|
3442
|
+
const records = await rt.guard.call(rt.groupId, 'store', () => listPendingIntentRecords(view));
|
|
3443
|
+
let heldByElectionOrProposal = false;
|
|
3444
|
+
for (const record of records) {
|
|
3445
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
3446
|
+
return heldByElectionOrProposal;
|
|
3447
|
+
if (opts?.unpublishedOnly === true && record.state !== 'pending-confirm' && record.envelope !== undefined)
|
|
3448
|
+
continue;
|
|
3449
|
+
// AC-RACE-3's negative control: `disableSelfUpdateReenqueueForTest` isolates every re-enqueue drive path —
|
|
3450
|
+
// this one (a queue drain, a load or post-sync resume) included, pending-confirm records too.
|
|
3451
|
+
if (rt.ctx.disableSelfUpdateReenqueueForTest === true && record.reenqueuedFrom !== undefined)
|
|
3452
|
+
continue;
|
|
3453
|
+
if (record.state === 'pending-confirm') {
|
|
3454
|
+
await confirmRecord(rt, record, 0, undefined);
|
|
3455
|
+
continue;
|
|
3456
|
+
}
|
|
3457
|
+
// AC-RACE-1: an envelope-less re-enqueued self-update follows the shared rules (leader, re-parent, deferrals,
|
|
3458
|
+
// the canonical-again cancel) — never driven here directly with its recorded `parentEpoch`.
|
|
3459
|
+
// A commit re-drive record (AC-REC-1) takes this branch too, to ITS drive: it is never built as the bare `{ type: 'commit' }` its
|
|
3460
|
+
// stored intent is (DD15), only through the DD11 checks.
|
|
3461
|
+
if (record.envelope === undefined && record.reenqueuedFrom !== undefined && record.state !== 'awaiting-sync') {
|
|
3462
|
+
const drive = isCommitRedrive(record) ? driveCommitRedrive : driveReenqueuedSelfUpdate;
|
|
3463
|
+
if ((await drive(rt, record.reenqueuedFrom)) === 'held')
|
|
3464
|
+
heldByElectionOrProposal = true;
|
|
3465
|
+
continue;
|
|
3466
|
+
}
|
|
3467
|
+
if (record.envelope === undefined) {
|
|
3468
|
+
// AC-PIPE-15: nothing is minted before the group's first post-load sync — neither an intent accepted
|
|
3469
|
+
// awaiting-sync nor one recorded before the load whose staging never happened (a crash in between).
|
|
3470
|
+
if (!rt.sendGateOpen)
|
|
3471
|
+
continue;
|
|
3472
|
+
let driven = record;
|
|
3473
|
+
if (record.state === 'awaiting-sync') {
|
|
3474
|
+
const released = await releaseAwaitingRecord(rt, record);
|
|
3475
|
+
if (released === undefined)
|
|
3476
|
+
continue;
|
|
3477
|
+
driven = released;
|
|
3478
|
+
}
|
|
3479
|
+
// S13: a self-update behind its staged-Adds commit waits for it, is dropped with it, or runs re-parented.
|
|
3480
|
+
const resolved = await resolveDependency(rt, driven);
|
|
3481
|
+
if (resolved === 'waiting' || resolved === 'requeued')
|
|
3482
|
+
continue;
|
|
3483
|
+
if (resolved === 'terminal') {
|
|
3484
|
+
const after = driven.intent.type === 'self-update' ? driven.intent.after : undefined;
|
|
3485
|
+
await terminalFail(rt, driven, false, new Error('the commit of the staged proposals it follows went terminal'), { reason: 'dependency-terminal', after });
|
|
3486
|
+
continue;
|
|
3487
|
+
}
|
|
3488
|
+
// AC-INVITE-4: a resumed own commit is built over the CURRENT staged set, which may have
|
|
3489
|
+
// changed since the record was written — the proposal policy runs first, exactly as for
|
|
3490
|
+
// every other commit-building path (it drops rejected proposals and rebuilds the engine, so
|
|
3491
|
+
// before driveIntent, never inside it). A deferred proposal leaves the record in place for
|
|
3492
|
+
// the next resume or drain.
|
|
3493
|
+
if (isCommitKind(resolved.kind) && resolved.kind !== 'proposal' && resolved.parentEpoch === rt.engine.state.groupContext.epoch) {
|
|
3494
|
+
// S5 (AC-RACE-2): same hold as every other commit-building path. Clearing `queuedHint` is the
|
|
3495
|
+
// same rule as `driveReenqueuedSelfUpdate`'s own — this loop runs INSIDE a drain already, but the
|
|
3496
|
+
// NEXT trigger (the election-wait timer, a staged-set change) must not find a stale "nothing queued"
|
|
3497
|
+
// hint and skip re-evaluating this record once this pass's own `advanceQueue` resets it below.
|
|
3498
|
+
if (awaitingElectedSelfRemove(rt)) {
|
|
3499
|
+
rt.queuedHint = undefined;
|
|
3500
|
+
heldByElectionOrProposal = true;
|
|
3501
|
+
continue;
|
|
3502
|
+
}
|
|
3503
|
+
const deferred = await decideStagedProposals(rt);
|
|
3504
|
+
if (rt.disposed || rt.ctx.isDisposed())
|
|
3505
|
+
return heldByElectionOrProposal;
|
|
3506
|
+
if (deferred > 0) {
|
|
3507
|
+
rt.queuedHint = undefined;
|
|
3508
|
+
heldByElectionOrProposal = true;
|
|
3509
|
+
continue;
|
|
3510
|
+
}
|
|
3511
|
+
}
|
|
3512
|
+
await driveIntent(rt, resolved, false);
|
|
3513
|
+
continue;
|
|
3514
|
+
}
|
|
3515
|
+
await settlePublishedRecord(rt, record, { probeFirst: true, live: undefined, hasCaller: false });
|
|
3516
|
+
}
|
|
3517
|
+
return heldByElectionOrProposal;
|
|
3518
|
+
}
|
|
3519
|
+
/**
|
|
3520
|
+
* Internal (tests only), a mutation-gate test seam: references to the private pipeline helpers, unchanged, so
|
|
3521
|
+
* a test can drive them on a real runtime (see InternalClientOptions.exposeRuntimesForTest). Not re-exported
|
|
3522
|
+
* from the package entry points.
|
|
3523
|
+
*/
|
|
3524
|
+
// Stryker disable next-line all: test seam
|
|
3525
|
+
export const __internal = Object.freeze({
|
|
3526
|
+
driveReenqueuedSelfUpdate,
|
|
3527
|
+
driveCommitRedrive,
|
|
3528
|
+
armReenqueueRetry,
|
|
3529
|
+
driveSelfUpdateOverStagedAdds,
|
|
3530
|
+
linkStagedAddsCommit,
|
|
3531
|
+
terminalFail,
|
|
3532
|
+
resolveDependency,
|
|
3533
|
+
followDependents,
|
|
3534
|
+
retryDeferredSelfUpdateReenqueues,
|
|
3535
|
+
reparentReenqueued,
|
|
3536
|
+
remintStaleReenqueue,
|
|
3537
|
+
resumePending,
|
|
3538
|
+
awaitingElectedSelfRemove,
|
|
3539
|
+
armElectionWaitTimer,
|
|
3540
|
+
leafHeldOnCanonicalPath,
|
|
3541
|
+
confirmRecord,
|
|
3542
|
+
removeProposalsFor,
|
|
3543
|
+
stateHoldsLeaf,
|
|
3544
|
+
ownCommitMarkerOf,
|
|
3545
|
+
});
|