@cotal-ai/core 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/acls.d.ts +31 -7
- package/dist/acls.d.ts.map +1 -1
- package/dist/acls.js +105 -17
- package/dist/acls.js.map +1 -1
- package/dist/agent-file.js +1 -1
- package/dist/agent-file.js.map +1 -1
- package/dist/auth-provider.d.ts +8 -0
- package/dist/auth-provider.d.ts.map +1 -1
- package/dist/auth-provider.js.map +1 -1
- package/dist/backup.d.ts.map +1 -1
- package/dist/backup.js +6 -4
- package/dist/backup.js.map +1 -1
- package/dist/broker-floor.d.ts +34 -0
- package/dist/broker-floor.d.ts.map +1 -0
- package/dist/broker-floor.js +39 -0
- package/dist/broker-floor.js.map +1 -0
- package/dist/canonical.d.ts +24 -0
- package/dist/canonical.d.ts.map +1 -0
- package/dist/canonical.js +159 -0
- package/dist/canonical.js.map +1 -0
- package/dist/connector.d.ts +4 -0
- package/dist/connector.d.ts.map +1 -1
- package/dist/endpoint-action.d.ts +424 -0
- package/dist/endpoint-action.d.ts.map +1 -0
- package/dist/endpoint-action.js +910 -0
- package/dist/endpoint-action.js.map +1 -0
- package/dist/endpoint-binding.d.ts +344 -0
- package/dist/endpoint-binding.d.ts.map +1 -0
- package/dist/endpoint-binding.js +753 -0
- package/dist/endpoint-binding.js.map +1 -0
- package/dist/endpoint-checkpoint.d.ts +292 -0
- package/dist/endpoint-checkpoint.d.ts.map +1 -0
- package/dist/endpoint-checkpoint.js +818 -0
- package/dist/endpoint-checkpoint.js.map +1 -0
- package/dist/endpoint-cluster.d.ts +95 -0
- package/dist/endpoint-cluster.d.ts.map +1 -0
- package/dist/endpoint-cluster.js +194 -0
- package/dist/endpoint-cluster.js.map +1 -0
- package/dist/endpoint-contract-store.d.ts +90 -0
- package/dist/endpoint-contract-store.d.ts.map +1 -0
- package/dist/endpoint-contract-store.js +348 -0
- package/dist/endpoint-contract-store.js.map +1 -0
- package/dist/endpoint-envelope.d.ts +165 -0
- package/dist/endpoint-envelope.d.ts.map +1 -0
- package/dist/endpoint-envelope.js +346 -0
- package/dist/endpoint-envelope.js.map +1 -0
- package/dist/endpoint-grants.d.ts +123 -0
- package/dist/endpoint-grants.d.ts.map +1 -0
- package/dist/endpoint-grants.js +215 -0
- package/dist/endpoint-grants.js.map +1 -0
- package/dist/endpoint-guard.d.ts +222 -0
- package/dist/endpoint-guard.d.ts.map +1 -0
- package/dist/endpoint-guard.js +468 -0
- package/dist/endpoint-guard.js.map +1 -0
- package/dist/endpoint-handle.d.ts +176 -0
- package/dist/endpoint-handle.d.ts.map +1 -0
- package/dist/endpoint-handle.js +608 -0
- package/dist/endpoint-handle.js.map +1 -0
- package/dist/endpoint-journal.d.ts +225 -0
- package/dist/endpoint-journal.d.ts.map +1 -0
- package/dist/endpoint-journal.js +431 -0
- package/dist/endpoint-journal.js.map +1 -0
- package/dist/endpoint-receipt.d.ts +225 -0
- package/dist/endpoint-receipt.d.ts.map +1 -0
- package/dist/endpoint-receipt.js +417 -0
- package/dist/endpoint-receipt.js.map +1 -0
- package/dist/endpoint-records.d.ts +243 -0
- package/dist/endpoint-records.d.ts.map +1 -0
- package/dist/endpoint-records.js +591 -0
- package/dist/endpoint-records.js.map +1 -0
- package/dist/endpoint-serve.d.ts +176 -0
- package/dist/endpoint-serve.d.ts.map +1 -0
- package/dist/endpoint-serve.js +446 -0
- package/dist/endpoint-serve.js.map +1 -0
- package/dist/endpoint-service.d.ts +518 -0
- package/dist/endpoint-service.d.ts.map +1 -0
- package/dist/endpoint-service.js +973 -0
- package/dist/endpoint-service.js.map +1 -0
- package/dist/endpoint-session.d.ts +398 -0
- package/dist/endpoint-session.d.ts.map +1 -0
- package/dist/endpoint-session.js +834 -0
- package/dist/endpoint-session.js.map +1 -0
- package/dist/endpoint-signing.d.ts +105 -0
- package/dist/endpoint-signing.d.ts.map +1 -0
- package/dist/endpoint-signing.js +190 -0
- package/dist/endpoint-signing.js.map +1 -0
- package/dist/endpoint-subjects.d.ts +222 -0
- package/dist/endpoint-subjects.d.ts.map +1 -0
- package/dist/endpoint-subjects.js +381 -0
- package/dist/endpoint-subjects.js.map +1 -0
- package/dist/endpoint-supervisor.d.ts +19 -0
- package/dist/endpoint-supervisor.d.ts.map +1 -0
- package/dist/endpoint-supervisor.js +12 -0
- package/dist/endpoint-supervisor.js.map +1 -0
- package/dist/endpoint-traits.d.ts +225 -0
- package/dist/endpoint-traits.d.ts.map +1 -0
- package/dist/endpoint-traits.js +469 -0
- package/dist/endpoint-traits.js.map +1 -0
- package/dist/endpoint-verbs.d.ts +277 -0
- package/dist/endpoint-verbs.d.ts.map +1 -0
- package/dist/endpoint-verbs.js +576 -0
- package/dist/endpoint-verbs.js.map +1 -0
- package/dist/endpoint-virtual.d.ts +308 -0
- package/dist/endpoint-virtual.d.ts.map +1 -0
- package/dist/endpoint-virtual.js +440 -0
- package/dist/endpoint-virtual.js.map +1 -0
- package/dist/endpoint-work.d.ts +304 -0
- package/dist/endpoint-work.d.ts.map +1 -0
- package/dist/endpoint-work.js +728 -0
- package/dist/endpoint-work.js.map +1 -0
- package/dist/endpoint.d.ts +66 -22
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +289 -89
- package/dist/endpoint.js.map +1 -1
- package/dist/evict.d.ts +95 -32
- package/dist/evict.d.ts.map +1 -1
- package/dist/evict.js +193 -3
- package/dist/evict.js.map +1 -1
- package/dist/index.d.ts +25 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -1
- package/dist/lease.d.ts +4 -0
- package/dist/lease.d.ts.map +1 -1
- package/dist/lease.js.map +1 -1
- package/dist/members.d.ts +4 -4
- package/dist/members.d.ts.map +1 -1
- package/dist/members.js +12 -12
- package/dist/members.js.map +1 -1
- package/dist/provision.d.ts +70 -18
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +194 -33
- package/dist/provision.js.map +1 -1
- package/dist/safe-pattern.d.ts +33 -0
- package/dist/safe-pattern.d.ts.map +1 -0
- package/dist/safe-pattern.js +486 -0
- package/dist/safe-pattern.js.map +1 -0
- package/dist/schema-profile.d.ts +93 -0
- package/dist/schema-profile.d.ts.map +1 -0
- package/dist/schema-profile.js +284 -0
- package/dist/schema-profile.js.map +1 -0
- package/dist/streams.d.ts +13 -8
- package/dist/streams.d.ts.map +1 -1
- package/dist/streams.js +35 -18
- package/dist/streams.js.map +1 -1
- package/dist/subjects.d.ts +100 -40
- package/dist/subjects.d.ts.map +1 -1
- package/dist/subjects.js +136 -60
- package/dist/subjects.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -1
|
@@ -0,0 +1,818 @@
|
|
|
1
|
+
import { jetstream, jetstreamManager } from "@nats-io/jetstream";
|
|
2
|
+
import { headers as natsHeaders } from "@nats-io/transport-node";
|
|
3
|
+
import { EpEnvelopeError } from "./endpoint-envelope.js";
|
|
4
|
+
import { canonicalJson } from "./canonical.js";
|
|
5
|
+
import { epfSubject, eptSubject, parseEpSubject, assertIdToken } from "./endpoint-subjects.js";
|
|
6
|
+
import { RECORD_KINDS, recordSpecKey, recordStatusKey, createRecordEntry, updateRecordEntry, assertStatusValue, readRecordLeader, isCasLoss } from "./endpoint-records.js";
|
|
7
|
+
import { epfStreamName, readLastFact } from "./endpoint-journal.js";
|
|
8
|
+
import { eptStreamName } from "./endpoint-binding.js";
|
|
9
|
+
/** The one-use settle-fact subject (`epf.<e>.cp.<token>`, §13.9). */
|
|
10
|
+
export function checkpointSettleSubject(space, ref) {
|
|
11
|
+
return epfSubject(space, ref.endpoint, ["cp", ref.token]);
|
|
12
|
+
}
|
|
13
|
+
function cpQualifiers(ref) {
|
|
14
|
+
return [ref.endpoint, ref.token];
|
|
15
|
+
}
|
|
16
|
+
/** Closed-schema guard: an unknown field in a persisted record (or a smuggled extra on an
|
|
17
|
+
* input) is refused, never carried along. */
|
|
18
|
+
function closedKeys(o, allowed, what, code) {
|
|
19
|
+
for (const k of Object.keys(o))
|
|
20
|
+
if (!allowed.includes(k))
|
|
21
|
+
throw new EpEnvelopeError(code, `${what} carries the unknown field "${k}"; checkpoint schemas are closed (SPEC 13.6)`);
|
|
22
|
+
}
|
|
23
|
+
function isHolder(v) {
|
|
24
|
+
if (v === null || typeof v !== "object" || Array.isArray(v))
|
|
25
|
+
return false;
|
|
26
|
+
const o = v;
|
|
27
|
+
return Object.keys(o).length === 2
|
|
28
|
+
&& typeof o.id === "string" && o.id.length > 0
|
|
29
|
+
&& typeof o.lifecycleUid === "string" && o.lifecycleUid.length > 0;
|
|
30
|
+
}
|
|
31
|
+
/** Detach a caller-supplied holder/presenter to a frozen exact copy at seam entry: a live
|
|
32
|
+
* object mutated across an await must never move an authority coordinate mid-seam. Every
|
|
33
|
+
* property is READ EXACTLY ONCE (a getter answering differently between a validation read and
|
|
34
|
+
* a copy read must not split what was checked from what is used). */
|
|
35
|
+
function snapshotHolder(v, what) {
|
|
36
|
+
if (v === null || typeof v !== "object" || Array.isArray(v) || Object.keys(v).length !== 2)
|
|
37
|
+
throw new EpEnvelopeError("failed-precondition", `${what} must be a closed {id, lifecycleUid} pair of nonempty strings (SPEC 13.6/13.10)`);
|
|
38
|
+
const id = v.id;
|
|
39
|
+
const lifecycleUid = v.lifecycleUid;
|
|
40
|
+
if (typeof id !== "string" || id.length === 0 || typeof lifecycleUid !== "string" || lifecycleUid.length === 0)
|
|
41
|
+
throw new EpEnvelopeError("failed-precondition", `${what} must be a closed {id, lifecycleUid} pair of nonempty strings (SPEC 13.6/13.10)`);
|
|
42
|
+
return Object.freeze({ id, lifecycleUid });
|
|
43
|
+
}
|
|
44
|
+
/** Detach a caller-supplied ref to a frozen exact copy at seam entry (single-read). */
|
|
45
|
+
function snapshotCpRef(ref) {
|
|
46
|
+
if (ref === null || typeof ref !== "object")
|
|
47
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint ref must carry a nonempty endpoint and token (SPEC 13.6)`);
|
|
48
|
+
const endpoint = ref.endpoint;
|
|
49
|
+
const token = ref.token;
|
|
50
|
+
if (typeof endpoint !== "string" || endpoint.length === 0 || typeof token !== "string" || token.length === 0)
|
|
51
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint ref must carry a nonempty endpoint and token (SPEC 13.6)`);
|
|
52
|
+
return Object.freeze({ endpoint, token });
|
|
53
|
+
}
|
|
54
|
+
/** Detach + validate a caller-supplied goal binding at seam entry (closed shapes, single-read). */
|
|
55
|
+
function snapshotGoal(v) {
|
|
56
|
+
if (v === null || typeof v !== "object" || Array.isArray(v))
|
|
57
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint goal binding must be a {caller, goalId} object (SPEC 13.6)`);
|
|
58
|
+
const o = v;
|
|
59
|
+
closedKeys(o, ["caller", "goalId"], "a checkpoint goal binding", "failed-precondition");
|
|
60
|
+
const goalId = o.goalId;
|
|
61
|
+
const rawCaller = o.caller;
|
|
62
|
+
if (rawCaller === null || typeof rawCaller !== "object" || Array.isArray(rawCaller) || Object.keys(rawCaller).length !== 3)
|
|
63
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint goal binding must carry a closed caller {owner, actor, uid} and a nonempty goalId (SPEC 13.6)`);
|
|
64
|
+
const owner = rawCaller.owner;
|
|
65
|
+
const actor = rawCaller.actor;
|
|
66
|
+
const uid = rawCaller.uid;
|
|
67
|
+
if (typeof owner !== "string" || owner.length === 0
|
|
68
|
+
|| typeof actor !== "string" || actor.length === 0
|
|
69
|
+
|| typeof uid !== "string" || uid.length === 0
|
|
70
|
+
|| typeof goalId !== "string" || goalId.length === 0)
|
|
71
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint goal binding must carry a closed caller {owner, actor, uid} and a nonempty goalId (SPEC 13.6)`);
|
|
72
|
+
return Object.freeze({ caller: Object.freeze({ owner, actor, uid }), goalId });
|
|
73
|
+
}
|
|
74
|
+
/** Validate + DETACH one persisted/incoming obligation (closed schema; the signed artifact's
|
|
75
|
+
* bytes are carried verbatim — this seam stores and returns them, it never re-verifies: the
|
|
76
|
+
* gate verified at HOLD time and the mediated record is the custody chain). */
|
|
77
|
+
function snapshotObligation(v, what, code) {
|
|
78
|
+
if (v === null || typeof v !== "object" || Array.isArray(v))
|
|
79
|
+
throw new EpEnvelopeError(code, `${what} is not an object; obligation schemas are closed (SPEC 13.6/13.10)`);
|
|
80
|
+
const o = v;
|
|
81
|
+
closedKeys(o, ["v", "space", "requestId", "signer", "attenuations", "iat", "exp", "sig"], what, code);
|
|
82
|
+
const signer = o.signer;
|
|
83
|
+
if (o.v !== 1 || typeof o.space !== "string" || o.space.length === 0
|
|
84
|
+
|| typeof o.requestId !== "string" || o.requestId.length === 0
|
|
85
|
+
|| signer === null || typeof signer !== "object" || Array.isArray(signer) || Object.keys(signer).length !== 1 || typeof signer.keyId !== "string" || signer.keyId.length === 0
|
|
86
|
+
|| !Array.isArray(o.attenuations) || o.attenuations.length === 0
|
|
87
|
+
|| typeof o.iat !== "number" || !Number.isSafeInteger(o.iat)
|
|
88
|
+
|| typeof o.exp !== "number" || !Number.isSafeInteger(o.exp)
|
|
89
|
+
|| typeof o.sig !== "string" || o.sig.length === 0)
|
|
90
|
+
throw new EpEnvelopeError(code, `${what} is malformed; a garbled obligation never rides a hold (SPEC 13.6/13.10)`);
|
|
91
|
+
return JSON.parse(JSON.stringify({
|
|
92
|
+
v: 1, space: o.space, requestId: o.requestId, signer: { keyId: signer.keyId },
|
|
93
|
+
attenuations: o.attenuations, iat: o.iat, exp: o.exp, sig: o.sig,
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
96
|
+
function parseCpSpec(raw, ref, key) {
|
|
97
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw))
|
|
98
|
+
throw new EpEnvelopeError("internal", `checkpoint spec ${key} is not an object; garbled state never authorizes (SPEC 13.6)`);
|
|
99
|
+
const o = raw;
|
|
100
|
+
closedKeys(o, ["v", "token", "goal", "obligations", "holder", "mintedAt", "initialDeadline"], `checkpoint spec ${key}`, "internal");
|
|
101
|
+
if (o.v !== 1 || o.token !== ref.token || !isHolder(o.holder)
|
|
102
|
+
|| typeof o.mintedAt !== "number" || !Number.isSafeInteger(o.mintedAt) || o.mintedAt < 0
|
|
103
|
+
|| typeof o.initialDeadline !== "number" || !Number.isSafeInteger(o.initialDeadline) || o.initialDeadline < 0)
|
|
104
|
+
throw new EpEnvelopeError("internal", `checkpoint spec ${key} is malformed or its token disagrees with its subject (SPEC 13.6); garbled state never authorizes`);
|
|
105
|
+
let goal;
|
|
106
|
+
if (o.goal !== undefined) {
|
|
107
|
+
try {
|
|
108
|
+
goal = snapshotGoal(o.goal);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
throw new EpEnvelopeError("internal", `checkpoint spec ${key} carries a malformed goal binding; garbled state never authorizes (SPEC 13.6)`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
let obligations;
|
|
115
|
+
if (o.obligations !== undefined) {
|
|
116
|
+
if (!Array.isArray(o.obligations) || o.obligations.length === 0 || goal === undefined)
|
|
117
|
+
throw new EpEnvelopeError("internal", `checkpoint spec ${key} carries a malformed obligations set (obligations ride a goal-bound hold as a non-empty array); garbled state never authorizes (SPEC 13.6/13.10)`);
|
|
118
|
+
obligations = o.obligations.map((e, i) => snapshotObligation(e, `checkpoint spec ${key} obligations[${i}]`, "internal"));
|
|
119
|
+
}
|
|
120
|
+
// Picked construction: the returned value carries exactly the schema fields, byte-derived.
|
|
121
|
+
return {
|
|
122
|
+
v: 1, token: o.token,
|
|
123
|
+
...(goal !== undefined ? { goal } : {}),
|
|
124
|
+
...(obligations !== undefined ? { obligations } : {}),
|
|
125
|
+
holder: { id: o.holder.id, lifecycleUid: o.holder.lifecycleUid },
|
|
126
|
+
mintedAt: o.mintedAt, initialDeadline: o.initialDeadline,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function assertOwnerClock(now) {
|
|
130
|
+
if (typeof now !== "number" || !Number.isSafeInteger(now) || now < 0)
|
|
131
|
+
throw new EpEnvelopeError("failed-precondition", `the owner clock must be a non-negative safe integer; got ${JSON.stringify(now)} (a NaN clock would bypass a deadline comparison, SPEC 13.6)`);
|
|
132
|
+
return now;
|
|
133
|
+
}
|
|
134
|
+
function parseSettle(raw, subject, ref) {
|
|
135
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw))
|
|
136
|
+
throw new EpEnvelopeError("internal", `checkpoint settle fact on ${subject} is not an object; garbled state never authorizes (SPEC 13.6)`);
|
|
137
|
+
const o = raw;
|
|
138
|
+
closedKeys(o, ["v", "token", "settle", "generation", "holder", "ts"], `checkpoint settle fact on ${subject}`, "internal");
|
|
139
|
+
if (o.v !== 1 || o.token !== ref.token || (o.settle !== "resumed" && o.settle !== "expired")
|
|
140
|
+
|| typeof o.generation !== "number" || !Number.isSafeInteger(o.generation) || o.generation < 1
|
|
141
|
+
|| typeof o.ts !== "number" || !Number.isSafeInteger(o.ts) || o.ts < 0)
|
|
142
|
+
throw new EpEnvelopeError("internal", `checkpoint settle fact on ${subject} is malformed or its token disagrees with its subject (SPEC 13.6); garbled state never authorizes`);
|
|
143
|
+
// Per-settle variant: a RESUMED fact carries the resuming holder; an EXPIRED fact never does
|
|
144
|
+
// (an expiry has no resuming principal; a holder on it would forge resume attribution).
|
|
145
|
+
if (o.settle === "resumed" ? !isHolder(o.holder) : o.holder !== undefined)
|
|
146
|
+
throw new EpEnvelopeError("internal", `checkpoint settle fact on ${subject} violates its ${String(o.settle)} variant (resumed requires the holder; expired forbids one) (SPEC 13.6)`);
|
|
147
|
+
return {
|
|
148
|
+
v: 1, token: o.token, settle: o.settle,
|
|
149
|
+
generation: o.generation,
|
|
150
|
+
...(o.holder !== undefined ? { holder: { id: o.holder.id, lifecycleUid: o.holder.lifecycleUid } } : {}),
|
|
151
|
+
ts: o.ts,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
function parseCpStatus(raw, key) {
|
|
155
|
+
if (raw === null || typeof raw !== "object" || Array.isArray(raw))
|
|
156
|
+
throw new EpEnvelopeError("internal", `checkpoint status ${key} is not an object; garbled state never authorizes (SPEC 13.6)`);
|
|
157
|
+
const o = raw;
|
|
158
|
+
closedKeys(o, ["state", "deadlineGeneration", "deadline", "observedSpecRevision", "settledGeneration", "settledHolder", "settledTs"], `checkpoint status ${key}`, "internal");
|
|
159
|
+
if ((o.state !== "waiting" && o.state !== "resumed" && o.state !== "expired")
|
|
160
|
+
|| typeof o.deadlineGeneration !== "number" || !Number.isSafeInteger(o.deadlineGeneration) || o.deadlineGeneration < 1
|
|
161
|
+
|| typeof o.deadline !== "number" || !Number.isSafeInteger(o.deadline) || o.deadline < 0
|
|
162
|
+
|| typeof o.observedSpecRevision !== "number" || !Number.isSafeInteger(o.observedSpecRevision) || o.observedSpecRevision < 1)
|
|
163
|
+
throw new EpEnvelopeError("internal", `checkpoint status ${key} is malformed; garbled state never authorizes (SPEC 13.6)`);
|
|
164
|
+
// Per-state variants: WAITING carries no settlement coordinates; a settled state carries its
|
|
165
|
+
// full settlement (generation + ts, and the holder exactly when RESUMED). A cross-variant
|
|
166
|
+
// record is garbled, never papered over with defaults.
|
|
167
|
+
const settledShape = typeof o.settledGeneration === "number" && Number.isSafeInteger(o.settledGeneration) && o.settledGeneration >= 1
|
|
168
|
+
&& typeof o.settledTs === "number" && Number.isSafeInteger(o.settledTs) && o.settledTs >= 0;
|
|
169
|
+
if (o.state === "waiting"
|
|
170
|
+
? (o.settledGeneration !== undefined || o.settledHolder !== undefined || o.settledTs !== undefined)
|
|
171
|
+
: (!settledShape || (o.state === "resumed" ? !isHolder(o.settledHolder) : o.settledHolder !== undefined)))
|
|
172
|
+
throw new EpEnvelopeError("internal", `checkpoint status ${key} violates its ${String(o.state)} variant (settled coordinates are exact, never defaulted) (SPEC 13.6)`);
|
|
173
|
+
// A settlement is the settlement of the CURRENT deadline generation (distsys 8dcad72 M7): the
|
|
174
|
+
// settledGeneration must EQUAL the deadlineGeneration, so `deriveSettleFact` and the one-use
|
|
175
|
+
// `.cp` fact can never publish a settled coordinate that contradicts the generation the
|
|
176
|
+
// checkpoint actually reached. A record naming two different generations is garbled, never split.
|
|
177
|
+
if (o.state !== "waiting" && o.settledGeneration !== o.deadlineGeneration)
|
|
178
|
+
throw new EpEnvelopeError("internal", `checkpoint status ${key} settles generation ${String(o.settledGeneration)} but its deadline generation is ${String(o.deadlineGeneration)}; a settlement is of the current generation (SPEC 13.6)`);
|
|
179
|
+
return {
|
|
180
|
+
state: o.state, deadlineGeneration: o.deadlineGeneration, deadline: o.deadline,
|
|
181
|
+
observedSpecRevision: o.observedSpecRevision,
|
|
182
|
+
...(o.settledGeneration !== undefined ? { settledGeneration: o.settledGeneration } : {}),
|
|
183
|
+
...(o.settledHolder !== undefined ? { settledHolder: { id: o.settledHolder.id, lifecycleUid: o.settledHolder.lifecycleUid } } : {}),
|
|
184
|
+
...(o.settledTs !== undefined ? { settledTs: o.settledTs } : {}),
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
/** Read the checkpoint's current status (`undefined` = unknown token). Fail-closed on DEL. */
|
|
188
|
+
export async function readCheckpointStatus(kv, ref) {
|
|
189
|
+
const key = recordStatusKey(RECORD_KINDS.cp, cpQualifiers(ref));
|
|
190
|
+
const entry = await kv.get(key);
|
|
191
|
+
if (!entry)
|
|
192
|
+
return undefined;
|
|
193
|
+
if (entry.operation !== "PUT")
|
|
194
|
+
throw new EpEnvelopeError("failed-precondition", `the checkpoint status ${key} carries a ${entry.operation} marker; a deletion never erases a pause - reconcile the store (SPEC 13.6)`);
|
|
195
|
+
return { value: parseCpStatus(JSON.parse(new TextDecoder().decode(entry.value)), key), revision: entry.revision };
|
|
196
|
+
}
|
|
197
|
+
/** Publish a `.schedule` REQUEST onto the mediated timer plane — the ONLY timer publish an
|
|
198
|
+
* instance holds (§13.9). Plain bytes, NO scheduling headers (a header would be inert on the
|
|
199
|
+
* schedules-disabled request stream and the writer rejects it anyway); the mint, the
|
|
200
|
+
* heartbeat, and the durable reconciler all share exactly this emission, which is what makes
|
|
201
|
+
* reconciliation over-emission harmless (same `(timerId, generation)` → the writer re-derives
|
|
202
|
+
* the same `.armed`, a no-op replacement). */
|
|
203
|
+
export async function emitScheduleRequest(js, space, args) {
|
|
204
|
+
const subject = eptSubject(space, args.endpoint, args.instanceId, args.epoch, args.token, "schedule");
|
|
205
|
+
const body = { v: 1, timerId: args.token, generation: args.generation, deadline: args.deadline };
|
|
206
|
+
await js.publish(subject, new TextEncoder().encode(JSON.stringify(body)));
|
|
207
|
+
}
|
|
208
|
+
/** Mint a checkpoint (the commit path, §13.6): the durable token's spec + a `waiting` status
|
|
209
|
+
* at generation 1 with the MANDATORY deadline, then the `.schedule` request. The record is
|
|
210
|
+
* durable BEFORE the timer exists; a crash between the two is exactly what the reconciler's
|
|
211
|
+
* re-emit repairs. */
|
|
212
|
+
export async function mintCheckpoint(kv, js, space, args) {
|
|
213
|
+
// Snapshot the FULL mint input to detached locals at entry: nothing below reads args again.
|
|
214
|
+
const ref = snapshotCpRef(args.ref);
|
|
215
|
+
assertIdToken(ref.token, "checkpoint token");
|
|
216
|
+
const holder = snapshotHolder(args.holder, "a checkpoint holder (resume is holder-bound, never a bearer token)");
|
|
217
|
+
const goal = args.goal !== undefined ? snapshotGoal(args.goal) : undefined;
|
|
218
|
+
let obligations;
|
|
219
|
+
if (args.obligations !== undefined && args.obligations.length > 0) {
|
|
220
|
+
if (goal === undefined)
|
|
221
|
+
throw new EpEnvelopeError("failed-precondition", "checkpoint obligations ride a goal-bound hold; a goal-less checkpoint carries none (SPEC 13.6/13.10)");
|
|
222
|
+
obligations = args.obligations.map((e, i) => snapshotObligation(e, `checkpoint mint obligations[${i}]`, "failed-precondition"));
|
|
223
|
+
}
|
|
224
|
+
const instanceId = args.instanceId;
|
|
225
|
+
const epoch = args.epoch;
|
|
226
|
+
const deadline = args.deadline;
|
|
227
|
+
const now = assertOwnerClock(args.now);
|
|
228
|
+
if (!Number.isSafeInteger(deadline) || deadline <= now)
|
|
229
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint deadline is mandatory and must be in the owner's future (deadline ${deadline}, now ${now}); deadlines are the §13.6 contract, never optional`);
|
|
230
|
+
if (deadline > MAX_SCHEDULE_MS)
|
|
231
|
+
throw new EpEnvelopeError("failed-precondition", `a checkpoint deadline ${deadline} exceeds the scheduler's representable range (${MAX_SCHEDULE_MS}); admission rejects a deadline the timer writer could never arm, so a MAX_SAFE deadline never strands a waiting checkpoint unarmable (distsys 8dcad72 M4, SPEC 13.6/13.9)`);
|
|
232
|
+
const spec = {
|
|
233
|
+
v: 1, token: ref.token,
|
|
234
|
+
...(goal !== undefined ? { goal } : {}),
|
|
235
|
+
...(obligations !== undefined ? { obligations } : {}),
|
|
236
|
+
holder, mintedAt: now, initialDeadline: deadline,
|
|
237
|
+
};
|
|
238
|
+
const specKey = recordSpecKey(RECORD_KINDS.cp, cpQualifiers(ref));
|
|
239
|
+
const statusKey = recordStatusKey(RECORD_KINDS.cp, cpQualifiers(ref));
|
|
240
|
+
// Idempotent-if-identical (the mint is a two-key composite; a crash between spec and status, or
|
|
241
|
+
// a retry, must not strand a spec-only token): create the spec; on a conflict re-read it and
|
|
242
|
+
// require an IDENTICAL spec (a differing spec under the same token is a loud conflict), then
|
|
243
|
+
// ensure the initial `waiting` status exists. The create CAS covers the key's ENTIRE history
|
|
244
|
+
// (createRecordEntry never recreates over a tombstone), so a DELETED spec is a permanent
|
|
245
|
+
// refusal here — a spec-DEL can never rebind the one-use resume holder to a new principal.
|
|
246
|
+
let specRevision;
|
|
247
|
+
try {
|
|
248
|
+
specRevision = await createRecordEntry(kv, specKey, spec);
|
|
249
|
+
}
|
|
250
|
+
catch (e) {
|
|
251
|
+
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
252
|
+
throw e;
|
|
253
|
+
const existing = await kv.get(specKey);
|
|
254
|
+
if (existing && existing.operation !== "PUT")
|
|
255
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" spec carries a ${existing.operation} marker; a deletion never erases a pause and a re-mint never rebinds its holder - reconcile the store (SPEC 13.6)`);
|
|
256
|
+
if (!existing)
|
|
257
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" spec is not readable after a create conflict; reconcile the store (SPEC 13.6)`);
|
|
258
|
+
const prior = parseCpSpec(JSON.parse(new TextDecoder().decode(existing.value)), ref, specKey);
|
|
259
|
+
if (prior.holder.id !== holder.id || prior.holder.lifecycleUid !== holder.lifecycleUid
|
|
260
|
+
|| prior.initialDeadline !== deadline
|
|
261
|
+
|| (prior.goal === undefined) !== (goal === undefined)
|
|
262
|
+
|| (goal !== undefined && prior.goal !== undefined && (prior.goal.goalId !== goal.goalId
|
|
263
|
+
|| prior.goal.caller.owner !== goal.caller.owner || prior.goal.caller.actor !== goal.caller.actor || prior.goal.caller.uid !== goal.caller.uid))
|
|
264
|
+
// The OBLIGATIONS are part of the mint's identity (distsys M1 / freelance HIGH): a retry
|
|
265
|
+
// presenting a different verified set - including absent-vs-present in either direction -
|
|
266
|
+
// is a DIFFERENT intent, and adopting the recorded spec here would silently swap or drop
|
|
267
|
+
// attenuations the gate verified (SPEC 13.6/13.10: MUST-apply).
|
|
268
|
+
|| canonicalJson(prior.obligations ?? null) !== canonicalJson(obligations ?? null))
|
|
269
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" already exists with a DIFFERENT spec (holder/goal/deadline/obligations); a token is minted once (SPEC 13.6)`);
|
|
270
|
+
specRevision = existing.revision;
|
|
271
|
+
// The spec's immutable initialDeadline is now the authority for the generation-1 status: a
|
|
272
|
+
// retry that reached here supplied the SAME deadline (the conflict check above), so creating
|
|
273
|
+
// the missing status at `deadline` re-establishes the ORIGINAL intent, never a new one - the
|
|
274
|
+
// crash-before-status window can no longer install a divergent deadline (distsys 8dcad72 M3).
|
|
275
|
+
}
|
|
276
|
+
// A DEL/PURGE marker on the status is a DELETION of one-use settlement state, never absence:
|
|
277
|
+
// recreating over it would re-open a settled one-use checkpoint and let the same holder
|
|
278
|
+
// resume twice. This marker read is only the FAST-PATH refusal (the same fail-closed rule as
|
|
279
|
+
// readCheckpointStatus); the ARBITER is the create below — createRecordEntry's CAS covers the
|
|
280
|
+
// key's entire history, so a delete landing between this read and the create loses THERE, and
|
|
281
|
+
// the conflict path classifies the tombstone. The check-then-create pair carries no race.
|
|
282
|
+
const existingStatus = await kv.get(statusKey);
|
|
283
|
+
if (existingStatus && existingStatus.operation !== "PUT")
|
|
284
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" status carries a ${existingStatus.operation} marker; a deletion never erases a pause and a mint never resurrects one - reconcile the store (SPEC 13.6)`);
|
|
285
|
+
let statusValue;
|
|
286
|
+
if (!existingStatus) {
|
|
287
|
+
const initial = assertStatusValue({ state: "waiting", deadlineGeneration: 1, deadline, observedSpecRevision: specRevision });
|
|
288
|
+
try {
|
|
289
|
+
await createRecordEntry(kv, statusKey, initial);
|
|
290
|
+
statusValue = initial;
|
|
291
|
+
}
|
|
292
|
+
catch (e) {
|
|
293
|
+
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
294
|
+
throw e;
|
|
295
|
+
// The create lost its CAS: either a concurrent mint won (re-read and PROVE the winner —
|
|
296
|
+
// parse + the replay rules below, never assume it) or the key carries a tombstone the
|
|
297
|
+
// fast-path read missed (the race the arbiter exists for) — refuse it fail-closed.
|
|
298
|
+
const won = await kv.get(statusKey);
|
|
299
|
+
if (won && won.operation !== "PUT")
|
|
300
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" status carries a ${won.operation} marker; a deletion never erases a pause and a mint never resurrects one - reconcile the store (SPEC 13.6)`);
|
|
301
|
+
if (!won)
|
|
302
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" status is not readable after a create conflict; reconcile the store (SPEC 13.6)`);
|
|
303
|
+
statusValue = parseCpStatus(JSON.parse(new TextDecoder().decode(won.value)), statusKey);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
statusValue = parseCpStatus(JSON.parse(new TextDecoder().decode(existingStatus.value)), statusKey);
|
|
308
|
+
}
|
|
309
|
+
// Replay identity covers the DEADLINE too: a still-initial (generation-1 waiting) status must
|
|
310
|
+
// carry exactly the requested deadline (a differing deadline is a different mint intent). A
|
|
311
|
+
// SETTLED status is never reset and arms nothing. A heartbeat-advanced waiting status is the
|
|
312
|
+
// same live checkpoint; the schedule re-emits at the CURRENT authoritative coordinates, never
|
|
313
|
+
// the caller's, so a replayed mint can repair the mint-crash window without rolling back.
|
|
314
|
+
if (statusValue.state !== "waiting")
|
|
315
|
+
return { specRevision };
|
|
316
|
+
if (statusValue.deadlineGeneration === 1 && statusValue.deadline !== deadline)
|
|
317
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" already exists with deadline ${statusValue.deadline}; a replayed mint with a different deadline (${deadline}) is a different intent (SPEC 13.6)`);
|
|
318
|
+
await emitScheduleRequest(js, space, { endpoint: ref.endpoint, instanceId, epoch, token: ref.token, generation: statusValue.deadlineGeneration, deadline: statusValue.deadline });
|
|
319
|
+
return { specRevision };
|
|
320
|
+
}
|
|
321
|
+
/** Reconstruct the one-use `epf.<e>.cp.<token>` fact from a SETTLED status (the status is the
|
|
322
|
+
* arbiter; the fact is its derived durable copy). A waiting status derives nothing, and the
|
|
323
|
+
* settled coordinates are EXACT (parseCpStatus enforces the per-state variant) - a missing
|
|
324
|
+
* coordinate is garbled state, never papered over with a default. */
|
|
325
|
+
function deriveSettleFact(ref, s) {
|
|
326
|
+
if (s.state === "waiting" || s.settledGeneration === undefined || s.settledTs === undefined
|
|
327
|
+
|| (s.state === "resumed") !== (s.settledHolder !== undefined))
|
|
328
|
+
throw new EpEnvelopeError("internal", `checkpoint "${ref.token}" status does not carry a full ${s.state} settlement; garbled state never authorizes (SPEC 13.6)`);
|
|
329
|
+
return {
|
|
330
|
+
v: 1, token: ref.token, settle: s.state === "expired" ? "expired" : "resumed",
|
|
331
|
+
generation: s.settledGeneration,
|
|
332
|
+
...(s.settledHolder !== undefined ? { holder: { id: s.settledHolder.id, lifecycleUid: s.settledHolder.lifecycleUid } } : {}),
|
|
333
|
+
ts: s.settledTs,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
/** Canonical settle-fact equality (field-exact, holder included). */
|
|
337
|
+
function settleFactsEqual(a, b) {
|
|
338
|
+
return a.token === b.token && a.settle === b.settle && a.generation === b.generation && a.ts === b.ts
|
|
339
|
+
&& (a.holder === undefined) === (b.holder === undefined)
|
|
340
|
+
&& (a.holder === undefined || b.holder === undefined
|
|
341
|
+
|| (a.holder.id === b.holder.id && a.holder.lifecycleUid === b.holder.lifecycleUid));
|
|
342
|
+
}
|
|
343
|
+
/** Publish a settled status's derived one-use fact (create-only; idempotent). On a lost CAS the
|
|
344
|
+
* recorded winner must be READABLE and CANONICALLY EQUAL to the status-derived fact - the
|
|
345
|
+
* status is the arbiter, so a contradicting or missing winner is a loud `internal`, never
|
|
346
|
+
* adopted and never fabricated. */
|
|
347
|
+
async function ensureSettleFact(kv, js, jsm, space, ref, s) {
|
|
348
|
+
const fact = deriveSettleFact(ref, s);
|
|
349
|
+
const subject = checkpointSettleSubject(space, ref);
|
|
350
|
+
try {
|
|
351
|
+
const h = natsHeaders();
|
|
352
|
+
h.set("Nats-Expected-Last-Subject-Sequence", "0");
|
|
353
|
+
await js.publish(subject, new TextEncoder().encode(JSON.stringify(fact)), { headers: h });
|
|
354
|
+
return fact;
|
|
355
|
+
}
|
|
356
|
+
catch (e) {
|
|
357
|
+
const code = e?.code;
|
|
358
|
+
if (code !== 10071 && code !== 10164)
|
|
359
|
+
throw e;
|
|
360
|
+
const winner = await readCheckpointSettle(jsm, space, ref);
|
|
361
|
+
if (winner === undefined)
|
|
362
|
+
throw new EpEnvelopeError("internal", `checkpoint "${ref.token}" settle-fact CAS lost but no winner is readable; reconcile the store (SPEC 13.6)`);
|
|
363
|
+
if (!settleFactsEqual(winner, fact))
|
|
364
|
+
throw new EpEnvelopeError("internal", `checkpoint "${ref.token}" recorded settle fact (${winner.settle} generation ${winner.generation} ts ${winner.ts}) contradicts the status arbiter (${fact.settle} generation ${fact.generation} ts ${fact.ts}); a contradicting winner is never adopted (SPEC 13.6)`);
|
|
365
|
+
return winner;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
/** The AUTHORITATIVE liveness gate: is this checkpoint settled? The STATUS record is the arbiter
|
|
369
|
+
* (its revision is the shared settlement coordinate). If settled, this ENSURES the derived
|
|
370
|
+
* one-use fact exists (repairing a crash between the status CAS and the fact publish) and
|
|
371
|
+
* returns it. `undefined` iff genuinely still waiting. */
|
|
372
|
+
async function settledOrConverge(kv, js, jsm, space, ref) {
|
|
373
|
+
const status = await readCheckpointStatus(kv, ref);
|
|
374
|
+
if (status === undefined || status.value.state === "waiting")
|
|
375
|
+
return undefined;
|
|
376
|
+
return ensureSettleFact(kv, js, jsm, space, ref, status.value);
|
|
377
|
+
}
|
|
378
|
+
/** Heartbeat/extend: gate on the SETTLE FACT first (a settled checkpoint refuses and its
|
|
379
|
+
* lagging status is converged), then CAS-advance the deadline generation IN STATUS FIRST, then
|
|
380
|
+
* replace the timer (a new `.schedule` at the new generation — the mediated writer's
|
|
381
|
+
* same-subject `.armed` publish is the server rollup; the 2.14 atomic stop-plus-publish is NOT
|
|
382
|
+
* assumed at the 2.12 floor). The generation order is load-bearing: a crash after the CAS and
|
|
383
|
+
* before the emission leaves a stale-generation timer whose fire NO-OPS at the handler, and the
|
|
384
|
+
* reconciler re-emits the current generation — never a fire acting on a superseded deadline. */
|
|
385
|
+
export async function heartbeatCheckpoint(kv, js, jsm, space, args) {
|
|
386
|
+
// Snapshot the FULL operation input to detached locals at entry; nothing below reads args again.
|
|
387
|
+
const ref = snapshotCpRef(args.ref);
|
|
388
|
+
const instanceId = args.instanceId;
|
|
389
|
+
const epoch = args.epoch;
|
|
390
|
+
const deadline = args.deadline;
|
|
391
|
+
const now = assertOwnerClock(args.now);
|
|
392
|
+
const current = await readCheckpointStatus(kv, ref);
|
|
393
|
+
if (current === undefined)
|
|
394
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is unknown; a heartbeat extends only a minted checkpoint (SPEC 13.6)`);
|
|
395
|
+
const settled = await settledOrConverge(kv, js, jsm, space, ref);
|
|
396
|
+
if (settled !== undefined)
|
|
397
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is settled ${settled.settle}; a settled checkpoint never extends (SPEC 13.6)`);
|
|
398
|
+
if (current.value.state !== "waiting")
|
|
399
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is ${current.value.state}; only a waiting checkpoint extends (SPEC 13.6)`);
|
|
400
|
+
// DEADLINE FENCE: a checkpoint at/after its current authoritative deadline is DUE — it must
|
|
401
|
+
// expire, not be revived. A heartbeat only extends a still-live checkpoint (SPEC 13.6).
|
|
402
|
+
if (now >= current.value.deadline)
|
|
403
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is at/after its deadline ${current.value.deadline} (now ${now}); a due checkpoint expires and cannot be extended (SPEC 13.6)`);
|
|
404
|
+
if (!Number.isSafeInteger(deadline) || deadline <= now)
|
|
405
|
+
throw new EpEnvelopeError("failed-precondition", `the extended deadline must be in the owner's future (deadline ${deadline}, now ${now})`);
|
|
406
|
+
if (deadline > MAX_SCHEDULE_MS)
|
|
407
|
+
throw new EpEnvelopeError("failed-precondition", `the extended deadline ${deadline} exceeds the scheduler's representable range (${MAX_SCHEDULE_MS}); a heartbeat never advances a checkpoint to an unarmable deadline (distsys 8dcad72 M4, SPEC 13.6/13.9)`);
|
|
408
|
+
if (current.value.deadlineGeneration + 1 > Number.MAX_SAFE_INTEGER)
|
|
409
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" generation would overflow; reconcile the store (SPEC 13.6)`);
|
|
410
|
+
const next = assertStatusValue({
|
|
411
|
+
state: "waiting", deadlineGeneration: current.value.deadlineGeneration + 1, deadline,
|
|
412
|
+
observedSpecRevision: current.value.observedSpecRevision,
|
|
413
|
+
});
|
|
414
|
+
// The status CAS is the shared coordinate: a concurrent settle CAS-ing the SAME revision loses,
|
|
415
|
+
// so this heartbeat and any fire/resume settlement serialize on the status revision.
|
|
416
|
+
try {
|
|
417
|
+
await updateRecordEntry(kv, recordStatusKey(RECORD_KINDS.cp, cpQualifiers(ref)), next, current.revision);
|
|
418
|
+
}
|
|
419
|
+
catch (e) {
|
|
420
|
+
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
421
|
+
throw e;
|
|
422
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" was settled or heartbeat concurrently; re-read and re-decide (SPEC 13.6)`);
|
|
423
|
+
}
|
|
424
|
+
await emitScheduleRequest(js, space, { endpoint: ref.endpoint, instanceId, epoch, token: ref.token, generation: next.deadlineGeneration, deadline });
|
|
425
|
+
return next;
|
|
426
|
+
}
|
|
427
|
+
// ---- the timer writer's seam (§13.2/§13.9/§13.12) --------------------------------------------
|
|
428
|
+
/** Every header that would make bytes scheduling-active — a `.schedule` REQUEST carrying any
|
|
429
|
+
* of them is REJECTED by the writer (the ADR-51 confused deputy: schedule headers are copied
|
|
430
|
+
* to the target verbatim, so a header-carrying request could install another instance's
|
|
431
|
+
* schedule state if the writer ever echoed it). */
|
|
432
|
+
const SCHEDULING_HEADERS = ["Nats-Schedule", "Nats-Schedule-Target", "Nats-Schedule-Rollup", "Nats-Scheduler"];
|
|
433
|
+
/** The largest ms-epoch the broker's `@at <ISO>` schedule can represent (JS Date's ISO range). */
|
|
434
|
+
const MAX_SCHEDULE_MS = Date.UTC(9999, 11, 31, 23, 59, 59, 999);
|
|
435
|
+
const TIMER_RESOURCES = new WeakMap();
|
|
436
|
+
export async function timerWriterContext(nc, space) {
|
|
437
|
+
if (nc === null || typeof nc !== "object")
|
|
438
|
+
throw new EpEnvelopeError("failed-precondition", `a timer-writer context requires a NATS connection (SPEC 13.2)`);
|
|
439
|
+
if (typeof space !== "string" || space.length === 0)
|
|
440
|
+
throw new EpEnvelopeError("failed-precondition", `a timer-writer context requires a nonempty space (SPEC 13.2)`);
|
|
441
|
+
const js = jetstream(nc);
|
|
442
|
+
const jsm = await jetstreamManager(nc);
|
|
443
|
+
const ctx = Object.freeze({ space });
|
|
444
|
+
TIMER_RESOURCES.set(ctx, { js, jsm });
|
|
445
|
+
return ctx;
|
|
446
|
+
}
|
|
447
|
+
/** Brand check AND resource fetch in one step: only a token minted by {@link timerWriterContext}
|
|
448
|
+
* has an entry. */
|
|
449
|
+
function timerResources(ctx) {
|
|
450
|
+
const r = ctx === null || typeof ctx !== "object" ? undefined : TIMER_RESOURCES.get(ctx);
|
|
451
|
+
if (r === undefined)
|
|
452
|
+
throw new EpEnvelopeError("permission-denied", `the timer-writer context was not constructed by timerWriterContext(); a hand-assembled context never attests its resources (SPEC 13.2)`);
|
|
453
|
+
return r;
|
|
454
|
+
}
|
|
455
|
+
/** Race the fresh-check against the writer's budget: a stuck status authority is a bounded
|
|
456
|
+
* `unavailable` refusal, never a hung writer. Races `Promise.resolve(p)` unconditionally so a
|
|
457
|
+
* non-native thenable cannot bypass the deadline. */
|
|
458
|
+
async function withStatusBudget(p, budgetMs, what) {
|
|
459
|
+
let timer;
|
|
460
|
+
const deadline = new Promise((_, reject) => {
|
|
461
|
+
timer = setTimeout(() => reject(new EpEnvelopeError("unavailable", `${what} did not answer within ${budgetMs}ms; the timer writer is bounded and fails closed (SPEC 13.2)`)), budgetMs);
|
|
462
|
+
});
|
|
463
|
+
try {
|
|
464
|
+
return await Promise.race([Promise.resolve(p), deadline]);
|
|
465
|
+
}
|
|
466
|
+
finally {
|
|
467
|
+
clearTimeout(timer);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
/** LEADER-SERVED status read for the writer's fresh-check (distsys 8dcad72 M2): the fence the
|
|
471
|
+
* arm decision rests on needs read-your-writes against the leader. `kv.get` on the
|
|
472
|
+
* `allow_direct` records bucket may be FOLLOWER-served and answer a superseded generation —
|
|
473
|
+
* a fresh-check over that answer would arm exactly the stale deadline the check exists to
|
|
474
|
+
* discard. Same rule and mechanism as {@link readLastFact}. */
|
|
475
|
+
async function readCheckpointStatusLeader(jsm, space_, ref) {
|
|
476
|
+
const key = recordStatusKey(RECORD_KINDS.cp, cpQualifiers(ref));
|
|
477
|
+
const entry = await readRecordLeader(jsm, space_, key);
|
|
478
|
+
return entry === undefined ? undefined : { value: parseCpStatus(entry.value, key), revision: entry.revision };
|
|
479
|
+
}
|
|
480
|
+
/** The `.armed` subject's current last stream sequence (0 = no armed message), leader-served:
|
|
481
|
+
* the CAS coordinate of the arm fence below. */
|
|
482
|
+
async function readLastArmedSeq(jsm, space_, armedSubject) {
|
|
483
|
+
try {
|
|
484
|
+
const m = await jsm.streams.getMessage(eptStreamName(space_), { last_by_subj: armedSubject });
|
|
485
|
+
return m?.seq ?? 0;
|
|
486
|
+
}
|
|
487
|
+
catch (e) {
|
|
488
|
+
if (e.code === 10037)
|
|
489
|
+
return 0; // no message on the subject
|
|
490
|
+
throw e;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
/** One FENCED arm of `(generation, deadline)` onto the derived armed/fire subjects — the arm
|
|
494
|
+
* fence (distsys 8dcad72 HIGH). Order is load-bearing: (1) read the `.armed` subject's last
|
|
495
|
+
* sequence, (2) prove the authoritative status LEADER-SERVED, (3) publish `.armed` with
|
|
496
|
+
* `Nats-Expected-Last-Subject-Sequence` pinned to read (1). A publish can then land ONLY if no
|
|
497
|
+
* other `.armed` landed after this writer's sequence read — and its status proof is newer than
|
|
498
|
+
* that read — so a DELAYED writer that proved a since-superseded generation cannot replace the
|
|
499
|
+
* newer schedule: the broker rejects its publish, the bounded retry re-proves the (moved)
|
|
500
|
+
* status, and the stale coordinate is discarded. The broker orders the fence, never process
|
|
501
|
+
* timing. A lost CAS with the coordinate still current retries (a competing writer armed the
|
|
502
|
+
* SAME coordinate — the retry converges on the rollup). */
|
|
503
|
+
async function armCoordinate(r, space_, parsed, want, budget, subject) {
|
|
504
|
+
const armedSubject = eptSubject(space_, parsed.endpoint, parsed.instanceId, parsed.epoch, parsed.timerId, "armed");
|
|
505
|
+
const fireSubject = eptSubject(space_, parsed.endpoint, parsed.instanceId, parsed.epoch, parsed.timerId, "fire");
|
|
506
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
507
|
+
const lastSeq = await withStatusBudget(readLastArmedSeq(r.jsm, space_, armedSubject), budget, `the .armed sequence read for ${subject}`);
|
|
508
|
+
const current = await withStatusBudget(readCheckpointStatusLeader(r.jsm, space_, { endpoint: parsed.endpoint, token: parsed.timerId }), budget, `the status fresh-check for ${subject}`);
|
|
509
|
+
if (current === undefined)
|
|
510
|
+
return { armed: false, reason: "unknown" };
|
|
511
|
+
if (current.value.state !== "waiting")
|
|
512
|
+
return { armed: false, reason: "settled" };
|
|
513
|
+
if (current.value.deadlineGeneration !== want.generation || current.value.deadline !== want.deadline)
|
|
514
|
+
return { armed: false, reason: "stale" }; // a heartbeat superseded this coordinate; arming it would roll back the live deadline
|
|
515
|
+
const h = natsHeaders();
|
|
516
|
+
h.set("Nats-Schedule", `@at ${new Date(want.deadline).toISOString()}`);
|
|
517
|
+
h.set("Nats-Schedule-Target", fireSubject);
|
|
518
|
+
h.set("Nats-Expected-Last-Subject-Sequence", String(lastSeq));
|
|
519
|
+
try {
|
|
520
|
+
await r.js.publish(armedSubject, new TextEncoder().encode(JSON.stringify({ v: 1, timerId: parsed.timerId, generation: want.generation, deadline: want.deadline })), { headers: h });
|
|
521
|
+
}
|
|
522
|
+
catch (e) {
|
|
523
|
+
if (isCasLoss(e))
|
|
524
|
+
continue; // a competing .armed landed after the sequence read — re-prove and retry
|
|
525
|
+
throw e;
|
|
526
|
+
}
|
|
527
|
+
return { armed: true, armedSubject, fireSubject, generation: want.generation };
|
|
528
|
+
}
|
|
529
|
+
throw new EpEnvelopeError("conflict", `the .armed subject for ${subject} moved on every fenced attempt; re-deliver the request and retry (SPEC 13.6)`);
|
|
530
|
+
}
|
|
531
|
+
/** The TIMER WRITER: turn one authenticated `.schedule` request into the authoritative `.armed`
|
|
532
|
+
* publish. The armed/fire subjects derive from the REQUEST SUBJECT's own tokens (never body
|
|
533
|
+
* fields); the body must agree with the subject's timerId; any scheduling header on the request
|
|
534
|
+
* is a loud refusal; a request whose subject space is not the context's space is refused (the
|
|
535
|
+
* writer's status authority answers for ONE space). The writer FRESH-CHECKS the authoritative
|
|
536
|
+
* `(generation, deadline)` LEADER-SERVED within a bounded budget and arms ONLY the current
|
|
537
|
+
* generation under the {@link armCoordinate} broker CAS fence — a stale/delayed request is
|
|
538
|
+
* DISCARDED (`{ armed: false }`) and a delayed PUBLISH is rejected by the broker itself, so
|
|
539
|
+
* `.armed` can never roll back to a superseded deadline. A current request re-derives the same
|
|
540
|
+
* `.armed`; the server rollup makes it an idempotent no-op replacement (what the reconciler's
|
|
541
|
+
* over-emission rests on). After a successful arm the writer RE-READS the status and, if a
|
|
542
|
+
* heartbeat advanced it mid-flight, immediately arms the LIVE coordinate (bounded self-heal;
|
|
543
|
+
* the durable reconciler stays the crash backstop). */
|
|
544
|
+
export async function armCheckpointTimer(ctx, msg, opts) {
|
|
545
|
+
const resources = timerResources(ctx);
|
|
546
|
+
// Snapshot the FULL request input to detached locals at entry (the subject, the headers ref, and
|
|
547
|
+
// the body bytes are read EXACTLY ONCE, before any parse/space-check/publish): a shifting subject
|
|
548
|
+
// getter must not let the writer parse one coordinate, then space-check, arm, or error against a
|
|
549
|
+
// different one (distsys 8dcad72 M5). Nothing below reads `msg` again.
|
|
550
|
+
const subject = msg.subject;
|
|
551
|
+
const headers = msg.headers;
|
|
552
|
+
const data = new Uint8Array(msg.data); // COPY, not a reference: a caller-controlled header getter invoked in the scheduling-header scan below could otherwise mutate the live msg.data buffer between here and the decode (a Node Buffer/Uint8Array aliases through `const`). new Uint8Array(...) detaches the bytes at entry (distsys 6e8634d M5 re-open, same class as db85c4f).
|
|
553
|
+
if (typeof subject !== "string" || subject.length === 0)
|
|
554
|
+
throw new EpEnvelopeError("failed-precondition", `a .schedule request carries a subject string (SPEC 13.2)`);
|
|
555
|
+
const budget = opts?.statusBudgetMs ?? 5_000;
|
|
556
|
+
if (!Number.isSafeInteger(budget) || budget <= 0)
|
|
557
|
+
throw new EpEnvelopeError("failed-precondition", `statusBudgetMs must be a positive integer; got ${JSON.stringify(opts?.statusBudgetMs)}`);
|
|
558
|
+
for (const h of SCHEDULING_HEADERS)
|
|
559
|
+
if (headers?.get(h))
|
|
560
|
+
throw new EpEnvelopeError("permission-denied", `the .schedule request on ${subject} carries the scheduling header ${h}; a request's headers are inert bytes and the writer rejects them - only the writer's own .armed publish schedules (SPEC 13.2, ADR-51)`);
|
|
561
|
+
const parsed = parseEpSubject(subject);
|
|
562
|
+
if (parsed === null || parsed.plane !== "timer" || parsed.phase !== "schedule")
|
|
563
|
+
throw new EpEnvelopeError("failed-precondition", `${subject} is not a .schedule request subject; the writer arms nothing else (SPEC 13.2)`);
|
|
564
|
+
if (space(subject) !== ctx.space)
|
|
565
|
+
throw new EpEnvelopeError("permission-denied", `the .schedule request on ${subject} is for space "${space(subject)}" but this writer serves "${ctx.space}"; a cross-space fresh-check would answer with the wrong authority (SPEC 13.2)`);
|
|
566
|
+
let body;
|
|
567
|
+
try {
|
|
568
|
+
body = JSON.parse(new TextDecoder().decode(data));
|
|
569
|
+
}
|
|
570
|
+
catch (e) {
|
|
571
|
+
throw new EpEnvelopeError("failed-precondition", `the .schedule request on ${subject} does not decode as JSON: ${e.message}`);
|
|
572
|
+
}
|
|
573
|
+
const o = (body ?? {});
|
|
574
|
+
if (o.v !== 1 || o.timerId !== parsed.timerId
|
|
575
|
+
|| typeof o.generation !== "number" || !Number.isSafeInteger(o.generation) || o.generation < 1
|
|
576
|
+
|| typeof o.deadline !== "number" || !Number.isSafeInteger(o.deadline) || o.deadline < 0 || o.deadline > MAX_SCHEDULE_MS)
|
|
577
|
+
throw new EpEnvelopeError("failed-precondition", `the .schedule request on ${subject} is malformed, its body timerId disagrees with the authenticated subject token, or its deadline is out of the scheduler's date range (SPEC 13.2)`);
|
|
578
|
+
// The FENCED arm: leader-served fresh-check + broker subject-CAS (armCoordinate).
|
|
579
|
+
const result = await armCoordinate(resources, ctx.space, parsed, { generation: o.generation, deadline: o.deadline }, budget, subject);
|
|
580
|
+
if (!result.armed)
|
|
581
|
+
return result;
|
|
582
|
+
// POST-PUBLISH SELF-HEAL: if a heartbeat advanced the status while this arm was in flight, the
|
|
583
|
+
// request for the NEW coordinate may itself be delayed — arm the LIVE coordinate now (one
|
|
584
|
+
// bounded pass) instead of leaving the superseded schedule live until that request or the
|
|
585
|
+
// reconciler's next scan lands. A `conflict` here means a competing writer is already arming
|
|
586
|
+
// the live schedule: the winner is observed, never fought.
|
|
587
|
+
const live = await withStatusBudget(readCheckpointStatusLeader(resources.jsm, ctx.space, { endpoint: parsed.endpoint, token: parsed.timerId }), budget, `the post-publish re-read for ${subject}`);
|
|
588
|
+
if (live !== undefined && live.value.state === "waiting"
|
|
589
|
+
&& (live.value.deadlineGeneration !== o.generation || live.value.deadline !== o.deadline)) {
|
|
590
|
+
try {
|
|
591
|
+
await armCoordinate(resources, ctx.space, parsed, { generation: live.value.deadlineGeneration, deadline: live.value.deadline }, budget, subject);
|
|
592
|
+
}
|
|
593
|
+
catch (e) {
|
|
594
|
+
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
595
|
+
throw e;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return result;
|
|
599
|
+
}
|
|
600
|
+
function space(subject) {
|
|
601
|
+
const parts = subject.split(".");
|
|
602
|
+
if (parts[0] !== "cotal" || parts.length < 3)
|
|
603
|
+
throw new EpEnvelopeError("internal", `subject ${subject} carries no space prefix`);
|
|
604
|
+
return parts[1];
|
|
605
|
+
}
|
|
606
|
+
/** Handle one fired timer message: act ONLY on a fire whose broker-authored `Nats-Scheduler`
|
|
607
|
+
* header equals the exact sibling `.armed` subject (anything else is forged and discarded),
|
|
608
|
+
* whose `(timerId, generation)` matches the CURRENT waiting status (stale fires no-op: a
|
|
609
|
+
* heartbeat superseded that deadline), and whose authoritative deadline is due at the
|
|
610
|
+
* owner's clock. A valid fire EXPIRES the checkpoint by claiming the one-use settle CAS —
|
|
611
|
+
* where a concurrent resume may have lawfully claimed first: the fire then no-ops observing
|
|
612
|
+
* it. Expiry fails the checkpoint CLOSED (status `expired`; the paused goal's own terminal
|
|
613
|
+
* path is the action commit point). */
|
|
614
|
+
export async function handleCheckpointFire(kv, js, jsm, space_, args) {
|
|
615
|
+
// Snapshot the FULL operation input to detached locals at entry (subject, header, and body
|
|
616
|
+
// bytes are read exactly once, before any await); nothing below reads args again.
|
|
617
|
+
const ref = snapshotCpRef(args.ref);
|
|
618
|
+
const instanceId = args.instanceId;
|
|
619
|
+
const epoch = args.epoch;
|
|
620
|
+
const now = assertOwnerClock(args.now);
|
|
621
|
+
const subject = args.msg.subject;
|
|
622
|
+
const schedulerHeader = args.msg.headers?.get("Nats-Scheduler");
|
|
623
|
+
const dataBytes = args.msg.data;
|
|
624
|
+
const parsed = parseEpSubject(subject);
|
|
625
|
+
// Bind the FULL resource coordinate (endpoint + instance + epoch + token), not just the token:
|
|
626
|
+
// a valid broker fire for another instance/endpoint sharing this token cannot settle this ref.
|
|
627
|
+
if (parsed === null || parsed.plane !== "timer" || parsed.phase !== "fire"
|
|
628
|
+
|| parsed.endpoint !== ref.endpoint || parsed.instanceId !== instanceId
|
|
629
|
+
|| parsed.epoch !== epoch || parsed.timerId !== ref.token)
|
|
630
|
+
return { acted: false, reason: "forged-origin" };
|
|
631
|
+
const expectedArmed = eptSubject(space_, parsed.endpoint, parsed.instanceId, parsed.epoch, parsed.timerId, "armed");
|
|
632
|
+
if (schedulerHeader !== expectedArmed)
|
|
633
|
+
return { acted: false, reason: "forged-origin" }; // only the broker's scheduler stamps this header with the schedule's own subject
|
|
634
|
+
let body;
|
|
635
|
+
try {
|
|
636
|
+
body = JSON.parse(new TextDecoder().decode(dataBytes));
|
|
637
|
+
}
|
|
638
|
+
catch {
|
|
639
|
+
return { acted: false, reason: "forged-origin" };
|
|
640
|
+
}
|
|
641
|
+
const generation = body.generation;
|
|
642
|
+
const settledAlready = await settledOrConverge(kv, js, jsm, space_, ref);
|
|
643
|
+
if (settledAlready !== undefined)
|
|
644
|
+
return { acted: false, reason: "not-waiting" };
|
|
645
|
+
const status = await readCheckpointStatus(kv, ref);
|
|
646
|
+
if (status === undefined || status.value.state !== "waiting")
|
|
647
|
+
return { acted: false, reason: "not-waiting" };
|
|
648
|
+
if (generation !== status.value.deadlineGeneration)
|
|
649
|
+
return { acted: false, reason: "stale-generation" };
|
|
650
|
+
if (now < status.value.deadline) {
|
|
651
|
+
// A genuine broker fire under owner-behind clock skew: re-emit the CURRENT generation's
|
|
652
|
+
// schedule so the deadline is not silently lost (over-emission is idempotent at the writer).
|
|
653
|
+
await emitScheduleRequest(js, space_, { endpoint: ref.endpoint, instanceId, epoch, token: ref.token, generation: status.value.deadlineGeneration, deadline: status.value.deadline });
|
|
654
|
+
return { acted: false, reason: "re-armed" };
|
|
655
|
+
}
|
|
656
|
+
const settled = await settleCheckpoint(kv, js, jsm, space_, { ref, settle: "expired", now, statusEntry: status });
|
|
657
|
+
if (settled.outcome === "stale")
|
|
658
|
+
return { acted: false, reason: "stale-generation" }; // a heartbeat advanced the generation on the shared status revision
|
|
659
|
+
return { acted: true, settle: settled.fact, won: settled.outcome === "won" };
|
|
660
|
+
}
|
|
661
|
+
/** Resume a waiting checkpoint: HOLDER-BOUND (the spec's recorded holder MUST be the
|
|
662
|
+
* authenticated presenter) and ONE-USE (the status-arbiter CAS is the single settlement). A
|
|
663
|
+
* resume AT/AFTER the authoritative deadline does NOT claim `resumed` — expiry fails closed, so
|
|
664
|
+
* it drives/observes the EXPIRED settlement instead. Contends on the status revision with a
|
|
665
|
+
* concurrent heartbeat/fire: a stale (heartbeat-superseded) attempt retries once. */
|
|
666
|
+
export async function resumeCheckpoint(kv, js, jsm, space_, args) {
|
|
667
|
+
// Snapshot the FULL operation input to detached locals at entry; nothing below reads args again.
|
|
668
|
+
const ref = snapshotCpRef(args.ref);
|
|
669
|
+
const presenter = snapshotHolder(args.presenter, "a resume presenter");
|
|
670
|
+
const now = assertOwnerClock(args.now);
|
|
671
|
+
const specKey = recordSpecKey(RECORD_KINDS.cp, cpQualifiers(ref));
|
|
672
|
+
const specEntry = await kv.get(specKey);
|
|
673
|
+
if (!specEntry || specEntry.operation !== "PUT")
|
|
674
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is unknown (or its spec carries a deletion marker); resume presents only a minted token (SPEC 13.6)`);
|
|
675
|
+
const spec = parseCpSpec(JSON.parse(new TextDecoder().decode(specEntry.value)), ref, specKey);
|
|
676
|
+
if (spec.holder.id !== presenter.id || spec.holder.lifecycleUid !== presenter.lifecycleUid)
|
|
677
|
+
throw new EpEnvelopeError("permission-denied", `resume of checkpoint "${ref.token}" is holder-bound to ${spec.holder.id}/${spec.holder.lifecycleUid}; the presenter ${presenter.id}/${presenter.lifecycleUid} is not the holder (SPEC 13.10)`);
|
|
678
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
679
|
+
const already = await settledOrConverge(kv, js, jsm, space_, ref);
|
|
680
|
+
if (already !== undefined)
|
|
681
|
+
throw new EpEnvelopeError(already.settle === "resumed" ? "conflict" : "failed-precondition", `checkpoint "${ref.token}" is already settled ${already.settle} at ${already.ts}; resume authorization is one-use and expiry fails closed (SPEC 13.6)`);
|
|
682
|
+
const status = await readCheckpointStatus(kv, ref);
|
|
683
|
+
if (status === undefined)
|
|
684
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" has no status; reconcile the store (SPEC 13.6)`);
|
|
685
|
+
if (status.value.state !== "waiting")
|
|
686
|
+
continue; // settled between the two reads — the loop's settledOrConverge will surface it
|
|
687
|
+
// DEADLINE FENCE: at/after the authoritative deadline, resume MUST NOT win — expiry fails
|
|
688
|
+
// closed, so drive the EXPIRED settlement (the caller sees failed-precondition below).
|
|
689
|
+
const settle = now >= status.value.deadline ? "expired" : "resumed";
|
|
690
|
+
const settled = await settleCheckpoint(kv, js, jsm, space_, {
|
|
691
|
+
ref, settle, now, statusEntry: status,
|
|
692
|
+
...(settle === "resumed" ? { holder: presenter } : {}),
|
|
693
|
+
});
|
|
694
|
+
if (settled.outcome === "stale")
|
|
695
|
+
continue; // a heartbeat advanced the generation — retry once
|
|
696
|
+
if (settled.outcome === "won" && settle === "resumed")
|
|
697
|
+
return settled.fact;
|
|
698
|
+
const winner = settled.fact;
|
|
699
|
+
throw new EpEnvelopeError(winner.settle === "resumed" ? "conflict" : "failed-precondition", winner.settle === "resumed"
|
|
700
|
+
? `checkpoint "${ref.token}" was already resumed at ${winner.ts}; resume authorization is one-use (SPEC 13.6)`
|
|
701
|
+
: `checkpoint "${ref.token}" expired at ${winner.ts}${settle === "expired" ? " (resume after deadline fails closed)" : " before this resume"}; expiry fails the checkpoint closed (SPEC 13.6)`);
|
|
702
|
+
}
|
|
703
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" status moved twice during resume; re-read and re-decide (SPEC 13.6)`);
|
|
704
|
+
}
|
|
705
|
+
/** Read the recorded settlement (`undefined` = still waiting). */
|
|
706
|
+
export async function readCheckpointSettle(jsm, space_, ref) {
|
|
707
|
+
const subject = checkpointSettleSubject(space_, ref);
|
|
708
|
+
const raw = await readLastFact(jsm, epfStreamName(space_), subject);
|
|
709
|
+
return raw === undefined ? undefined : parseSettle(raw, subject, ref);
|
|
710
|
+
}
|
|
711
|
+
/** Read the checkpoint's recorded SPEC (`undefined` = unknown token; fail-closed on a deletion
|
|
712
|
+
* marker). The spec records WHAT the checkpoint pauses (its optional `goal` binding) and WHO
|
|
713
|
+
* may resume (`holder`) — a caller that must confirm a token pauses THIS goal reads it here. */
|
|
714
|
+
export async function readCheckpointSpec(kv, ref) {
|
|
715
|
+
const key = recordSpecKey(RECORD_KINDS.cp, cpQualifiers(ref));
|
|
716
|
+
const entry = await kv.get(key);
|
|
717
|
+
if (!entry)
|
|
718
|
+
return undefined;
|
|
719
|
+
if (entry.operation !== "PUT")
|
|
720
|
+
throw new EpEnvelopeError("failed-precondition", `the checkpoint spec ${key} carries a ${entry.operation} marker; a deletion never erases a pause - reconcile the store (SPEC 13.6)`);
|
|
721
|
+
return parseCpSpec(JSON.parse(new TextDecoder().decode(entry.value)), ref, key);
|
|
722
|
+
}
|
|
723
|
+
/** OWNER-forced expiry (§13.6): the pause's owner settles a DUE checkpoint `expired` without a
|
|
724
|
+
* broker fire — used when the owner already knows the hold deadline passed (a guard-hold
|
|
725
|
+
* expiry). Idempotent and fail-closed: an already-settled checkpoint returns its recorded
|
|
726
|
+
* settlement (the owner observes the winner); a still-live checkpoint (`now < deadline`)
|
|
727
|
+
* REFUSES (only a due checkpoint expires); a stale-generation CAS loss re-reads the winner.
|
|
728
|
+
* Never resets a settled checkpoint and never fabricates a settlement. */
|
|
729
|
+
export async function expireCheckpoint(kv, js, jsm, space_, args) {
|
|
730
|
+
const ref = snapshotCpRef(args.ref);
|
|
731
|
+
const now = assertOwnerClock(args.now);
|
|
732
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
733
|
+
const already = await settledOrConverge(kv, js, jsm, space_, ref);
|
|
734
|
+
if (already !== undefined)
|
|
735
|
+
return already; // already settled (resumed or expired) — observe the winner
|
|
736
|
+
const status = await readCheckpointStatus(kv, ref);
|
|
737
|
+
if (status === undefined)
|
|
738
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is unknown; an owner expiry settles only a minted checkpoint (SPEC 13.6)`);
|
|
739
|
+
if (status.value.state !== "waiting")
|
|
740
|
+
continue; // settled between the two reads — the loop surfaces it
|
|
741
|
+
if (now < status.value.deadline)
|
|
742
|
+
throw new EpEnvelopeError("failed-precondition", `checkpoint "${ref.token}" is not yet due (deadline ${status.value.deadline}, now ${now}); only a DUE checkpoint is owner-expired (SPEC 13.6)`);
|
|
743
|
+
const settled = await settleCheckpoint(kv, js, jsm, space_, { ref, settle: "expired", now, statusEntry: status });
|
|
744
|
+
if (settled.outcome === "stale")
|
|
745
|
+
continue; // a heartbeat advanced the generation — retry once
|
|
746
|
+
return settled.fact;
|
|
747
|
+
}
|
|
748
|
+
throw new EpEnvelopeError("conflict", `checkpoint "${ref.token}" status moved twice during owner expiry; re-read and re-decide (SPEC 13.6)`);
|
|
749
|
+
}
|
|
750
|
+
/** Settle the checkpoint by CAS-ing the STATUS record (the arbiter) on the exact revision the
|
|
751
|
+
* caller validated, THEN deriving the one-use fact. The status CAS is the shared coordinate: a
|
|
752
|
+
* concurrent heartbeat (which CAS-advances the same revision) or a competing settle contends
|
|
753
|
+
* here, so a stale settler loses. Outcomes:
|
|
754
|
+
* - `won`: our CAS won → the fact is derived and published;
|
|
755
|
+
* - `lost`: a competing settle already settled the status → return the WINNER (derived +
|
|
756
|
+
* ensured); the loser observes it;
|
|
757
|
+
* - `stale`: the status advanced but is still `waiting` (a heartbeat moved the generation) →
|
|
758
|
+
* this settler's generation is superseded; the caller (fire) no-ops or (resume) retries. */
|
|
759
|
+
async function settleCheckpoint(kv, js, jsm, space_, args) {
|
|
760
|
+
const settledStatus = assertStatusValue({
|
|
761
|
+
state: args.settle,
|
|
762
|
+
deadlineGeneration: args.statusEntry.value.deadlineGeneration,
|
|
763
|
+
deadline: args.statusEntry.value.deadline,
|
|
764
|
+
observedSpecRevision: args.statusEntry.value.observedSpecRevision,
|
|
765
|
+
settledGeneration: args.statusEntry.value.deadlineGeneration,
|
|
766
|
+
...(args.holder !== undefined ? { settledHolder: args.holder } : {}),
|
|
767
|
+
settledTs: args.now,
|
|
768
|
+
});
|
|
769
|
+
try {
|
|
770
|
+
await updateRecordEntry(kv, recordStatusKey(RECORD_KINDS.cp, cpQualifiers(args.ref)), settledStatus, args.statusEntry.revision);
|
|
771
|
+
}
|
|
772
|
+
catch (e) {
|
|
773
|
+
if (!(e instanceof EpEnvelopeError && e.code === "conflict"))
|
|
774
|
+
throw e;
|
|
775
|
+
const now2 = await readCheckpointStatus(kv, args.ref);
|
|
776
|
+
if (now2 === undefined)
|
|
777
|
+
throw new EpEnvelopeError("internal", `checkpoint "${args.ref.token}" status vanished mid-settle (SPEC 13.6)`);
|
|
778
|
+
if (now2.value.state !== "waiting")
|
|
779
|
+
return { outcome: "lost", fact: await ensureSettleFact(kv, js, jsm, space_, args.ref, now2.value) };
|
|
780
|
+
return { outcome: "stale" }; // a heartbeat advanced the generation; this settler is superseded
|
|
781
|
+
}
|
|
782
|
+
return { outcome: "won", fact: await ensureSettleFact(kv, js, jsm, space_, args.ref, settledStatus) };
|
|
783
|
+
}
|
|
784
|
+
/** The durable reconciler's re-emission (§13.6): for every genuinely-`waiting` checkpoint the
|
|
785
|
+
* endpoint owns, re-emit the `.schedule` request at the CURRENT generation. Idempotent at the
|
|
786
|
+
* writer (same-generation re-arm is a rollup no-op), so over-emission is harmless and a missing
|
|
787
|
+
* schedule is repaired without observing whether one exists. GATED ON THE SETTLE FACT, not the
|
|
788
|
+
* status projection: an already-settled checkpoint whose status still lags `waiting` (the
|
|
789
|
+
* crash window) is converged and NOT re-armed — without this gate the reconciler would re-emit
|
|
790
|
+
* schedules forever for a settled checkpoint (the C1 timer leak). */
|
|
791
|
+
export async function reconcileCheckpointSchedule(kv, js, jsm, space_, args) {
|
|
792
|
+
const ref = snapshotCpRef(args.ref);
|
|
793
|
+
const instanceId = args.instanceId;
|
|
794
|
+
const epoch = args.epoch;
|
|
795
|
+
const status = await readCheckpointStatus(kv, ref);
|
|
796
|
+
if (status === undefined)
|
|
797
|
+
return { reEmitted: false };
|
|
798
|
+
// The settled gate is settledOrConverge for EVERY state, not a bare status early-return: a
|
|
799
|
+
// SETTLED status whose derived fact is missing (the crash window between the status CAS and
|
|
800
|
+
// the fact publish) is exactly what the durable reconciler must repair - converge ensures the
|
|
801
|
+
// fact exists before this seam declares nothing to do.
|
|
802
|
+
if ((await settledOrConverge(kv, js, jsm, space_, ref)) !== undefined)
|
|
803
|
+
return { reEmitted: false };
|
|
804
|
+
// Emit from a FRESH authoritative read taken AFTER the settle-gate (distsys 8dcad72 M6): the
|
|
805
|
+
// status may have advanced (a heartbeat CAS-ed a new generation) between the first read above and
|
|
806
|
+
// here. Re-emitting the STALE first-read generation would be discarded by the writer, leaving the
|
|
807
|
+
// LIVE generation unarmed until another scan. Re-read and emit the current generation; a
|
|
808
|
+
// checkpoint that settled or vanished in the meantime re-emits nothing.
|
|
809
|
+
const live = await readCheckpointStatus(kv, ref);
|
|
810
|
+
if (live === undefined || live.value.state !== "waiting")
|
|
811
|
+
return { reEmitted: false };
|
|
812
|
+
await emitScheduleRequest(js, space_, {
|
|
813
|
+
endpoint: ref.endpoint, instanceId, epoch,
|
|
814
|
+
token: ref.token, generation: live.value.deadlineGeneration, deadline: live.value.deadline,
|
|
815
|
+
});
|
|
816
|
+
return { reEmitted: true, generation: live.value.deadlineGeneration };
|
|
817
|
+
}
|
|
818
|
+
//# sourceMappingURL=endpoint-checkpoint.js.map
|