@allocator-one/rcl 4.5.8 → 4.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/dist/consensus/semantic-deduper.d.ts +2 -1
- package/dist/consensus/semantic-deduper.js +21 -10
- package/dist/consensus/semantic-deduper.js.map +1 -1
- package/dist/converge/attempt-budget.d.ts +11 -0
- package/dist/converge/attempt-budget.js +119 -61
- package/dist/converge/attempt-budget.js.map +1 -1
- package/dist/converge/cycle-remote.d.ts +6 -1
- package/dist/converge/cycle-remote.js +13 -3
- package/dist/converge/cycle-remote.js.map +1 -1
- package/dist/converge/delivery-reconciliation.d.ts +2 -1
- package/dist/converge/delivery-reconciliation.js +105 -11
- package/dist/converge/delivery-reconciliation.js.map +1 -1
- package/dist/converge/launch-guard.d.ts +3 -0
- package/dist/converge/launch-guard.js +53 -17
- package/dist/converge/launch-guard.js.map +1 -1
- package/dist/converge/ordinary-pending-export.d.ts +5 -0
- package/dist/converge/ordinary-pending-export.js +9 -1
- package/dist/converge/ordinary-pending-export.js.map +1 -1
- package/dist/converge/recovered-production.d.ts +14 -0
- package/dist/converge/recovered-production.js +61 -0
- package/dist/converge/recovered-production.js.map +1 -0
- package/dist/converge/recovery-launch.js +82 -35
- package/dist/converge/recovery-launch.js.map +1 -1
- package/dist/converge/run-state.d.ts +2 -0
- package/dist/converge/run-state.js +26 -12
- package/dist/converge/run-state.js.map +1 -1
- package/dist/converge/semantic-state.js +18 -4
- package/dist/converge/semantic-state.js.map +1 -1
- package/dist/coordination/registry-lock.d.ts +2 -0
- package/dist/coordination/registry-lock.js +42 -13
- package/dist/coordination/registry-lock.js.map +1 -1
- package/dist/evidence/claim-recovery/delivery.d.ts +25 -0
- package/dist/evidence/claim-recovery/delivery.js +100 -0
- package/dist/evidence/claim-recovery/delivery.js.map +1 -0
- package/dist/evidence/claim-recovery/public-adoption.d.ts +31 -0
- package/dist/evidence/claim-recovery/public-adoption.js +177 -0
- package/dist/evidence/claim-recovery/public-adoption.js.map +1 -0
- package/dist/evidence/claim-recovery/public-manifest.d.ts +114 -0
- package/dist/evidence/claim-recovery/public-manifest.js +11 -0
- package/dist/evidence/claim-recovery/public-manifest.js.map +1 -0
- package/dist/evidence/claim-recovery/public-model.d.ts +85 -0
- package/dist/evidence/claim-recovery/public-model.js +177 -0
- package/dist/evidence/claim-recovery/public-model.js.map +1 -0
- package/dist/evidence/original-run/journal.d.ts +10 -0
- package/dist/evidence/original-run/journal.js +55 -3
- package/dist/evidence/original-run/journal.js.map +1 -1
- package/dist/evidence/original-run/lock.js +22 -9
- package/dist/evidence/original-run/lock.js.map +1 -1
- package/dist/evidence/recover-claim.d.ts +21 -0
- package/dist/evidence/recover-claim.js +504 -0
- package/dist/evidence/recover-claim.js.map +1 -0
- package/dist/index.js +239 -137
- package/dist/index.js.map +1 -1
- package/dist/report/assembly.d.ts +2 -0
- package/dist/report/assembly.js +1 -0
- package/dist/report/assembly.js.map +1 -1
- package/dist/report/consensus-assembly.d.ts +2 -0
- package/dist/report/consensus-assembly.js +6 -3
- package/dist/report/consensus-assembly.js.map +1 -1
- package/dist/report/reviewer-artifact.js +3 -1
- package/dist/report/reviewer-artifact.js.map +1 -1
- package/dist/report/run-header.js +2 -0
- package/dist/report/run-header.js.map +1 -1
- package/dist/telemetry/deliver.d.ts +8 -0
- package/dist/telemetry/deliver.js +55 -8
- package/dist/telemetry/deliver.js.map +1 -1
- package/dist/telemetry/events.js +2 -0
- package/dist/telemetry/events.js.map +1 -1
- package/dist/telemetry/outbox.d.ts +20 -0
- package/dist/telemetry/outbox.js +324 -103
- package/dist/telemetry/outbox.js.map +1 -1
- package/dist/telemetry/reviewer-delivery.d.ts +95 -3
- package/dist/telemetry/reviewer-delivery.js +600 -12
- package/dist/telemetry/reviewer-delivery.js.map +1 -1
- package/dist/telemetry/sink.d.ts +14 -0
- package/dist/telemetry/sink.js +44 -2
- package/dist/telemetry/sink.js.map +1 -1
- package/dist/telemetry/terminal-reviewer-delivery.d.ts +154 -5
- package/dist/telemetry/terminal-reviewer-delivery.js +196 -21
- package/dist/telemetry/terminal-reviewer-delivery.js.map +1 -1
- package/package.json +1 -1
|
@@ -4,21 +4,61 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { isDeepStrictEqual } from 'node:util';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
import { isReviewerArtifact } from '../report/reviewer-artifact.js';
|
|
7
|
-
import { withRecoveryLock, syncDirectory, writeExclusiveBytes } from '../evidence/original-run/journal.js';
|
|
7
|
+
import { MAX_RECOVERY_CHECKPOINT_BYTES, MAX_RECOVERY_DOCUMENT_BYTES, openJournal, withRecoveryLock, syncDirectory, writeExclusiveBytes } from '../evidence/original-run/journal.js';
|
|
8
8
|
import { prepareLockRoot, inspectRecoveryDirectory } from '../evidence/original-run/lock-path.js';
|
|
9
9
|
import { platformPath, readStable, sha256 } from './recovery/files.js';
|
|
10
10
|
import { normalizeUrl } from './credentials.js';
|
|
11
11
|
import { MAX_ARTIFACT_BYTES, MAX_ENVELOPE_BYTES, validateRunEnvelope } from './envelope-validation.js';
|
|
12
|
+
import { MAX_RESPONSE_BYTES } from './sink.js';
|
|
12
13
|
export const REVIEWER_OUTBOX_DIR = 'reviewer-outbox';
|
|
13
14
|
const uuid = z.string().regex(/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}(?![\s\S])/i);
|
|
14
15
|
const digest = z.string().regex(/^[0-9a-f]{64}(?![\s\S])/);
|
|
15
16
|
const reference = z.object({ sha256: digest, bytes: z.number().int().min(0).max(MAX_ARTIFACT_BYTES) }).strict();
|
|
16
17
|
const manifestSchema = z.object({ version: z.literal(1), runId: uuid, host: z.string(), credentialKind: z.enum(['login', 'env']),
|
|
17
18
|
envelope: reference, report_json: reference, report_md: reference.optional(), reviewer: reference }).strict();
|
|
19
|
+
const putOutcomePhases = ['report_json_put_outcome', 'report_md_put_outcome', 'reviewer_put_outcome'];
|
|
20
|
+
const MAX_PUT_RETRIES_PER_ARTIFACT = 100;
|
|
21
|
+
const MAX_PUT_OUTCOMES_PER_ARTIFACT = 1 + MAX_PUT_RETRIES_PER_ARTIFACT;
|
|
22
|
+
const PUT_COMPLETION_RESERVATION_CHECKPOINTS = 7;
|
|
23
|
+
const PUT_COMPLETION_RESERVATION_BYTES = PUT_COMPLETION_RESERVATION_CHECKPOINTS * MAX_RECOVERY_CHECKPOINT_BYTES;
|
|
24
|
+
const PUT_COMPLETION_AFTER_INTENT_CHECKPOINTS = PUT_COMPLETION_RESERVATION_CHECKPOINTS - 1;
|
|
25
|
+
const PUT_COMPLETION_AFTER_INTENT_BYTES = PUT_COMPLETION_AFTER_INTENT_CHECKPOINTS * MAX_RECOVERY_CHECKPOINT_BYTES;
|
|
26
|
+
// Sink response bodies are bounded before classification. The retained outcome
|
|
27
|
+
// keeps only kind/status/error. Invalid UTF-8 can expand to one three-byte
|
|
28
|
+
// replacement character per input byte, with room for its checkpoint wrapper.
|
|
29
|
+
const MAX_OUTCOME_CHECKPOINT_BYTES = 3 * MAX_RESPONSE_BYTES + 4096;
|
|
30
|
+
function putOutcomeAttempt(phase) {
|
|
31
|
+
for (const base of putOutcomePhases) {
|
|
32
|
+
if (phase === base)
|
|
33
|
+
return { base, attempt: 1 };
|
|
34
|
+
if (!phase.startsWith(`${base}_`))
|
|
35
|
+
continue;
|
|
36
|
+
const suffix = phase.slice(base.length + 1);
|
|
37
|
+
if (!/^(?:[2-9]|[1-9][0-9]+)$/.test(suffix))
|
|
38
|
+
return undefined;
|
|
39
|
+
const attempt = Number(suffix);
|
|
40
|
+
return Number.isSafeInteger(attempt) && attempt >= 2 ? { base, attempt } : undefined;
|
|
41
|
+
}
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
const isOutcomePhase = (phase) => phase === 'activation_post_outcome' || putOutcomeAttempt(phase) !== undefined;
|
|
45
|
+
const operationPhases = new Set(['prepared', 'activation_post_intent', 'activation_post_outcome', ...putOutcomePhases, 'activation_post_uncertain',
|
|
46
|
+
'envelope_verified', 'report_json_put_intent', 'report_json_verified',
|
|
47
|
+
'report_md_put_intent', 'report_md_verified', 'reviewer_put_intent',
|
|
48
|
+
'reviewer_verified', 'recovery_acknowledged', 'complete']);
|
|
49
|
+
const outcomeSchema = z.discriminatedUnion('kind', [
|
|
50
|
+
z.object({ kind: z.literal('ok'), http_status: z.number().int() }).strict(),
|
|
51
|
+
z.object({ kind: z.literal('disabled') }).strict(),
|
|
52
|
+
z.object({ kind: z.literal('conflict') }).strict(),
|
|
53
|
+
z.object({ kind: z.literal('rejected'), http_status: z.number().int(), error: z.string() }).strict(),
|
|
54
|
+
z.object({ kind: z.literal('unavailable'), http_status: z.number().int().optional() }).strict(),
|
|
55
|
+
]);
|
|
18
56
|
const files = { report_json: 'report.json', report_md: 'report.md' };
|
|
19
57
|
const ref = (text) => ({ sha256: sha256(text), bytes: Buffer.byteLength(text) });
|
|
20
58
|
function fail(reason) { throw new Error(`reviewer_delivery_${reason}`); }
|
|
21
59
|
function safeFailure(error) {
|
|
60
|
+
if (error instanceof Error && error.message === 'recovery_run_locked')
|
|
61
|
+
return 'reviewer_delivery_recovery_run_locked';
|
|
22
62
|
return error instanceof Error && /^reviewer_delivery_[a-z_]+$/.test(error.message) ? error.message : 'reviewer_delivery_local_refusal';
|
|
23
63
|
}
|
|
24
64
|
function accepted(outcome) {
|
|
@@ -59,6 +99,7 @@ async function publish(path, bytes) {
|
|
|
59
99
|
function validate(entry) {
|
|
60
100
|
const { manifest: m, envelope, artifacts, privateBytes } = entry;
|
|
61
101
|
if (normalizeUrl(m.host) !== m.host || m.envelope.bytes > MAX_ENVELOPE_BYTES || envelope.run.id !== m.runId ||
|
|
102
|
+
entry.manifestBytes !== JSON.stringify(m) || entry.envelopeBytes !== JSON.stringify(envelope) ||
|
|
62
103
|
validateRunEnvelope(envelope, artifacts).length || !isDeepStrictEqual(ref(entry.envelopeBytes), m.envelope) ||
|
|
63
104
|
!isDeepStrictEqual(ref(artifacts.report_json), m.report_json) || !isDeepStrictEqual(ref(privateBytes), m.reviewer) ||
|
|
64
105
|
(m.report_md === undefined ? artifacts.report_md !== undefined : !isDeepStrictEqual(ref(artifacts.report_md), m.report_md)))
|
|
@@ -96,14 +137,16 @@ export class ReviewerDeliveryQueue {
|
|
|
96
137
|
await this.lock(entry.manifest.runId, async () => { await this.save(entry); });
|
|
97
138
|
}
|
|
98
139
|
/** Initial delivery is the only path allowed to create an as-yet unknown run. */
|
|
99
|
-
async deliver(input, options = {}) {
|
|
140
|
+
async deliver(input, options = {}, authorizeTransfer) {
|
|
100
141
|
const entry = this.snapshot(input), request = budget(options);
|
|
101
142
|
await this.lock(entry.manifest.runId, async () => {
|
|
143
|
+
await this.assertGenericDeliveryAllowed(entry.manifest.runId);
|
|
144
|
+
await authorizeTransfer?.();
|
|
102
145
|
request();
|
|
103
146
|
await this.save(entry);
|
|
104
147
|
// An existing manifest might belong to an earlier process/principal.
|
|
105
148
|
// Only this invocation's first publication can use the creation path.
|
|
106
|
-
await this.transfer(await this.load(entry.manifest.runId), input.sink, request, entryWasNew.delete(entry));
|
|
149
|
+
await this.transfer(await this.load(entry.manifest.runId), input.sink, request, entryWasNew.delete(entry) ? 'initial' : 'retry');
|
|
107
150
|
});
|
|
108
151
|
}
|
|
109
152
|
/** Explicit/current-credential retry; no provider calls or changed run identity. */
|
|
@@ -127,11 +170,12 @@ export class ReviewerDeliveryQueue {
|
|
|
127
170
|
request();
|
|
128
171
|
await this.lock(id, async () => {
|
|
129
172
|
const entry = await this.load(id);
|
|
173
|
+
await this.assertGenericDeliveryAllowed(id);
|
|
130
174
|
if (await this.acknowledged(entry))
|
|
131
175
|
return;
|
|
132
176
|
await authorizeTransfer?.();
|
|
133
177
|
request();
|
|
134
|
-
await this.transfer(entry, sink, request,
|
|
178
|
+
await this.transfer(entry, sink, request, 'retry');
|
|
135
179
|
summary.delivered.push(entry.manifest.runId);
|
|
136
180
|
});
|
|
137
181
|
}
|
|
@@ -163,6 +207,106 @@ export class ReviewerDeliveryQueue {
|
|
|
163
207
|
return false;
|
|
164
208
|
}
|
|
165
209
|
}
|
|
210
|
+
/** Presence alone blocks legacy delivery; malformed retained state must fail closed too. */
|
|
211
|
+
async hasEntry(runId) {
|
|
212
|
+
if (!uuid.safeParse(runId).success)
|
|
213
|
+
return false;
|
|
214
|
+
try {
|
|
215
|
+
await lstat(join(this.root, runId.toLowerCase()));
|
|
216
|
+
return true;
|
|
217
|
+
}
|
|
218
|
+
catch (error) {
|
|
219
|
+
if (error.code === 'ENOENT')
|
|
220
|
+
return false;
|
|
221
|
+
throw error;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
/** Generic retry paths must never bypass an explicit retained activation operation. */
|
|
225
|
+
async assertGenericDeliveryAllowed(runId) {
|
|
226
|
+
if (!uuid.safeParse(runId).success)
|
|
227
|
+
fail('invalid_run');
|
|
228
|
+
try {
|
|
229
|
+
await lstat(join(this.root, runId.toLowerCase(), 'activation-intent.json'));
|
|
230
|
+
fail('explicit_activation_required');
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
if (error instanceof Error && error.message === 'reviewer_delivery_explicit_activation_required')
|
|
234
|
+
throw error;
|
|
235
|
+
if (error.code !== 'ENOENT')
|
|
236
|
+
throw error;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
/** Hold the run lock across a selected generic transmission so activation cannot race it. */
|
|
240
|
+
async withGenericDeliveryAllowed(runId, work) {
|
|
241
|
+
return this.lock(runId, async () => {
|
|
242
|
+
await this.assertGenericDeliveryAllowed(runId);
|
|
243
|
+
return work();
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
/** Read-only exact activation proposal derived from authenticated local lineage. */
|
|
247
|
+
async previewRecovery(selection) {
|
|
248
|
+
return (await this.prepareRecovery(selection)).preview;
|
|
249
|
+
}
|
|
250
|
+
recoveryJournalPath(runId, operationId) {
|
|
251
|
+
if (!uuid.safeParse(runId).success || !uuid.safeParse(operationId).success)
|
|
252
|
+
fail('recovery_operation_mismatch');
|
|
253
|
+
return join(this.root, runId.toLowerCase(), 'recovery.journal');
|
|
254
|
+
}
|
|
255
|
+
/** Read-only retained bytes after checkpoint lineage and private outbox authentication. */
|
|
256
|
+
async prepareRecovery(selection) {
|
|
257
|
+
if (!selection || typeof selection !== 'object' || !uuid.safeParse(selection.runId).success)
|
|
258
|
+
fail('invalid_recovery_selection');
|
|
259
|
+
return this.lock(selection.runId, async () => {
|
|
260
|
+
const entry = await this.load(selection.runId);
|
|
261
|
+
return { preview: this.recoveryPreview(entry, selection), envelopeBytes: entry.envelopeBytes,
|
|
262
|
+
...(entry.artifacts.report_md === undefined ? {} : { reportMarkdownBytes: entry.artifacts.report_md }) };
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
/** Reject conflicting local acknowledgements before notice, journal creation, or transport. */
|
|
266
|
+
async preflightRecoveryAcknowledgements(selection, preview, operationInput) {
|
|
267
|
+
if (!selection || typeof selection !== 'object' || !uuid.safeParse(selection.runId).success)
|
|
268
|
+
fail('invalid_recovery_selection');
|
|
269
|
+
await this.lock(selection.runId, async () => {
|
|
270
|
+
const entry = await this.load(selection.runId);
|
|
271
|
+
if (!isDeepStrictEqual(this.recoveryPreview(entry, selection), preview))
|
|
272
|
+
fail('recovery_selection_mismatch');
|
|
273
|
+
if (operationInput === undefined) {
|
|
274
|
+
await this.assertOrdinaryAcknowledgement(entry);
|
|
275
|
+
if (await this.optionalPrivateRead(join(entry.directory, 'recovery-acknowledged.json'), 4096) !== undefined) {
|
|
276
|
+
fail('immutable_conflict');
|
|
277
|
+
}
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const operation = await this.resolveRecoveryOperation(operationInput, selection.runId);
|
|
281
|
+
this.validateRecoveryOperation(entry, operation);
|
|
282
|
+
const activationIntent = this.activationIntent(entry, operation);
|
|
283
|
+
const recoveryAck = this.recoveryAcknowledgement(entry, operation, activationIntent);
|
|
284
|
+
this.assertRecoveryJournalAcknowledgements(operation, recoveryAck);
|
|
285
|
+
await this.assertRecoveryAcknowledgements(entry, recoveryAck);
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
/** Activate an exact retained entry. Generic flush never receives this creation authority. */
|
|
289
|
+
async applyRecovery(sink, selection, preview, operationInput, options = {}) {
|
|
290
|
+
await this.applyRecoveryOperation(sink, selection, preview, () => this.resolveRecoveryOperation(operationInput, selection.runId), options);
|
|
291
|
+
}
|
|
292
|
+
async applyRecoveryOperation(sink, selection, preview, resolveOperation, options = {}) {
|
|
293
|
+
if (!selection || typeof selection !== 'object' || !uuid.safeParse(selection.runId).success)
|
|
294
|
+
fail('invalid_recovery_selection');
|
|
295
|
+
const request = budget(options);
|
|
296
|
+
await this.lock(selection.runId, async () => {
|
|
297
|
+
const entry = await this.load(selection.runId);
|
|
298
|
+
const current = this.recoveryPreview(entry, selection);
|
|
299
|
+
if (!isDeepStrictEqual(current, preview))
|
|
300
|
+
fail('recovery_selection_mismatch');
|
|
301
|
+
const operation = await resolveOperation();
|
|
302
|
+
request();
|
|
303
|
+
await this.recover(entry, sink, request, operation);
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
/** Resume is the same exact operation; server readback decides which writes remain. */
|
|
307
|
+
async resumeRecovery(sink, selection, preview, operation, options = {}) {
|
|
308
|
+
await this.applyRecovery(sink, selection, preview, operation, options);
|
|
309
|
+
}
|
|
166
310
|
snapshot(input) {
|
|
167
311
|
if (!isReviewerArtifact(input.artifact))
|
|
168
312
|
fail('unvalidated_artifact');
|
|
@@ -172,13 +316,23 @@ export class ReviewerDeliveryQueue {
|
|
|
172
316
|
const artifacts = { ...input.artifacts }, privateBytes = input.artifact.bytes;
|
|
173
317
|
const manifest = manifestSchema.parse({ version: 1, runId: envelope.run.id, host: input.sink.baseUrl, credentialKind: input.sink.credentialSource,
|
|
174
318
|
envelope: ref(envelopeBytes), report_json: ref(artifacts.report_json), ...(artifacts.report_md === undefined ? {} : { report_md: ref(artifacts.report_md) }), reviewer: ref(privateBytes) });
|
|
175
|
-
const entry = { manifest, envelope, envelopeBytes, artifacts, privateBytes,
|
|
319
|
+
const entry = { manifest, manifestBytes: JSON.stringify(manifest), envelope, envelopeBytes, artifacts, privateBytes,
|
|
320
|
+
directory: join(this.root, manifest.runId.toLowerCase()) };
|
|
176
321
|
validate(entry);
|
|
177
322
|
return entry;
|
|
178
323
|
}
|
|
179
324
|
async lock(runId, work) {
|
|
180
325
|
return withRecoveryLock(join(this.root, 'locks'), runId.toLowerCase(), work);
|
|
181
326
|
}
|
|
327
|
+
async resolveRecoveryOperation(operation, runId) {
|
|
328
|
+
const manifestPath = platformPath(operation.manifestPath);
|
|
329
|
+
const retained = await readStable(manifestPath, MAX_RECOVERY_DOCUMENT_BYTES, { sync: true });
|
|
330
|
+
if (retained.sha256 !== operation.recoveryManifestSha256)
|
|
331
|
+
fail('manifest_digest_mismatch');
|
|
332
|
+
return { mode: operation.mode, operationId: operation.operationId,
|
|
333
|
+
recoveryManifestSha256: operation.recoveryManifestSha256, destination: operation.destination,
|
|
334
|
+
journal: await openJournal(this.recoveryJournalPath(runId, operation.operationId), operation.recoveryManifestSha256, operation.operationId, operation.mode) };
|
|
335
|
+
}
|
|
182
336
|
async save(entry) {
|
|
183
337
|
await prepareLockRoot(this.root);
|
|
184
338
|
try {
|
|
@@ -199,24 +353,107 @@ export class ReviewerDeliveryQueue {
|
|
|
199
353
|
if (entry.artifacts[kind] !== undefined)
|
|
200
354
|
await publish(join(entry.directory, files[kind]), entry.artifacts[kind]);
|
|
201
355
|
// Publish manifest last; a crash before it leaves evidence, never an uploadable partial entry.
|
|
202
|
-
await publish(join(entry.directory, 'manifest.json'),
|
|
356
|
+
await publish(join(entry.directory, 'manifest.json'), entry.manifestBytes);
|
|
203
357
|
}
|
|
204
358
|
async load(id) {
|
|
205
359
|
const directory = join(this.root, id.toLowerCase());
|
|
206
360
|
await inspectRecoveryDirectory(directory, true);
|
|
207
|
-
const
|
|
361
|
+
const manifestBytes = await privateRead(join(directory, 'manifest.json'), 4096);
|
|
362
|
+
const manifest = manifestSchema.parse(JSON.parse(manifestBytes));
|
|
208
363
|
if (manifest.runId.toLowerCase() !== id.toLowerCase())
|
|
209
364
|
fail('invalid_run');
|
|
210
|
-
const allowed = ['manifest.json', 'envelope.json', 'reviewer-artifact.json', 'report.json', 'acknowledged.json',
|
|
365
|
+
const allowed = ['manifest.json', 'envelope.json', 'reviewer-artifact.json', 'report.json', 'acknowledged.json',
|
|
366
|
+
'activation-intent.json', 'recovery-acknowledged.json', 'recovery.journal',
|
|
367
|
+
...(manifest.report_md ? ['report.md'] : [])];
|
|
211
368
|
if ((await readdir(directory)).some(name => !allowed.includes(name)))
|
|
212
369
|
fail('unknown_file');
|
|
213
370
|
const envelopeBytes = await privateRead(join(directory, 'envelope.json'), MAX_ENVELOPE_BYTES);
|
|
214
|
-
const entry = { manifest, directory, envelopeBytes, envelope: JSON.parse(envelopeBytes), privateBytes: await privateRead(join(directory, 'reviewer-artifact.json'), MAX_ARTIFACT_BYTES),
|
|
371
|
+
const entry = { manifest, manifestBytes, directory, envelopeBytes, envelope: JSON.parse(envelopeBytes), privateBytes: await privateRead(join(directory, 'reviewer-artifact.json'), MAX_ARTIFACT_BYTES),
|
|
215
372
|
artifacts: { report_json: await privateRead(join(directory, 'report.json'), MAX_ARTIFACT_BYTES), ...(manifest.report_md ? { report_md: await privateRead(join(directory, 'report.md'), MAX_ARTIFACT_BYTES) } : {}) } };
|
|
216
373
|
validate(entry);
|
|
217
374
|
return entry;
|
|
218
375
|
}
|
|
219
|
-
|
|
376
|
+
recoveryPreview(entry, selection) {
|
|
377
|
+
const report = ref(selection.reportBytes), reviewer = ref(selection.reviewerArtifactBytes);
|
|
378
|
+
const run = entry.envelope.run;
|
|
379
|
+
if (typeof selection.target !== 'string' || !selection.target || !/^[a-f0-9]{40}$/.test(selection.headSha) ||
|
|
380
|
+
selection.runId.toLowerCase() !== entry.manifest.runId.toLowerCase() || run.id !== entry.manifest.runId ||
|
|
381
|
+
run.converge?.target !== selection.target || run.target.head_sha !== selection.headSha ||
|
|
382
|
+
selection.reportSha256 !== report.sha256 || selection.reportByteLength !== report.bytes ||
|
|
383
|
+
selection.reviewerArtifactSha256 !== reviewer.sha256 || selection.reviewerArtifactByteLength !== reviewer.bytes ||
|
|
384
|
+
entry.artifacts.report_json !== selection.reportBytes || entry.privateBytes !== selection.reviewerArtifactBytes ||
|
|
385
|
+
!isDeepStrictEqual(entry.envelope.reviewer_recovery, selection.reviewerRecovery) ||
|
|
386
|
+
!isDeepStrictEqual(entry.manifest.report_json, report) || !isDeepStrictEqual(entry.manifest.reviewer, reviewer)) {
|
|
387
|
+
fail('recovery_selection_mismatch');
|
|
388
|
+
}
|
|
389
|
+
return { version: 1, target: selection.target, runId: entry.manifest.runId, headSha: selection.headSha,
|
|
390
|
+
host: entry.manifest.host, credentialKind: entry.manifest.credentialKind,
|
|
391
|
+
manifest: ref(entry.manifestBytes), envelope: { ...entry.manifest.envelope },
|
|
392
|
+
report_json: { ...entry.manifest.report_json },
|
|
393
|
+
...(entry.manifest.report_md ? { report_md: { ...entry.manifest.report_md } } : {}),
|
|
394
|
+
reviewer: { ...entry.manifest.reviewer } };
|
|
395
|
+
}
|
|
396
|
+
ack(entry) { return JSON.stringify({ version: 1, runId: entry.manifest.runId, manifestSha256: sha256(entry.manifestBytes) }); }
|
|
397
|
+
async optionalPrivateRead(path, limit) {
|
|
398
|
+
try {
|
|
399
|
+
return await privateRead(path, limit);
|
|
400
|
+
}
|
|
401
|
+
catch (error) {
|
|
402
|
+
if (error.code === 'ENOENT')
|
|
403
|
+
return undefined;
|
|
404
|
+
throw error;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
async assertOrdinaryAcknowledgement(entry) {
|
|
408
|
+
const bytes = await this.optionalPrivateRead(join(entry.directory, 'acknowledged.json'), 4096);
|
|
409
|
+
if (bytes !== undefined && bytes !== this.ack(entry))
|
|
410
|
+
fail('immutable_conflict');
|
|
411
|
+
}
|
|
412
|
+
activationIntent(entry, operation) {
|
|
413
|
+
return JSON.stringify({ version: 1, runId: entry.manifest.runId, operationId: operation.operationId,
|
|
414
|
+
recoveryManifestSha256: operation.recoveryManifestSha256, envelopeSha256: entry.manifest.envelope.sha256 });
|
|
415
|
+
}
|
|
416
|
+
validateRecoveryOperation(entry, operation) {
|
|
417
|
+
const m = entry.manifest;
|
|
418
|
+
if (!operation || !uuid.safeParse(operation.operationId).success || !digest.safeParse(operation.recoveryManifestSha256).success ||
|
|
419
|
+
!operation.journal || (operation.mode !== 'apply' && operation.mode !== 'resume') ||
|
|
420
|
+
operation.destination.host !== m.host || operation.destination.credentialKind !== m.credentialKind ||
|
|
421
|
+
operation.destination.activationProtocol !== 1)
|
|
422
|
+
fail('recovery_operation_mismatch');
|
|
423
|
+
}
|
|
424
|
+
recoveryOutcomeDigest(operation) {
|
|
425
|
+
const outcomes = operation.journal.checkpoints()
|
|
426
|
+
.filter(checkpoint => isOutcomePhase(checkpoint.phase))
|
|
427
|
+
.map(checkpoint => {
|
|
428
|
+
const parsed = outcomeSchema.safeParse(checkpoint.data);
|
|
429
|
+
if (!parsed.success || !isDeepStrictEqual(parsed.data, checkpoint.data))
|
|
430
|
+
fail('journal_checkpoint_conflict');
|
|
431
|
+
return { phase: checkpoint.phase, data: parsed.data };
|
|
432
|
+
});
|
|
433
|
+
return sha256(JSON.stringify(outcomes));
|
|
434
|
+
}
|
|
435
|
+
recoveryAcknowledgement(entry, operation, activationIntent) {
|
|
436
|
+
const m = entry.manifest;
|
|
437
|
+
return JSON.stringify({ version: 1, operationId: operation.operationId,
|
|
438
|
+
recoveryManifestSha256: operation.recoveryManifestSha256, outboxManifestSha256: sha256(entry.manifestBytes),
|
|
439
|
+
activationIntentSha256: sha256(activationIntent), journalOutcomesSha256: this.recoveryOutcomeDigest(operation),
|
|
440
|
+
destination: operation.destination, envelope: m.envelope, report_json: m.report_json,
|
|
441
|
+
...(m.report_md ? { report_md: m.report_md } : {}), reviewer: m.reviewer });
|
|
442
|
+
}
|
|
443
|
+
assertRecoveryJournalAcknowledgements(operation, recoveryAck) {
|
|
444
|
+
const acknowledgement = operation.journal.checkpoints().find(checkpoint => checkpoint.phase === 'recovery_acknowledged');
|
|
445
|
+
if (acknowledgement && !isDeepStrictEqual(acknowledgement.data, JSON.parse(recoveryAck)))
|
|
446
|
+
fail('journal_checkpoint_conflict');
|
|
447
|
+
const complete = operation.journal.checkpoints().find(checkpoint => checkpoint.phase === 'complete');
|
|
448
|
+
if (complete && !isDeepStrictEqual(complete.data, { recovery_ack_sha256: sha256(recoveryAck) }))
|
|
449
|
+
fail('journal_checkpoint_conflict');
|
|
450
|
+
}
|
|
451
|
+
async assertRecoveryAcknowledgements(entry, recoveryAck) {
|
|
452
|
+
await this.assertOrdinaryAcknowledgement(entry);
|
|
453
|
+
const bytes = await this.optionalPrivateRead(join(entry.directory, 'recovery-acknowledged.json'), 4096);
|
|
454
|
+
if (bytes !== undefined && bytes !== recoveryAck)
|
|
455
|
+
fail('immutable_conflict');
|
|
456
|
+
}
|
|
220
457
|
async acknowledged(entry) {
|
|
221
458
|
try {
|
|
222
459
|
if (await privateRead(join(entry.directory, 'acknowledged.json'), 4096) !== this.ack(entry))
|
|
@@ -229,12 +466,12 @@ export class ReviewerDeliveryQueue {
|
|
|
229
466
|
throw error;
|
|
230
467
|
}
|
|
231
468
|
}
|
|
232
|
-
async transfer(entry, sink, request,
|
|
469
|
+
async transfer(entry, sink, request, mode) {
|
|
233
470
|
const m = entry.manifest;
|
|
234
471
|
if (sink.baseUrl !== m.host || sink.credentialSource !== m.credentialKind)
|
|
235
472
|
fail('credential_mismatch');
|
|
236
473
|
accepted(await sink.checkReviewerRecovery(request()));
|
|
237
|
-
let read = initial ? undefined : await sink.getReviewerArtifact(m.runId, m.reviewer, request());
|
|
474
|
+
let read = mode === 'initial' ? undefined : await sink.getReviewerArtifact(m.runId, m.reviewer, request());
|
|
238
475
|
if (read && read.kind !== 'ok' && read.kind !== 'pending')
|
|
239
476
|
accepted(read);
|
|
240
477
|
// A matching private read is the server's completed admission proof: it
|
|
@@ -273,6 +510,357 @@ export class ReviewerDeliveryQueue {
|
|
|
273
510
|
fail('private_mismatch');
|
|
274
511
|
await publish(join(entry.directory, 'acknowledged.json'), this.ack(entry));
|
|
275
512
|
}
|
|
513
|
+
async recover(entry, sink, request, operation) {
|
|
514
|
+
const m = entry.manifest;
|
|
515
|
+
this.validateRecoveryOperation(entry, operation);
|
|
516
|
+
if (sink.baseUrl !== m.host || sink.credentialSource !== m.credentialKind)
|
|
517
|
+
fail('recovery_operation_mismatch');
|
|
518
|
+
const checkpoints = operation.journal.checkpoints();
|
|
519
|
+
const phases = new Set();
|
|
520
|
+
const putAttempts = new Map();
|
|
521
|
+
const putHistories = new Map();
|
|
522
|
+
const phasePositions = new Map();
|
|
523
|
+
for (const [index, checkpoint] of checkpoints.entries()) {
|
|
524
|
+
if (checkpoint.phase === 'interrupted_checkpoints_retained')
|
|
525
|
+
continue;
|
|
526
|
+
const putAttempt = putOutcomeAttempt(checkpoint.phase);
|
|
527
|
+
if (!operationPhases.has(checkpoint.phase) && putAttempt === undefined)
|
|
528
|
+
fail('journal_checkpoint_conflict');
|
|
529
|
+
if (phases.has(checkpoint.phase))
|
|
530
|
+
fail('journal_checkpoint_conflict');
|
|
531
|
+
phases.add(checkpoint.phase);
|
|
532
|
+
phasePositions.set(checkpoint.phase, index);
|
|
533
|
+
if (isOutcomePhase(checkpoint.phase)) {
|
|
534
|
+
const parsed = outcomeSchema.safeParse(checkpoint.data);
|
|
535
|
+
if (!parsed.success || !isDeepStrictEqual(parsed.data, checkpoint.data))
|
|
536
|
+
fail('journal_checkpoint_conflict');
|
|
537
|
+
}
|
|
538
|
+
if (putAttempt !== undefined) {
|
|
539
|
+
const expected = (putAttempts.get(putAttempt.base) ?? 0) + 1;
|
|
540
|
+
if (putAttempt.attempt !== expected)
|
|
541
|
+
fail('journal_checkpoint_conflict');
|
|
542
|
+
putAttempts.set(putAttempt.base, expected);
|
|
543
|
+
const parsed = outcomeSchema.safeParse(checkpoint.data);
|
|
544
|
+
if (!parsed.success)
|
|
545
|
+
fail('journal_checkpoint_conflict');
|
|
546
|
+
const history = putHistories.get(putAttempt.base) ?? [];
|
|
547
|
+
history.push({ index, data: parsed.data });
|
|
548
|
+
putHistories.set(putAttempt.base, history);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
for (const base of putOutcomePhases) {
|
|
552
|
+
const prefix = base.slice(0, -'_put_outcome'.length);
|
|
553
|
+
const intentPosition = phasePositions.get(`${prefix}_put_intent`);
|
|
554
|
+
const verifiedPosition = phasePositions.get(`${prefix}_verified`);
|
|
555
|
+
const history = putHistories.get(base) ?? [];
|
|
556
|
+
if (history.length > 0 && intentPosition === undefined)
|
|
557
|
+
fail('journal_checkpoint_conflict');
|
|
558
|
+
if (intentPosition !== undefined && verifiedPosition !== undefined && intentPosition > verifiedPosition) {
|
|
559
|
+
fail('journal_checkpoint_conflict');
|
|
560
|
+
}
|
|
561
|
+
let terminal = false;
|
|
562
|
+
for (const item of history) {
|
|
563
|
+
if (item.index < intentPosition || (verifiedPosition !== undefined && item.index > verifiedPosition) || terminal) {
|
|
564
|
+
fail('journal_checkpoint_conflict');
|
|
565
|
+
}
|
|
566
|
+
if (item.data.kind !== 'unavailable')
|
|
567
|
+
terminal = true;
|
|
568
|
+
}
|
|
569
|
+
if (history.length > MAX_PUT_OUTCOMES_PER_ARTIFACT)
|
|
570
|
+
fail('put_retry_limit');
|
|
571
|
+
}
|
|
572
|
+
const appendOnce = async (phase, data) => {
|
|
573
|
+
const normalized = JSON.parse(JSON.stringify(data));
|
|
574
|
+
const matches = operation.journal.checkpoints().filter(checkpoint => checkpoint.phase === phase);
|
|
575
|
+
if (matches.length === 0)
|
|
576
|
+
await operation.journal.append(phase, normalized);
|
|
577
|
+
else if (matches.length !== 1 || !isDeepStrictEqual(matches[0].data, normalized))
|
|
578
|
+
fail('journal_checkpoint_conflict');
|
|
579
|
+
};
|
|
580
|
+
const appendPutOutcome = async (phase, data) => {
|
|
581
|
+
if (!putOutcomePhases.includes(phase))
|
|
582
|
+
fail('journal_checkpoint_conflict');
|
|
583
|
+
const attempts = operation.journal.checkpoints().filter(checkpoint => putOutcomeAttempt(checkpoint.phase)?.base === phase).length;
|
|
584
|
+
if (attempts >= MAX_PUT_OUTCOMES_PER_ARTIFACT)
|
|
585
|
+
fail('put_retry_limit');
|
|
586
|
+
const attemptPhase = attempts === 0 ? phase : `${phase}_${attempts + 1}`;
|
|
587
|
+
await operation.journal.append(attemptPhase, JSON.parse(JSON.stringify(data)));
|
|
588
|
+
};
|
|
589
|
+
const assertPutMayProceed = (phase) => {
|
|
590
|
+
const verifiedPhase = `${phase.slice(0, -'_put_outcome'.length)}_verified`;
|
|
591
|
+
if (operation.journal.checkpoints().some(checkpoint => checkpoint.phase === verifiedPhase))
|
|
592
|
+
fail('unavailable');
|
|
593
|
+
const attempts = operation.journal.checkpoints()
|
|
594
|
+
.filter(checkpoint => putOutcomeAttempt(checkpoint.phase)?.base === phase);
|
|
595
|
+
const previous = attempts.at(-1);
|
|
596
|
+
if (!previous)
|
|
597
|
+
return;
|
|
598
|
+
const parsed = outcomeSchema.safeParse(previous.data);
|
|
599
|
+
if (!parsed.success || !isDeepStrictEqual(parsed.data, previous.data))
|
|
600
|
+
fail('journal_checkpoint_conflict');
|
|
601
|
+
if (parsed.data.kind === 'unavailable') {
|
|
602
|
+
if (attempts.length >= MAX_PUT_OUTCOMES_PER_ARTIFACT)
|
|
603
|
+
fail('put_retry_limit');
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
fail(parsed.data.kind === 'ok' ? 'unavailable' : 'refused');
|
|
607
|
+
};
|
|
608
|
+
const assertPutCapacityBeforeIntent = (prefix) => {
|
|
609
|
+
const intentExists = operation.journal.checkpoints().some(checkpoint => checkpoint.phase === `${prefix}_put_intent`);
|
|
610
|
+
operation.journal.assertAppendCapacity(intentExists ? PUT_COMPLETION_AFTER_INTENT_CHECKPOINTS : PUT_COMPLETION_RESERVATION_CHECKPOINTS, intentExists ? PUT_COMPLETION_AFTER_INTENT_BYTES : PUT_COMPLETION_RESERVATION_BYTES);
|
|
611
|
+
};
|
|
612
|
+
const assertExisting = (phase, data) => {
|
|
613
|
+
const checkpoint = operation.journal.checkpoints().find(candidate => candidate.phase === phase);
|
|
614
|
+
if (checkpoint && !isDeepStrictEqual(checkpoint.data, JSON.parse(JSON.stringify(data))))
|
|
615
|
+
fail('journal_checkpoint_conflict');
|
|
616
|
+
};
|
|
617
|
+
const outcome = (value) => {
|
|
618
|
+
const rawError = value.error;
|
|
619
|
+
return { kind: value.kind,
|
|
620
|
+
...('httpStatus' in value ? { http_status: value.httpStatus } : {}),
|
|
621
|
+
...(rawError !== undefined ? { error: typeof rawError === 'string' ? rawError : 'malformed_response' } : {}) };
|
|
622
|
+
};
|
|
623
|
+
const assertDestination = async () => {
|
|
624
|
+
const capability = accepted(await sink.checkReviewerRecoveryActivation(request()));
|
|
625
|
+
if (capability.protocol !== operation.destination.activationProtocol ||
|
|
626
|
+
!isDeepStrictEqual(capability.principal, operation.destination.principal))
|
|
627
|
+
fail('principal_mismatch');
|
|
628
|
+
};
|
|
629
|
+
const privateReadback = async () => sink.getReviewerArtifact(m.runId, m.reviewer, request());
|
|
630
|
+
const activationIntent = this.activationIntent(entry, operation);
|
|
631
|
+
const initialRecoveryAck = this.recoveryAcknowledgement(entry, operation, activationIntent);
|
|
632
|
+
const activationIntentPath = join(entry.directory, 'activation-intent.json');
|
|
633
|
+
let activationIntentPresent = false;
|
|
634
|
+
try {
|
|
635
|
+
if (await privateRead(activationIntentPath, 4096) !== activationIntent)
|
|
636
|
+
fail('activation_operation_conflict');
|
|
637
|
+
activationIntentPresent = true;
|
|
638
|
+
}
|
|
639
|
+
catch (error) {
|
|
640
|
+
if (error.code !== 'ENOENT')
|
|
641
|
+
throw error;
|
|
642
|
+
}
|
|
643
|
+
const ensureActivationIntent = async () => {
|
|
644
|
+
if (activationIntentPresent)
|
|
645
|
+
return;
|
|
646
|
+
await publish(activationIntentPath, activationIntent);
|
|
647
|
+
activationIntentPresent = true;
|
|
648
|
+
};
|
|
649
|
+
const absent = (value) => value.kind === 'rejected' &&
|
|
650
|
+
value.httpStatus === 404 && value.error === 'reviewer_artifact_http_404';
|
|
651
|
+
const inspectOrdinary = async (kind, bytes) => {
|
|
652
|
+
const read = await sink.getArtifact(m.runId, kind, Buffer.byteLength(bytes), request());
|
|
653
|
+
if (read.kind === 'ok')
|
|
654
|
+
return read.value.bytes.equals(Buffer.from(bytes)) ? 'exact' : 'mismatch';
|
|
655
|
+
if (read.kind === 'rejected' && read.httpStatus === 404)
|
|
656
|
+
return 'missing';
|
|
657
|
+
accepted(read);
|
|
658
|
+
return 'mismatch';
|
|
659
|
+
};
|
|
660
|
+
const assertEnvelopeReceipt = (receipt) => {
|
|
661
|
+
if (receipt.kind === 'recorded')
|
|
662
|
+
return;
|
|
663
|
+
if (receipt.kind === 'unavailable')
|
|
664
|
+
fail('unavailable');
|
|
665
|
+
fail(receipt.kind === 'absent' ? 'envelope_receipt_absent' : 'envelope_receipt_mismatch');
|
|
666
|
+
};
|
|
667
|
+
assertExisting('prepared', { outbox_manifest_sha256: sha256(entry.manifestBytes), destination: operation.destination });
|
|
668
|
+
assertExisting('activation_post_intent', { envelope: m.envelope });
|
|
669
|
+
assertExisting('activation_post_uncertain', { reason: 'intent_without_remote_readback' });
|
|
670
|
+
assertExisting('report_json_put_intent', m.report_json);
|
|
671
|
+
if (m.report_md)
|
|
672
|
+
assertExisting('report_md_put_intent', m.report_md);
|
|
673
|
+
assertExisting('reviewer_put_intent', m.reviewer);
|
|
674
|
+
assertExisting('envelope_verified', { sha256: m.envelope.sha256, artifacts_declared: entry.envelope.artifacts_declared });
|
|
675
|
+
assertExisting('report_json_verified', m.report_json);
|
|
676
|
+
if (m.report_md)
|
|
677
|
+
assertExisting('report_md_verified', m.report_md);
|
|
678
|
+
assertExisting('reviewer_verified', m.reviewer);
|
|
679
|
+
this.assertRecoveryJournalAcknowledgements(operation, initialRecoveryAck);
|
|
680
|
+
await this.assertRecoveryAcknowledgements(entry, initialRecoveryAck);
|
|
681
|
+
await assertDestination();
|
|
682
|
+
let read = await privateReadback();
|
|
683
|
+
if (read.kind !== 'ok' && read.kind !== 'pending' && !absent(read))
|
|
684
|
+
accepted(read);
|
|
685
|
+
if (read.kind === 'ok' && !read.value.bytes.equals(Buffer.from(entry.privateBytes)))
|
|
686
|
+
fail('private_mismatch');
|
|
687
|
+
const plannedEnvelopeReceipt = await sink.getRunReceipt(entry.envelope, entry.envelopeBytes, request());
|
|
688
|
+
if (!absent(read))
|
|
689
|
+
assertEnvelopeReceipt(plannedEnvelopeReceipt);
|
|
690
|
+
const hasCheckpoint = (phase) => operation.journal.checkpoints()
|
|
691
|
+
.some(checkpoint => checkpoint.phase === phase);
|
|
692
|
+
const appendPlan = [];
|
|
693
|
+
const putBoundaries = [];
|
|
694
|
+
const planExact = (phase, data) => {
|
|
695
|
+
if (!hasCheckpoint(phase))
|
|
696
|
+
appendPlan.push({ phase, data });
|
|
697
|
+
};
|
|
698
|
+
const planOutcome = () => appendPlan.push({ maximumBytes: MAX_OUTCOME_CHECKPOINT_BYTES });
|
|
699
|
+
const planVerified = (prefix, data) => {
|
|
700
|
+
planExact(`${prefix}_verified`, data);
|
|
701
|
+
};
|
|
702
|
+
const planPut = (prefix, data) => {
|
|
703
|
+
const intentExists = hasCheckpoint(`${prefix}_put_intent`);
|
|
704
|
+
const index = appendPlan.length;
|
|
705
|
+
planExact(`${prefix}_put_intent`, data);
|
|
706
|
+
planOutcome();
|
|
707
|
+
planVerified(prefix, data);
|
|
708
|
+
putBoundaries.push({ index, reservation: intentExists ? PUT_COMPLETION_AFTER_INTENT_CHECKPOINTS :
|
|
709
|
+
PUT_COMPLETION_RESERVATION_CHECKPOINTS });
|
|
710
|
+
};
|
|
711
|
+
planExact('prepared', { outbox_manifest_sha256: sha256(entry.manifestBytes), destination: operation.destination });
|
|
712
|
+
const activationUncertain = absent(read) && hasCheckpoint('activation_post_intent');
|
|
713
|
+
if (absent(read)) {
|
|
714
|
+
if (activationUncertain)
|
|
715
|
+
planExact('activation_post_uncertain', { reason: 'intent_without_remote_readback' });
|
|
716
|
+
else {
|
|
717
|
+
planExact('activation_post_intent', { envelope: m.envelope });
|
|
718
|
+
planOutcome();
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
if (!activationUncertain)
|
|
722
|
+
planExact('envelope_verified', {
|
|
723
|
+
sha256: m.envelope.sha256, artifacts_declared: entry.envelope.artifacts_declared,
|
|
724
|
+
});
|
|
725
|
+
for (const kind of ['report_json', 'report_md']) {
|
|
726
|
+
const bytes = entry.artifacts[kind];
|
|
727
|
+
if (bytes === undefined)
|
|
728
|
+
continue;
|
|
729
|
+
const state = await inspectOrdinary(kind, bytes);
|
|
730
|
+
if (state === 'mismatch')
|
|
731
|
+
fail('ordinary_mismatch');
|
|
732
|
+
if (state === 'missing') {
|
|
733
|
+
assertPutMayProceed(`${kind}_put_outcome`);
|
|
734
|
+
if (!activationUncertain)
|
|
735
|
+
planPut(kind, m[kind]);
|
|
736
|
+
}
|
|
737
|
+
else if (!activationUncertain)
|
|
738
|
+
planVerified(kind, m[kind]);
|
|
739
|
+
}
|
|
740
|
+
if (read.kind === 'pending' || absent(read)) {
|
|
741
|
+
assertPutMayProceed('reviewer_put_outcome');
|
|
742
|
+
if (!activationUncertain)
|
|
743
|
+
planPut('reviewer', m.reviewer);
|
|
744
|
+
}
|
|
745
|
+
else if (!activationUncertain)
|
|
746
|
+
planVerified('reviewer', m.reviewer);
|
|
747
|
+
if (!activationUncertain) {
|
|
748
|
+
planExact('recovery_acknowledged', JSON.parse(initialRecoveryAck));
|
|
749
|
+
planExact('complete', { recovery_ack_sha256: '0'.repeat(64) });
|
|
750
|
+
}
|
|
751
|
+
for (const boundary of putBoundaries) {
|
|
752
|
+
operation.journal.assertAppendPlanCapacity([
|
|
753
|
+
...appendPlan.slice(0, boundary.index),
|
|
754
|
+
...Array.from({ length: boundary.reservation }, () => ({ maximumBytes: MAX_RECOVERY_CHECKPOINT_BYTES })),
|
|
755
|
+
]);
|
|
756
|
+
}
|
|
757
|
+
operation.journal.assertAppendPlanCapacity(appendPlan);
|
|
758
|
+
await appendOnce('prepared', { outbox_manifest_sha256: sha256(entry.manifestBytes), destination: operation.destination });
|
|
759
|
+
if (absent(read)) {
|
|
760
|
+
const postIntent = operation.journal.checkpoints().filter(checkpoint => checkpoint.phase === 'activation_post_intent');
|
|
761
|
+
if (postIntent.length > 0) {
|
|
762
|
+
await appendOnce('activation_post_intent', { envelope: m.envelope });
|
|
763
|
+
await appendOnce('activation_post_uncertain', { reason: 'intent_without_remote_readback' });
|
|
764
|
+
fail('activation_post_uncertain');
|
|
765
|
+
}
|
|
766
|
+
// This entry-level immutable intent prevents a second manifest or journal
|
|
767
|
+
// from granting another POST after any process loss around transport.
|
|
768
|
+
await ensureActivationIntent();
|
|
769
|
+
await appendOnce('activation_post_intent', { envelope: m.envelope });
|
|
770
|
+
await assertDestination();
|
|
771
|
+
const posted = await sink.postRun(entry.envelope, request(), entry.envelopeBytes);
|
|
772
|
+
await appendOnce('activation_post_outcome', outcome(posted));
|
|
773
|
+
read = await privateReadback();
|
|
774
|
+
if (absent(read)) {
|
|
775
|
+
await appendOnce('activation_post_uncertain', { reason: 'intent_without_remote_readback' });
|
|
776
|
+
fail('activation_post_uncertain');
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
if (read.kind !== 'ok' && read.kind !== 'pending')
|
|
780
|
+
accepted(read);
|
|
781
|
+
if (read.kind === 'ok' && !read.value.bytes.equals(Buffer.from(entry.privateBytes)))
|
|
782
|
+
fail('private_mismatch');
|
|
783
|
+
const envelopeReceipt = await sink.getRunReceipt(entry.envelope, entry.envelopeBytes, request());
|
|
784
|
+
assertEnvelopeReceipt(envelopeReceipt);
|
|
785
|
+
await appendOnce('envelope_verified', { sha256: m.envelope.sha256, artifacts_declared: entry.envelope.artifacts_declared });
|
|
786
|
+
for (const kind of ['report_json', 'report_md']) {
|
|
787
|
+
const bytes = entry.artifacts[kind];
|
|
788
|
+
if (bytes === undefined)
|
|
789
|
+
continue;
|
|
790
|
+
const state = await inspectOrdinary(kind, bytes);
|
|
791
|
+
if (state === 'exact') {
|
|
792
|
+
await appendOnce(`${kind}_verified`, m[kind]);
|
|
793
|
+
continue;
|
|
794
|
+
}
|
|
795
|
+
if (state === 'mismatch')
|
|
796
|
+
fail('ordinary_mismatch');
|
|
797
|
+
assertPutMayProceed(`${kind}_put_outcome`);
|
|
798
|
+
assertPutCapacityBeforeIntent(kind);
|
|
799
|
+
await assertDestination();
|
|
800
|
+
await ensureActivationIntent();
|
|
801
|
+
await appendOnce(`${kind}_put_intent`, m[kind]);
|
|
802
|
+
operation.journal.assertAppendCapacity(PUT_COMPLETION_AFTER_INTENT_CHECKPOINTS, PUT_COMPLETION_AFTER_INTENT_BYTES);
|
|
803
|
+
const uploaded = await sink.putArtifact(m.runId, kind, bytes, request());
|
|
804
|
+
await appendPutOutcome(`${kind}_put_outcome`, outcome(uploaded));
|
|
805
|
+
if (await inspectOrdinary(kind, bytes) !== 'exact') {
|
|
806
|
+
if (uploaded.kind !== 'ok')
|
|
807
|
+
accepted(uploaded);
|
|
808
|
+
fail('ordinary_mismatch');
|
|
809
|
+
}
|
|
810
|
+
await appendOnce(`${kind}_verified`, m[kind]);
|
|
811
|
+
}
|
|
812
|
+
read = await privateReadback();
|
|
813
|
+
if (read.kind !== 'ok' && read.kind !== 'pending')
|
|
814
|
+
accepted(read);
|
|
815
|
+
if (read.kind === 'ok' && !read.value.bytes.equals(Buffer.from(entry.privateBytes)))
|
|
816
|
+
fail('private_mismatch');
|
|
817
|
+
if (read.kind === 'pending') {
|
|
818
|
+
assertPutMayProceed('reviewer_put_outcome');
|
|
819
|
+
assertPutCapacityBeforeIntent('reviewer');
|
|
820
|
+
await assertDestination();
|
|
821
|
+
await ensureActivationIntent();
|
|
822
|
+
await appendOnce('reviewer_put_intent', m.reviewer);
|
|
823
|
+
operation.journal.assertAppendCapacity(PUT_COMPLETION_AFTER_INTENT_CHECKPOINTS, PUT_COMPLETION_AFTER_INTENT_BYTES);
|
|
824
|
+
const uploaded = await sink.putReviewerArtifact(m.runId, entry.privateBytes, m.reviewer, request());
|
|
825
|
+
await appendPutOutcome('reviewer_put_outcome', outcome(uploaded));
|
|
826
|
+
read = await privateReadback();
|
|
827
|
+
if (read.kind === 'pending') {
|
|
828
|
+
if (uploaded.kind !== 'ok')
|
|
829
|
+
accepted(uploaded);
|
|
830
|
+
fail('unavailable');
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
if (read.kind !== 'ok' || !read.value.bytes.equals(Buffer.from(entry.privateBytes))) {
|
|
834
|
+
if (read.kind !== 'ok')
|
|
835
|
+
accepted(read);
|
|
836
|
+
fail('private_mismatch');
|
|
837
|
+
}
|
|
838
|
+
await appendOnce('reviewer_verified', m.reviewer);
|
|
839
|
+
// Completion depends on fresh exact readback of every retained server byte,
|
|
840
|
+
// even when the private artifact was already present after a lost local ACK.
|
|
841
|
+
for (const kind of ['report_json', 'report_md']) {
|
|
842
|
+
const bytes = entry.artifacts[kind];
|
|
843
|
+
if (bytes !== undefined && await inspectOrdinary(kind, bytes) !== 'exact')
|
|
844
|
+
fail('ordinary_mismatch');
|
|
845
|
+
}
|
|
846
|
+
const finalPrivate = await privateReadback();
|
|
847
|
+
if (finalPrivate.kind !== 'ok' || !finalPrivate.value.bytes.equals(Buffer.from(entry.privateBytes))) {
|
|
848
|
+
if (finalPrivate.kind === 'pending')
|
|
849
|
+
fail('unavailable');
|
|
850
|
+
if (finalPrivate.kind !== 'ok')
|
|
851
|
+
accepted(finalPrivate);
|
|
852
|
+
fail('private_mismatch');
|
|
853
|
+
}
|
|
854
|
+
await assertDestination();
|
|
855
|
+
await ensureActivationIntent();
|
|
856
|
+
const recoveryAck = this.recoveryAcknowledgement(entry, operation, activationIntent);
|
|
857
|
+
this.assertRecoveryJournalAcknowledgements(operation, recoveryAck);
|
|
858
|
+
await this.assertRecoveryAcknowledgements(entry, recoveryAck);
|
|
859
|
+
await publish(join(entry.directory, 'recovery-acknowledged.json'), recoveryAck);
|
|
860
|
+
await appendOnce('recovery_acknowledged', JSON.parse(recoveryAck));
|
|
861
|
+
await publish(join(entry.directory, 'acknowledged.json'), this.ack(entry));
|
|
862
|
+
await appendOnce('complete', { recovery_ack_sha256: sha256(recoveryAck) });
|
|
863
|
+
}
|
|
276
864
|
}
|
|
277
865
|
// Ephemeral creation authority is never persisted or reconstructed from a manifest.
|
|
278
866
|
const entryWasNew = new WeakSet();
|