@planu/cli 4.14.1 → 5.1.1

package/dist/engine/execution/validate-job-executor.js

@@ -1,19 +1,80 @@
+/* eslint-disable max-lines -- durable validation, resumable receipt publication, and fencing stay co-located for auditability */
 import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
 import { DurableJobRuntime } from './job-runtime.js';
 import { RuntimeDatabase } from '../../storage/runtime-db.js';
 import { createExecutionScope } from './context.js';
 import { runValidateWorker } from './validate-worker-runtime.js';
 import { redactValidationProgress, computeDurableValidationBindings, toValidationReceiptBindings, writeValidationArtifactProvenance, } from '../validation/durable-validation.js';
+import { parseValidationReceiptPendingCheckpoint, isValidationReceiptPendingIntent, } from '../../types/durable-validation.js';
 import { resolveStorageLayout } from '../../storage/storage-layout.js';
 import { reportClassifiedDegradation } from '../../errors/classified-degradation.js';
-import { createValidationReceiptAuthority, ensureValidationIssuerKey, issueValidationReceipt, LOCAL_VALIDATION_RECEIPT_ISSUER, } from '../validation/validation-receipt.js';
+import { createValidationReceiptAuthority, ensureValidationIssuerKey, issueValidationReceipt, LOCAL_VALIDATION_RECEIPT_ISSUER, preflightValidationReceiptSigner, } from '../validation/validation-receipt.js';
 import { appendArtifact, readArtifact } from '../handoff-artifacts/io.js';
-import { assertValidationFreshnessBarrier, rebaselineValidationFreshnessLeaseAfterRecompute, } from '../validation/validation-freshness.js';
+import { assertValidationFreshnessBarrier, beginValidationFreshnessBarrier, bindValidationFreshnessLease, closeValidationFreshnessLease, rebaselineValidationFreshnessLeaseAfterRecompute, } from '../validation/validation-freshness.js';
 import { recoverValidateJobs } from './validate-job-recovery.js';
 import { getRuntimePolicy } from '../runtime-policy.js';
 import { isValidateJob, parseValidatePayload, rehydrateValidationPayload, } from './validate-job-payload.js';
+import { revalidateValidationWorktree } from '../validation/validation-worktree.js';
+import { projectDataDir } from '../../storage/base-store.js';
+import { redactSensitiveValue, redactText } from '../../security/redactor.js';
+import { MacOSKeychainOperationError } from '../../security/adapters/macos-keychain.js';
+import { createDurableJobHeartbeat } from './durable-job-heartbeat-runtime.js';
 const WORKSPACE_ID = 'global';
-const activeScopes = new Set();
+const activeScopes = new Map();
+const RECEIPT_PENDING_ERROR = '[Planu] VALIDATION_RECEIPT_PENDING action=restart_job';
+class ReceiptPublicationTimeoutError extends Error {
+    constructor() {
+        super('[Planu] Receipt publication deadline elapsed');
+        this.name = 'ReceiptPublicationTimeoutError';
+    }
+}
+class ReceiptPublicationInterruptedError extends Error {
+    constructor(options) {
+        super('[Planu] Receipt publication was interrupted after persistence', options);
+        this.name = 'ReceiptPublicationInterruptedError';
+    }
+}
+class ValidationAuthorityLostError extends Error {
+    reason;
+    constructor(reason, options) {
+        super(`[Planu] Durable validate job authority was lost (${reason})`, options);
+        this.reason = reason;
+        this.name = 'ValidationAuthorityLostError';
+    }
+}
+function recoverableReceiptError(error) {
+    return (error instanceof ReceiptPublicationTimeoutError ||
+        error instanceof ReceiptPublicationInterruptedError ||
+        (error instanceof MacOSKeychainOperationError && error.code === 'KEYCHAIN_TIMEOUT'));
+}
+function decodeUtf8Prefix(encoded, prefixLength) {
+    try {
+        return {
+            valid: true,
+            value: new TextDecoder('utf-8', { fatal: true }).decode(encoded.subarray(0, prefixLength)),
+        };
+    }
+    catch {
+        return { valid: false };
+    }
+}
+function truncateUtf8ByBytes(value, maximumBytes) {
+    const encoded = Buffer.from(value, 'utf8');
+    if (encoded.length <= maximumBytes) {
+        return value;
+    }
+    for (let length = maximumBytes; length >= 0; length -= 1) {
+        const decoded = decodeUtf8Prefix(encoded, length);
+        if (decoded.valid) {
+            return decoded.value;
+        }
+    }
+    return '';
+}
+function canonicalJson(value) {
+    return JSON.stringify(value);
+}
 function hasReceipt(result) {
     const structured = result.structuredContent;
     return typeof structured?.receiptId === 'string' && typeof structured.receiptKey === 'string';
@@ -23,6 +84,28 @@ function validationPassed(result) {
     const report = structured?.validationReport;
     return structured?.ready === true && report?.passed === true;
 }
+async function closeFreshnessLeaseSafely(lease) {
+    if (!lease) {
+        return;
+    }
+    try {
+        await closeValidationFreshnessLease(lease);
+    }
+    catch (error) {
+        reportClassifiedDegradation('VALIDATION_FRESHNESS_CHECK_FAILED', error);
+    }
+}
+async function stopHeartbeatSafely(heartbeat) {
+    if (!heartbeat) {
+        return;
+    }
+    try {
+        await heartbeat.stop();
+    }
+    catch (error) {
+        reportClassifiedDegradation('VALIDATE_JOB_HEARTBEAT_FAILURE', error);
+    }
+}
 function toolResultError(result) {
     return result.content
         .map((item) => item.text)
@@ -53,11 +136,32 @@ export class ValidateJobExecutor {
         return Promise.all(candidates.map((job) => this.execute(job)));
     }
     async runUntilIdle(projectId, workspaceId = WORKSPACE_ID) {
+        const capacity = getRuntimePolicy().validation.workerConcurrency;
+        const active = new Set();
         for (;;) {
-            const completed = await this.runOnce(projectId, workspaceId);
-            if (completed.length === 0) {
+            const available = capacity - active.size;
+            const candidates = this.runtime
+                .list(projectId, workspaceId)
+                .filter((job) => (job.state === 'accepted' || job.state === 'checkpointed') &&
+                isValidateJob(job.payload))
+                .slice(0, available);
+            for (const candidate of candidates) {
+                const execution = this.execute(candidate)
+                    .then(() => undefined)
+                    .catch((error) => {
+                    reportClassifiedDegradation('VALIDATE_JOB_EXECUTION_FAILURE', error);
+                })
+                    .finally(() => {
+                    active.delete(execution);
+                });
+                active.add(execution);
+            }
+            if (active.size === 0) {
                 return;
             }
+            if (active.size >= capacity || candidates.length === 0) {
+                await Promise.race(active);
+            }
         }
     }
     async execute(job) {
@@ -69,12 +173,24 @@ export class ValidateJobExecutor {
         const claimed = this.runtime.claim({
             ...identity,
             workerId: this.workerId,
-            leaseMs: getRuntimePolicy().validation.workerLeaseMs,
+            leaseMs: this.workerTimingPolicy().workerLeaseMs,
         });
         if (!claimed) {
             return this.runtime.get(identity);
         }
-        const parsedPayload = parseValidatePayload(claimed.payload);
+        let parsedPayload;
+        try {
+            parsedPayload = parseValidatePayload(claimed.payload);
+        }
+        catch (error) {
+            reportClassifiedDegradation('VALIDATE_JOB_PAYLOAD_PARSE_FAILURE', error);
+            return this.runtime.fail({
+                ...identity,
+                workerId: this.workerId,
+                fencingToken: claimed.fencingToken,
+                error: error instanceof Error ? error.message : '[Planu] Invalid validate job payload',
+            });
+        }
         if (!parsedPayload) {
             return this.runtime.fail({
                 ...identity,
@@ -83,54 +199,73 @@ export class ValidateJobExecutor {
                 error: '[Planu] Invalid validate job payload',
             });
         }
-        const deadlineAt = parsedPayload.deadlineAt ?? Date.now() + getRuntimePolicy().validation.defaultDeadlineMs;
-        if (deadlineAt <= Date.now()) {
+        const pendingCheckpoint = parseValidationReceiptPendingCheckpoint(claimed.checkpoint, this.receiptCheckpointLimits());
+        if (pendingCheckpoint) {
+            return this.resumeReceiptPublication(identity, parsedPayload, pendingCheckpoint, claimed.fencingToken);
+        }
+        if (isValidationReceiptPendingIntent(claimed.checkpoint)) {
             return this.runtime.fail({
                 ...identity,
                 workerId: this.workerId,
                 fencingToken: claimed.fencingToken,
-                error: '[Planu] Durable validate job deadline elapsed',
+                error: '[Planu] Invalid validation receipt checkpoint',
             });
         }
-        let payload;
-        try {
-            payload = await rehydrateValidationPayload(parsedPayload);
-        }
-        catch (error) {
-            /* reliability-optional: VALIDATE_JOB_SUBMISSION_FAILURE — runtime records typed failure */
-            reportClassifiedDegradation('VALIDATE_JOB_SUBMISSION_FAILURE', error);
+        const deadlineAt = parsedPayload.deadlineAt ?? Date.now() + getRuntimePolicy().validation.defaultDeadlineMs;
+        if (deadlineAt <= Date.now()) {
             return this.runtime.fail({
                 ...identity,
                 workerId: this.workerId,
                 fencingToken: claimed.fencingToken,
-                error: (error instanceof Error ? error.message : String(error)).slice(0, 4_096),
+                error: '[Planu] Durable validate job deadline elapsed',
             });
         }
-        return this.runClaimedValidation(identity, payload, deadlineAt, claimed.fencingToken);
+        return this.runClaimedValidation(identity, parsedPayload, deadlineAt, claimed.fencingToken);
     }
-    async runClaimedValidation(identity, payload, deadlineAt, fencingToken) {
-        const scope = createExecutionScope({
-            requestId: identity.operationId,
-            deadlineAt,
-        });
-        const heartbeat = this.startHeartbeat(identity, fencingToken, (reason) => {
-            scope.cancel(reason);
-        });
+    // eslint-disable-next-line max-lines-per-function -- worker and receipt failure cleanup share one lease boundary
+    async runClaimedValidation(identity, submittedPayload, deadlineAt, fencingToken) {
+        let scope;
+        let heartbeat;
+        let payload;
         try {
+            const executionScope = createExecutionScope({ requestId: identity.operationId, deadlineAt });
+            scope = executionScope;
+            heartbeat = await this.startHeartbeat(identity, fencingToken, (reason) => {
+                executionScope.cancel(reason);
+            });
+            let hydratedPayload;
+            try {
+                hydratedPayload = await rehydrateValidationPayload(submittedPayload);
+            }
+            catch (error) {
+                reportClassifiedDegradation('VALIDATE_JOB_SUBMISSION_FAILURE', error);
+                throw error;
+            }
+            payload = hydratedPayload;
+            await executionScope.run(() => preflightValidationReceiptSigner({
+                projectId: identity.projectId,
+                issuerId: LOCAL_VALIDATION_RECEIPT_ISSUER,
+            }));
+            const afterPreflight = this.observeForSettlement(identity);
+            if (afterPreflight.state === 'cancelled' || afterPreflight.cancelRequested) {
+                return afterPreflight;
+            }
+            this.requireCurrentFence(afterPreflight, fencingToken);
             const startedAt = Date.now();
-            const result = await scope.run(() => (this.options.runWorker ?? runValidateWorker)({
+            const result = await executionScope.run(() => (this.options.runWorker ?? runValidateWorker)({
                 requestId: identity.operationId,
-                input: payload.input,
+                input: { ...hydratedPayload.input, specPath: hydratedPayload.specPath },
                 deadlineAt,
-                signal: scope.context.signal,
+                signal: executionScope.context.signal,
                 onProgress: (progress) => {
                     this.persistProgress(identity, fencingToken, progress);
                 },
             }));
-            const current = this.runtime.get(identity);
-            if (current.state === 'cancelled') {
+            const current = this.observeForSettlement(identity);
+            if (current.state === 'cancelled' || current.cancelRequested) {
                 return current;
             }
+            this.requireCurrentFence(current, fencingToken);
             if (result.isError === true) {
                 return this.runtime.fail({
                     ...identity,
@@ -143,8 +278,9 @@ export class ValidateJobExecutor {
                 throw new Error('[Planu] Validate workers cannot self-issue authoritative receipts');
             }
             const durableResult = validationPassed(result)
-                ? await this.issueReceipt(identity, payload, result, Date.now() - startedAt, scope.context.signal)
+                ? await this.publishInitialReceipt(identity, payload, result, Date.now() - startedAt, fencingToken, executionScope.context.signal)
                 : result;
+            this.currentFence(identity, fencingToken);
             return this.runtime.complete({
                 ...identity,
                 workerId: this.workerId,
@@ -155,10 +291,34 @@ export class ValidateJobExecutor {
         catch (error) {
             /* reliability-optional: VALIDATE_JOB_EXECUTION_FAILURE — runtime records typed failure */
             reportClassifiedDegradation('VALIDATE_JOB_EXECUTION_FAILURE', error);
-            const current = this.runtime.get(identity);
+            const disposition = this.failureDisposition(identity, fencingToken, error);
+            if (disposition.settleLocally) {
+                if (disposition.current) {
+                    return disposition.current;
+                }
+                throw error;
+            }
+            const current = disposition.current;
             if (current.state === 'cancelled') {
                 return current;
             }
+            if (recoverableReceiptError(error) &&
+                parseValidationReceiptPendingCheckpoint(current.checkpoint, this.receiptCheckpointLimits())) {
+                return this.runtime.fail({
+                    ...identity,
+                    workerId: this.workerId,
+                    fencingToken,
+                    error: RECEIPT_PENDING_ERROR,
+                });
+            }
+            if (parseValidationReceiptPendingCheckpoint(current.checkpoint, this.receiptCheckpointLimits())) {
+                this.recordFencedProgress(identity, fencingToken, {
+                    phase: 'receipt-publication',
+                    command: '[Planu] Receipt publication failed closed',
+                    durationMs: 0,
+                    status: 'failed',
+                });
+            }
             return this.runtime.fail({
                 ...identity,
                 workerId: this.workerId,
@@ -172,49 +332,325 @@ export class ValidateJobExecutor {
             });
         }
         finally {
-            clearInterval(heartbeat);
-            scope.dispose();
+            await stopHeartbeatSafely(heartbeat);
+            scope?.dispose();
+            await closeFreshnessLeaseSafely(payload?.freshnessLease);
         }
     }
-    startHeartbeat(identity, fencingToken, cancel) {
-        const heartbeat = setInterval(() => {
-            const current = this.runtime.get(identity);
-            if (current.state === 'cancelled' || current.cancelRequested) {
-                cancel(new Error('[Planu] Durable validate job cancelled'));
-                return;
+    // eslint-disable-next-line max-lines-per-function -- receipt resume must keep its fresh lease and fail-closed cleanup together
+    async resumeReceiptPublication(identity, payload, checkpoint, fencingToken) {
+        const publicationDeadlineMs = getRuntimePolicy().validation.receiptPublicationDeadlineMs;
+        const deadlineAt = Date.now() + publicationDeadlineMs;
+        const scope = createExecutionScope({ requestId: identity.operationId, deadlineAt });
+        const deadlineTimer = setTimeout(() => {
+            scope.cancel(new ReceiptPublicationTimeoutError());
+        }, publicationDeadlineMs);
+        deadlineTimer.unref();
+        let heartbeat;
+        let ownedLease;
+        try {
+            heartbeat = await this.startHeartbeat(identity, fencingToken, (reason) => {
+                scope.cancel(reason);
+            });
+            const receiptBindings = checkpoint.receiptInput.bindings;
+            const resolved = await revalidateValidationWorktree({
+                projectPath: payload.input.projectPath ?? '',
+                specPath: payload.specPath,
+                specId: payload.input.specId,
+                validationRootId: payload.validationRootId,
+            });
+            const lease = await beginValidationFreshnessBarrier({
+                projectId: identity.projectId,
+                specId: payload.input.specId,
+                projectPath: resolved.projectPath,
+                handoffPath: join(projectDataDir(identity.projectId), 'handoffs', payload.input.specId),
+                invalidateCurrentBindings: false,
+            });
+            ownedLease = lease;
+            const observed = await computeDurableValidationBindings({
+                canonicalProjectId: identity.projectId,
+                specId: payload.input.specId,
+                projectPath: resolved.projectPath,
+                specPath: resolved.specPath,
+            });
+            const observedBindings = toValidationReceiptBindings(observed.bindings, observed.artifactDigests);
+            if (canonicalJson(observedBindings) !== canonicalJson(receiptBindings) ||
+                !(await bindValidationFreshnessLease(lease, observedBindings))) {
+                throw new Error('[Planu] Durable validation bindings became stale; submit a new validation');
             }
-            if (!this.runtime.renew({
+            const hydratedPayload = {
+                ...payload,
+                input: { ...payload.input, projectPath: resolved.projectPath },
+                specPath: resolved.specPath,
+                bindings: observed.bindings,
+                artifactDigests: observed.artifactDigests,
+                freshnessLease: lease,
+            };
+            const result = await scope.run(() => {
+                this.currentFence(identity, fencingToken);
+                return this.issueReceipt(identity, hydratedPayload, checkpoint, lease, fencingToken, scope.context.signal);
+            });
+            this.currentFence(identity, fencingToken);
+            return this.runtime.complete({
                 ...identity,
                 workerId: this.workerId,
                 fencingToken,
-                leaseMs: getRuntimePolicy().validation.workerLeaseMs,
-            })) {
-                cancel(new Error('[Planu] Durable validate job lease was lost'));
+                result,
+            });
+        }
+        catch (error) {
+            reportClassifiedDegradation('VALIDATE_JOB_EXECUTION_FAILURE', error);
+            const disposition = this.failureDisposition(identity, fencingToken, error);
+            if (disposition.settleLocally) {
+                if (disposition.current) {
+                    return disposition.current;
+                }
+                throw error;
             }
-        }, getRuntimePolicy().validation.workerHeartbeatMs);
-        heartbeat.unref();
-        return heartbeat;
-    }
-    persistProgress(identity, fencingToken, progress) {
-        const current = this.runtime.get(identity);
-        if (current.state !== 'running') {
-            return;
+            const current = disposition.current;
+            if (current.state === 'cancelled') {
+                return current;
+            }
+            if (recoverableReceiptError(error)) {
+                return this.runtime.fail({
+                    ...identity,
+                    workerId: this.workerId,
+                    fencingToken,
+                    error: RECEIPT_PENDING_ERROR,
+                });
+            }
+            this.recordFencedProgress(identity, fencingToken, {
+                phase: 'receipt-publication',
+                command: '[Planu] Receipt publication failed closed',
+                durationMs: 0,
+                status: 'failed',
+            });
+            return this.runtime.fail({
+                ...identity,
+                workerId: this.workerId,
+                fencingToken,
+                error: '[Planu] Receipt publication failed closed',
+            });
         }
-        this.runtime.recordProgress({
+        finally {
+            clearTimeout(deadlineTimer);
+            await stopHeartbeatSafely(heartbeat);
+            scope.dispose();
+            await closeFreshnessLeaseSafely(ownedLease);
+        }
+    }
+    startHeartbeat(identity, fencingToken, cancel) {
+        const timing = this.workerTimingPolicy();
+        const policy = getRuntimePolicy().validation;
+        const factory = this.options.heartbeatFactory ?? createDurableJobHeartbeat;
+        return factory({
             ...identity,
             workerId: this.workerId,
             fencingToken,
-            checkpoint: redactValidationProgress(progress),
+            databasePath: this.runtime.runtimeDatabasePath(),
+            leaseMs: timing.workerLeaseMs,
+            heartbeatMs: timing.workerHeartbeatMs,
+            databaseBusyTimeoutMs: policy.heartbeatDatabaseBusyTimeoutMs,
+            startupTimeoutMs: policy.heartbeatStartupTimeoutMs,
+            shutdownTimeoutMs: policy.heartbeatShutdownTimeoutMs,
+            onAuthorityLost: (reason) => {
+                reportClassifiedDegradation('VALIDATE_JOB_HEARTBEAT_FAILURE', reason);
+                cancel(new ValidationAuthorityLostError('infrastructure', { cause: reason }));
+            },
+        }).catch((error) => {
+            const authorityError = new ValidationAuthorityLostError('infrastructure', { cause: error });
+            reportClassifiedDegradation('VALIDATE_JOB_HEARTBEAT_FAILURE', error);
+            cancel(authorityError);
+            throw authorityError;
         });
     }
-    async issueReceipt(identity, payload, result, durationMs, signal) {
+    persistProgress(identity, fencingToken, progress) {
+        const current = this.observeForSettlement(identity);
+        if (current.state !== 'running') {
+            return;
+        }
+        const previous = current.checkpoint;
+        if (previous &&
+            (progress.completedGates < (previous.completedGates ?? 0) ||
+                (typeof previous.lastProgressAt === 'string' &&
+                    progress.lastProgressAt < previous.lastProgressAt) ||
+                (progress.completedGates === previous.completedGates &&
+                    progress.phase === previous.phase &&
+                    progress.status === previous.status))) {
+            return;
+        }
+        this.recordFencedProgress(identity, fencingToken, redactValidationProgress(progress));
+    }
+    async publishInitialReceipt(identity, payload, result, durationMs, fencingToken, signal) {
+        this.currentFence(identity, fencingToken);
         if (!hasReceiptInputs(payload)) {
             throw new Error('[Planu] Passing durable validation is missing receipt bindings');
         }
         const receiptBindings = toValidationReceiptBindings(payload.bindings, payload.artifactDigests);
-        const publicationLease = await this.preparePublicationLease(payload, receiptBindings);
-        if (executionAborted(signal) ||
-            !(await assertValidationFreshnessBarrier(publicationLease, receiptBindings))) {
+        const checkpoint = this.buildReceiptPendingCheckpoint(payload, receiptBindings, result, durationMs);
+        this.recordFencedProgress(identity, fencingToken, checkpoint);
+        const publicationDeadlineMs = getRuntimePolicy().validation.receiptPublicationDeadlineMs;
+        const publicationScope = createExecutionScope({
+            requestId: identity.operationId,
+            deadlineAt: Date.now() + publicationDeadlineMs,
+            upstreamSignal: signal,
+        });
+        const deadlineTimer = setTimeout(() => {
+            publicationScope.cancel(new ReceiptPublicationTimeoutError());
+        }, publicationDeadlineMs);
+        deadlineTimer.unref();
+        try {
+            return await publicationScope.run(async () => {
+                const publicationLease = await this.preparePublicationLease(payload, receiptBindings);
+                this.currentFence(identity, fencingToken);
+                return this.issueReceipt(identity, payload, checkpoint, publicationLease, fencingToken, publicationScope.context.signal);
+            });
+        }
+        finally {
+            clearTimeout(deadlineTimer);
+            publicationScope.dispose();
+        }
+    }
+    workerTimingPolicy() {
+        const production = getRuntimePolicy().validation;
+        const injected = this.options.workerTimingPolicy;
+        if (!injected) {
+            return production;
+        }
+        const { workerLeaseMs, workerHeartbeatMs } = injected;
+        if (!Number.isSafeInteger(workerLeaseMs) ||
+            workerLeaseMs <= 0 ||
+            !Number.isSafeInteger(workerHeartbeatMs) ||
+            workerHeartbeatMs <= 0 ||
+            workerHeartbeatMs >= workerLeaseMs ||
+            BigInt(workerHeartbeatMs) * BigInt(production.workerLeaseMs) !==
+                BigInt(workerLeaseMs) * BigInt(production.workerHeartbeatMs) ||
+            production.heartbeatDatabaseBusyTimeoutMs >= workerLeaseMs - workerHeartbeatMs) {
+            throw new Error('[Planu] Invalid validation worker timing policy');
+        }
+        return injected;
+    }
+    requireCurrentFence(current, fencingToken) {
+        if (!this.hasCurrentFence(current, fencingToken)) {
+            throw new ValidationAuthorityLostError('reclaimed');
+        }
+        if (!current.leaseExpiresAt || Date.parse(current.leaseExpiresAt) <= Date.now()) {
+            throw new ValidationAuthorityLostError('expired');
+        }
+    }
+    hasCurrentFence(current, fencingToken) {
+        return (current.state === 'running' &&
+            current.leaseOwner === this.workerId &&
+            current.fencingToken === fencingToken &&
+            !current.cancelRequested);
+    }
+    currentFence(identity, fencingToken) {
+        const current = this.observeForSettlement(identity);
+        this.requireCurrentFence(current, fencingToken);
+        return current;
+    }
+    observeForSettlement(identity) {
+        try {
+            return this.runtime.get(identity);
+        }
+        catch (error) {
+            reportClassifiedDegradation('VALIDATE_JOB_STATE_READ_FAILURE', error);
+            throw new ValidationAuthorityLostError('infrastructure', { cause: error });
+        }
+    }
+    recordFencedProgress(identity, fencingToken, checkpoint) {
+        try {
+            this.runtime.recordProgress({
+                ...identity,
+                workerId: this.workerId,
+                fencingToken,
+                leaseMs: this.workerTimingPolicy().workerLeaseMs,
+                checkpoint,
+            });
+        }
+        catch (error) {
+            reportClassifiedDegradation('VALIDATE_JOB_PROGRESS_FAILURE', error);
+            throw new ValidationAuthorityLostError('infrastructure', { cause: error });
+        }
+    }
+    failureDisposition(identity, fencingToken, error) {
+        let current;
+        try {
+            current = this.runtime.get(identity);
+        }
+        catch (readError) {
+            reportClassifiedDegradation('VALIDATE_JOB_STATE_READ_FAILURE', readError);
+            return { settleLocally: true };
+        }
+        if (current.state === 'cancelled' || current.cancelRequested) {
+            return { settleLocally: true, current };
+        }
+        if (!this.hasCurrentFence(current, fencingToken) ||
+            !current.leaseExpiresAt ||
+            Date.parse(current.leaseExpiresAt) <= Date.now()) {
+            return { settleLocally: true, current };
+        }
+        return error instanceof ValidationAuthorityLostError
+            ? { settleLocally: true, current }
+            : { settleLocally: false, current };
+    }
+    buildReceiptPendingCheckpoint(payload, receiptBindings, result, durationMs) {
+        const { logMaxBytes, resultMaxBytes } = this.receiptCheckpointLimits();
+        if (!Number.isSafeInteger(logMaxBytes) ||
+            logMaxBytes <= 0 ||
+            !Number.isSafeInteger(resultMaxBytes) ||
+            resultMaxBytes <= 0 ||
+            logMaxBytes > resultMaxBytes) {
+            throw new Error('[Planu] Invalid receipt checkpoint policy');
+        }
+        const redactedResult = redactSensitiveValue(result);
+        if (!redactedResult ||
+            typeof redactedResult !== 'object' ||
+            Buffer.byteLength(canonicalJson(redactedResult), 'utf8') > resultMaxBytes) {
+            throw new Error('[Planu] VALIDATION_CHECKPOINT_TOO_LARGE');
+        }
+        const receiptInput = {
+            bindings: receiptBindings,
+            command: {
+                executable: 'planu',
+                args: ['validate', payload.bindings.specId],
+                cwd: '<project>',
+            },
+            versions: { node: process.version, receiptSchema: '1' },
+            exitCode: 0,
+            durationMs,
+            log: truncateUtf8ByBytes(redactText(result.content.map((item) => item.text).join('\n')), logMaxBytes),
+            gates: [{ gate: 'validation', status: 'passed', durationMs }],
+            issuedAt: new Date().toISOString(),
+        };
+        const checkpoint = {
+            kind: 'validation-receipt-pending',
+            schemaVersion: 1,
+            errorCode: 'VALIDATION_RECEIPT_PENDING',
+            recoveryAction: 'restart_job',
+            result: redactedResult,
+            receiptInput,
+        };
+        const parsed = parseValidationReceiptPendingCheckpoint(checkpoint, this.receiptCheckpointLimits());
+        if (!parsed) {
+            throw new Error('[Planu] Validation receipt checkpoint is not persistable');
+        }
+        return parsed;
+    }
+    receiptCheckpointLimits() {
+        const policy = getRuntimePolicy().validation;
+        return {
+            logMaxBytes: this.options.receiptCheckpointPolicy?.logMaxBytes ?? policy.receiptCheckpointLogMaxBytes,
+            resultMaxBytes: this.options.receiptCheckpointPolicy?.resultMaxBytes ??
+                policy.receiptCheckpointResultMaxBytes,
+        };
+    }
+    async issueReceipt(identity, payload, checkpoint, publicationLease, fencingToken, signal) {
+        const receiptBindings = checkpoint.receiptInput.bindings;
+        if (executionAborted(signal)) {
+            signal.throwIfAborted();
+        }
+        if (!(await assertValidationFreshnessBarrier(publicationLease, receiptBindings))) {
             throw new Error('[Planu] Validation inputs changed before receipt publication');
         }
         const database = new RuntimeDatabase({ path: resolveStorageLayout().runtimeDatabase });
@@ -223,40 +659,47 @@ export class ValidateJobExecutor {
                 projectId: identity.projectId,
                 issuerId: LOCAL_VALIDATION_RECEIPT_ISSUER,
             });
+            signal.throwIfAborted();
             const { store } = createValidationReceiptAuthority({
                 database,
                 workspaceId: identity.workspaceId,
                 projectId: identity.projectId,
                 issuerId: LOCAL_VALIDATION_RECEIPT_ISSUER,
-            });
-            const receipt = await issueValidationReceipt({
-                bindings: receiptBindings,
-                command: {
-                    executable: 'planu',
-                    args: ['validate', payload.bindings.specId],
-                    ...(payload.input.projectPath === undefined ? {} : { cwd: payload.input.projectPath }),
+                authorizePersistence: (persistenceDatabase) => {
+                    persistenceDatabase.assertActiveJobLease({
+                        id: identity.operationId,
+                        projectId: identity.projectId,
+                        workspaceId: identity.workspaceId,
+                        workerId: this.workerId,
+                        fencingToken,
+                        leaseMs: this.workerTimingPolicy().workerLeaseMs,
+                        now: new Date().toISOString(),
+                    });
                 },
-                versions: { node: process.version, receiptSchema: '1' },
-                exitCode: 0,
-                durationMs,
-                log: result.content.map((item) => item.text).join('\n'),
-                gates: [{ gate: 'validation', status: 'passed', durationMs }],
-            }, {
+            });
+            const receipt = await issueValidationReceipt(checkpoint.receiptInput, {
                 issuerId: LOCAL_VALIDATION_RECEIPT_ISSUER,
                 signingKeyReference,
                 store,
             });
-            if (executionAborted(signal) ||
-                !(await assertValidationFreshnessBarrier(publicationLease, receiptBindings))) {
+            if (executionAborted(signal)) {
+                signal.throwIfAborted();
+            }
+            if (!(await assertValidationFreshnessBarrier(publicationLease, receiptBindings))) {
                 await store.invalidateCurrentBindings(identity.projectId, payload.bindings.specId);
                 throw new Error('[Planu] Validation inputs changed during receipt publication');
             }
             const receiptKey = `validation-receipt:${receipt.receiptId.slice('sha256:'.length)}`;
-            await this.persistReceiptArtifacts(identity, payload, receipt.receiptId, receiptKey);
+            try {
+                await this.persistReceiptArtifacts(identity, payload, receipt.receiptId, receiptKey);
+            }
+            catch (error) {
+                throw new ReceiptPublicationInterruptedError({ cause: error });
+            }
             return {
-                ...result,
+                ...checkpoint.result,
                 structuredContent: {
-                    ...(result.structuredContent ?? {}),
+                    ...(checkpoint.result.structuredContent ?? {}),
                     receiptId: receipt.receiptId,
                     receiptKey,
                 },
@@ -275,8 +718,14 @@ export class ValidateJobExecutor {
             return lease;
         }
         const bindings = payload.bindings;
-        const projectPath = payload.input.projectPath;
-        const specPath = payload.specPath;
+        const resolved = await revalidateValidationWorktree({
+            projectPath: payload.input.projectPath,
+            specPath: payload.specPath,
+            specId: payload.input.specId,
+            validationRootId: payload.validationRootId,
+        });
+        const projectPath = resolved.projectPath;
+        const specPath = resolved.specPath;
         const rebaselined = await rebaselineValidationFreshnessLeaseAfterRecompute({
             lease,
             expectedBindings,
@@ -322,19 +771,33 @@ export class ValidateJobExecutor {
 /** Schedules background execution after the submit response can be flushed. */
 export function scheduleValidateJobExecution(projectId, workspaceId = WORKSPACE_ID) {
     const scope = `${projectId}:${workspaceId}`;
-    if (activeScopes.has(scope)) {
+    const active = activeScopes.get(scope);
+    if (active) {
+        active.rescheduleRequested = true;
         return;
     }
-    activeScopes.add(scope);
+    const state = { rescheduleRequested: false };
+    activeScopes.set(scope, state);
     setImmediate(() => {
         const runtime = new DurableJobRuntime();
         const executor = new ValidateJobExecutor(runtime);
-        void executor.runUntilIdle(projectId, workspaceId).finally(() => {
-            runtime.close();
-            activeScopes.delete(scope);
-        });
+        void (async () => {
+            try {
+                do {
+                    state.rescheduleRequested = false;
+                    await executor.runUntilIdle(projectId, workspaceId);
+                } while (rescheduleRequested(state));
+            }
+            finally {
+                runtime.close();
+                activeScopes.delete(scope);
+            }
+        })();
     });
 }
+function rescheduleRequested(state) {
+    return state.rescheduleRequested;
+}
 /** Re-enqueues accepted/checkpointed validate work discovered after a process restart. */
 export function recoverValidateJobsAtStartup(options = {}) {
     recoverValidateJobs(scheduleValidateJobExecution, options);