firedrill-run 2.2.5 → 3.0.1

package/dist/cli/src/runner-evidence.js

@@ -0,0 +1,326 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, mkdir, open, rename, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { api } from './auth.js';
+import { privateDirectory } from './credentials.js';
+import { CliError } from './http.js';
+import { identifier } from './world-contracts.js';
+import { historicalRunnerSigner } from './runner-identity.js';
+import { withLocalJobInputs } from './runner-ownership.js';
+import { executeLocalValidation } from './runner-container.js';
+import { syncLocalCompletion } from './runner-completion.js';
+import { evidenceNeedles } from './runner-evidence-privacy.js';
+import { localEvidenceReceiptMatches } from '../../contracts/local-evidence-contracts.js';
+import { acquireLocalEvidence, assertEvidenceRetentionCurrent, evidenceRetentionSignal, frozenEvidenceRetention, retentionMatches } from './runner-evidence-retention.js';
+const CHUNK = 1024 ** 2, HASH = /^[a-f0-9]{64}$/;
+const fail = () => new CliError('local_evidence_unconfirmed', 'Evidence is not confirmed in the cloud. Its private local journal remains; no target was run again.');
+const canonical = (value) => {
+    if (Array.isArray(value))
+        return '[' + value.map(canonical).join(',') + ']';
+    if (value && typeof value === 'object')
+        return '{' + Object.keys(value).sort().map(key => JSON.stringify(key) + ':' + canonical(value[key])).join(',') + '}';
+    const result = JSON.stringify(value);
+    if (result === undefined)
+        throw fail();
+    return result;
+};
+const hash = (value) => createHash('sha256').update(value).digest('hex');
+async function folder(owner, operation) {
+    const path = join(await privateDirectory(), `evidence-${identifier(owner.owner)}-${identifier(owner.workspace)}-${identifier(operation)}`);
+    await mkdir(path, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST')
+        throw error; });
+    const stat = await lstat(path);
+    if (!stat.isDirectory() || stat.isSymbolicLink() || stat.mode & 0o077 || process.getuid && stat.uid !== process.getuid())
+        throw fail();
+    return path;
+}
+async function persist(directory, value, initial = false) {
+    assertEvidenceRetentionCurrent(value.retention);
+    const path = join(directory, 'receipt.json'), temporary = path + '.' + randomUUID() + '.tmp';
+    const file = await open(initial ? path : temporary, 'wx', 0o600);
+    try {
+        await file.writeFile(JSON.stringify(value));
+        await file.sync();
+    }
+    finally {
+        await file.close();
+    }
+    if (!initial)
+        await rename(temporary, path);
+    const parent = await open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0));
+    try {
+        await parent.sync();
+    }
+    finally {
+        await parent.close();
+    }
+}
+async function ownedFile(path, maximum) {
+    const before = await lstat(path);
+    if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.mode & 0o077 || before.size > maximum
+        || process.getuid && before.uid !== process.getuid())
+        throw fail();
+    const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    const actual = await file.stat();
+    if (actual.ino !== before.ino || actual.dev !== before.dev || actual.size !== before.size) {
+        await file.close();
+        throw fail();
+    }
+    return file;
+}
+async function collect(owner, manifestSha256, observation, read, configuration, retention, signal) {
+    signal = evidenceRetentionSignal(retention, signal);
+    const directory = await folder(owner, owner.lease);
+    const journal = { schema: 'firedrill.local-evidence-journal.v1', owner, manifest_sha256: manifestSha256,
+        key: randomUUID(), stage: 'collecting', observation, document: null, retention };
+    await persist(directory, journal, true);
+    const kind = observation.launch.state === 'exited' && observation.launch.exit_code === 0 && observation.status?.state === 'native_validated'
+        ? 'native_validation' : observation.launch.state === 'exited' && observation.launch.exit_code !== 0
+        && observation.status?.state === 'failed' && observation.status.operation_attempt_available === true ? 'native_operation_attempt' : null;
+    if (!kind) {
+        // Preserve actual redacted failure/unknown metadata, never invent an archive
+        // or treat it as a passing native validation. Failed-trial sync is separate.
+        journal.stage = 'local_retained';
+        await persist(directory, journal);
+        return journal;
+    }
+    const exported = await read(kind), blocked = evidenceNeedles(configuration), overlap = blocked.reduce((maximum, value) => Math.max(maximum, value.length), 1);
+    if (exported.kind !== kind)
+        throw fail();
+    const name = kind === 'native_validation' ? 'native' : 'operations';
+    const partial = join(directory, name + '.partial'), final = join(directory, kind === 'native_validation' ? 'native.zip' : 'operations.json');
+    const file = await open(partial, 'wx', 0o600), digest = createHash('sha256');
+    let offset = 0, tail = Buffer.alloc(0), accepted = false;
+    try {
+        while (offset < exported.receipt.size) {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(retention);
+            const body = await exported.read(offset, Math.min(16384, exported.receipt.size - offset));
+            assertEvidenceRetentionCurrent(retention);
+            const scan = Buffer.concat([tail, body]);
+            if (blocked.some(value => scan.includes(value)))
+                throw new CliError('local_evidence_private_value', 'A private mapped value was found in local evidence. It was not uploaded.');
+            tail = Buffer.from(scan.subarray(Math.max(0, scan.length - overlap + 1)));
+            digest.update(body);
+            let written = 0;
+            while (written < body.length) {
+                const result = await file.write(body, written, body.length - written);
+                if (!result.bytesWritten)
+                    throw fail();
+                written += result.bytesWritten;
+            }
+            offset += body.length;
+        }
+        if (offset !== exported.receipt.size || digest.digest('hex') !== exported.receipt.sha256)
+            throw fail();
+        await file.sync();
+        accepted = true;
+    }
+    finally {
+        await file.close();
+        if (!accepted)
+            await unlink(partial).catch(() => undefined);
+        blocked.length = 0;
+        tail.fill(0);
+    }
+    await rename(partial, final);
+    journal.document = { kind, receipt: exported.receipt, launch: observation.launch, status: observation.status };
+    journal.stage = 'local_retained';
+    await persist(directory, journal);
+    return journal;
+}
+const endpoint = (owner, key) => `/api/workspaces/${identifier(owner.workspace)}/local-runners/${identifier(owner.device)}/jobs/${identifier(owner.job)}/claims/${identifier(owner.request)}/evidence/${identifier(key)}`;
+function matching(value, journal) {
+    if (!localEvidenceReceiptMatches(value, { ...journal.owner, manifestSha256: journal.manifest_sha256 }, journal.key)
+        || value.schema !== 'firedrill.local-evidence-receipt.v1' || value.job_id !== journal.owner.job
+        || value.transfer_key !== journal.key || value.lease_id !== journal.owner.lease || value.provenance !== 'customer_device'
+        || value.cloud_completion_recorded !== false || !Array.isArray(value.chunks)
+        || !['not_retained', 'receiving', 'verifying', 'retained'].includes(value.state)
+        || value.state !== 'not_retained' && (canonical(value.document) !== canonical(journal.document) || value.request_digest !== hash(canonical(journal.document))))
+        throw fail();
+    if (value.state !== 'retained' && (!Number.isSafeInteger(Date.parse(value.server_time)) || value.upload_available !== true
+        || !Number.isSafeInteger(Date.parse(value.upload_until)) || Date.parse(value.upload_until) <= Date.parse(value.server_time)))
+        throw fail();
+}
+async function confirmCloud(journal, receipt, directory) {
+    const artifact = receipt.artifact;
+    if (!journal.document || receipt.state !== 'retained' || !artifact || identifier(artifact.id) !== artifact.id
+        || typeof artifact.generation !== 'string' || !/^[1-9][0-9]*$/.test(artifact.generation)
+        || artifact.sha256 !== journal.document.receipt.sha256 || artifact.size !== journal.document.receipt.size)
+        throw fail();
+    journal.stage = 'cloud_retained';
+    journal.cloud = artifact;
+    await persist(directory, journal);
+    return { state: 'cloud_retained', artifact, provenance: 'customer_device', cloud_completion_recorded: false };
+}
+async function synchronize(journal, signal) {
+    signal = evidenceRetentionSignal(journal.retention, signal);
+    const owner = journal.owner, directory = await folder(owner, owner.lease);
+    if (!journal.document)
+        return { state: 'local_metadata_only', cloud_completion_recorded: false };
+    if (!owner.evidenceUpload)
+        return { state: 'local_only', cloud_completion_recorded: false };
+    const auth = { actor: owner.owner, workspace: owner.workspace, signal }, path = endpoint(owner, journal.key);
+    if (journal.stage === 'cloud_retained') {
+        const receipt = await api(path, auth);
+        matching(receipt, journal);
+        return confirmCloud(journal, receipt, directory); // Local bytes can be removed after verified cloud retention.
+    }
+    if (!['native_validation', 'native_operation_attempt'].includes(journal.document.kind))
+        throw fail();
+    const file = await ownedFile(join(directory, journal.document.kind === 'native_validation' ? 'native.zip' : 'operations.json'), (journal.document.kind === 'native_validation' ? 128 : 8) * CHUNK);
+    try {
+        const bytes = await file.stat(), digest = createHash('sha256');
+        for await (const body of file.createReadStream({ autoClose: false, start: 0 })) {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(journal.retention);
+            digest.update(body);
+        }
+        if (bytes.size !== journal.document.receipt.size || digest.digest('hex') !== journal.document.receipt.sha256)
+            throw fail();
+        let receipt = await api(path, auth);
+        matching(receipt, journal);
+        const signer = await historicalRunnerSigner(owner, owner.device, signal);
+        const send = async (action, fields) => {
+            assertEvidenceRetentionCurrent(journal.retention);
+            const sent = Date.parse(receipt.server_time);
+            if (!Number.isSafeInteger(sent))
+                throw fail();
+            const proof = await signer.sign({ schema: 'firedrill.local-evidence-write.v1', job_id: owner.job,
+                attempt_id: owner.attempt, lease_id: owner.lease, request_key: owner.request, device_key: owner.device,
+                native_session_id: signer.nativeSession, transfer_key: journal.key, manifest_sha256: journal.manifest_sha256,
+                sent_at_ms: sent, action, ...fields });
+            try {
+                receipt = await api(path, { ...auth, method: 'POST', body: proof });
+            }
+            catch {
+                receipt = await api(path, auth);
+            } // Exact read, no target or write replay.
+            matching(receipt, journal);
+        };
+        if (receipt.state === 'not_retained') {
+            journal.stage = 'register_dispatched';
+            await persist(directory, journal);
+            await send('register', { document: journal.document });
+            if (!['receiving', 'verifying', 'retained'].includes(receipt.state))
+                throw fail();
+        }
+        if (!['receiving', 'verifying', 'retained'].includes(receipt.state))
+            throw fail();
+        journal.stage = 'uploading';
+        await persist(directory, journal);
+        const buffer = Buffer.alloc(CHUNK), count = Math.ceil(bytes.size / CHUNK);
+        for (let index = 0; index < count && receipt.state === 'receiving'; index++) {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(journal.retention);
+            const size = Math.min(CHUNK, bytes.size - index * CHUNK);
+            let received = 0;
+            while (received < size) {
+                const part = await file.read(buffer, received, size - received, index * CHUNK + received);
+                if (!part.bytesRead)
+                    throw fail();
+                received += part.bytesRead;
+            }
+            const body = buffer.subarray(0, size), checksum = hash(body), current = receipt.chunks.find(item => item.index === index);
+            if (current && (current.size !== size || current.sha256 !== checksum))
+                throw fail();
+            if (!current?.retained)
+                await send('chunk', { index, sha256: checksum, data: body.toString('base64') });
+            if (!receipt.chunks.some(item => item.index === index && item.retained && item.sha256 === checksum && item.size === size))
+                throw fail();
+        }
+        if (receipt.state === 'receiving' || receipt.state === 'verifying') {
+            journal.stage = 'commit_dispatched';
+            await persist(directory, journal);
+            await send('commit', {});
+        }
+        while (receipt.state === 'verifying') {
+            // Observe the exact retained outbox intent, not invented progress or a
+            // repeated commit. The existing job keeper independently maintains access.
+            if (receipt.authorization_current !== true)
+                throw new CliError('local_evidence_authorization_required', 'The saved evidence needs authorization from this terminal session. Run the same evidence sync command again. No target was run again.');
+            await sleep(1000, undefined, { signal });
+            receipt = await api(path, auth);
+            matching(receipt, journal);
+        }
+        return confirmCloud(journal, receipt, directory);
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Internal full evidence handoff; public execution awaits cloud completion. */
+export async function retainLocalValidation(context, device, job, grantRequest, signal) {
+    const progress = {};
+    let executionError;
+    try {
+        await withLocalJobInputs(context, device, job, grantRequest, 'validation', signal, async (owner, inputs, configuration, active) => {
+            progress.owner = owner;
+            if (owner.purpose !== 'validation')
+                throw fail();
+            const evidence = await acquireLocalEvidence(owner, owner.lease, frozenEvidenceRetention(owner, inputs.manifestSha256, inputs.manifest.evidence_recovery_until));
+            try {
+                active = AbortSignal.any([active, evidence.signal]);
+                return await executeLocalValidation(owner, inputs, configuration, (current, observation, exported, running) => collect(current, inputs.manifestSha256, observation, exported, configuration, evidence.retention, running), active);
+            }
+            finally {
+                await evidence.close();
+            }
+        });
+    }
+    catch (error) {
+        executionError = error;
+    }
+    const claimed = progress.owner;
+    if (!claimed)
+        throw executionError ?? fail();
+    // The keeper has stopped and transient configuration has been cleared. Exact
+    // container cleanup is already journaled. Closing facts use a fresh bounded
+    // channel even after interruption; this neither renews nor executes work.
+    try {
+        await syncLocalCompletion(claimed, claimed.lease, AbortSignal.timeout(30000));
+    }
+    catch (error) {
+        throw executionError ?? error;
+    }
+    if (executionError)
+        throw executionError;
+    const evidence = await syncLocalEvidence(claimed, claimed.lease, signal);
+    const completion = await syncLocalCompletion(claimed, claimed.lease, signal, { wait: true });
+    return { evidence, completion };
+}
+/** Resume artifact transfer only. Never call the execution callback again. */
+export async function syncLocalEvidence(scope, operation, signal) {
+    const evidence = await acquireLocalEvidence(scope, identifier(operation));
+    try {
+        return await syncRetainedLocalEvidence(scope, operation, AbortSignal.any([signal, evidence.signal]), evidence.retention);
+    }
+    finally {
+        await evidence.close();
+    }
+}
+async function syncRetainedLocalEvidence(scope, operation, signal, retention) {
+    const directory = await folder(scope, operation), lock = join(directory, 'sync.lock');
+    // The outer process-lifetime evidence lock owns all new transfers. A legacy
+    // mkdir lock is ambiguous and is preserved for explicit inspection, not
+    // silently treated as a released kernel lock or removed by erasure.
+    if (await lstat(lock).then(() => true).catch(error => { if (error.code === 'ENOENT')
+        return false; throw error; }))
+        throw new CliError('local_evidence_legacy_lock', 'This older evidence transfer has an ambiguous sync.lock. Inspect that exact legacy lock before continuing; it was not removed.');
+    const file = await ownedFile(join(directory, 'receipt.json'), 65536);
+    let journal;
+    try {
+        journal = JSON.parse(await file.readFile('utf8'));
+    }
+    finally {
+        await file.close();
+    }
+    if (journal.schema !== 'firedrill.local-evidence-journal.v1' || !journal.owner || journal.owner.owner !== scope.owner || journal.owner.workspace !== scope.workspace
+        || journal.owner.lease !== identifier(operation) || !HASH.test(journal.manifest_sha256)
+        || !retentionMatches(journal.retention, retention) || journal.manifest_sha256 !== retention.manifest_sha256
+        || !['local_retained', 'register_dispatched', 'uploading', 'commit_dispatched', 'cloud_retained'].includes(journal.stage))
+        throw fail();
+    return synchronize(journal, signal);
+}

package/dist/cli/src/runner-execute.js

@@ -0,0 +1,92 @@
+import { randomUUID } from 'node:crypto';
+import { localAssignmentsPage, readLocalClaim } from './runner-claim-reads.js';
+import { CliError } from './http.js';
+import { selectOne } from './prompts.js';
+import { executionPermission, localPermissionPage } from './runner-grants.js';
+import { withRunnerPresence } from './runner-identity.js';
+import { retainLocalValidation } from './runner-evidence.js';
+import { retainLocalTrial } from './runner-trial-journal.js';
+import { retainLocalWorldSession } from './runner-world-collector.js';
+import { selectedWorld } from './world-access.js';
+import { identifier } from './world-contracts.js';
+import { retainedLocalClaim } from './runner-ownership.js';
+const purposeLabel = (value) => value === 'trial' ? 'Simulation drill' : value === 'world_session' ? 'Manual world session' : 'World validation';
+async function choose(label, choices, signal, readPage) {
+    let page = 1;
+    while (true) {
+        const result = await readPage(page);
+        const items = result.results.flatMap(row => { const item = choices(row); return item ? [item] : []; });
+        if (page > 1)
+            items.unshift({ value: 'previous', label: 'Previous page' });
+        if (result.next)
+            items.push({ value: 'next', label: 'Next page' });
+        if (!items.length)
+            throw new CliError('local_selection_empty', `No ${label.toLowerCase()} is available for this world and device.`, 2);
+        const selected = await selectOne(label, items, signal);
+        if (selected === 'previous') {
+            page--;
+            continue;
+        }
+        if (selected === 'next') {
+            page++;
+            continue;
+        }
+        return identifier(selected);
+    }
+}
+/** Execute exactly one already-admitted assignment, never arbitrary host code,
+ * a new campaign, an implicit target retry or a cloud fallback.
+ */
+export async function executeAssignment(device, job, options, output) {
+    device = identifier(device);
+    const interactive = !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY;
+    const context = await selectedWorld({ workspace: options.workspace, world: options.world,
+        interactive, openBrowser: options.openBrowser, signal: options.signal });
+    if (!job && interactive)
+        job = await choose('Saved local job', row => row.world_id === context.world && row.device_key === device && row.job_state === 'queued' && !row.claim
+            ? { value: row.job_id, label: row.world_name, detail: purposeLabel(row.purpose) } : null, options.signal, page => localAssignmentsPage(context, device, { world: context.world, available: true, page }, options.signal));
+    if (!job)
+        throw new CliError('local_job_required', 'Provide a saved job ID, or run interactively to choose an assignment.', 2);
+    job = identifier(job);
+    // GET of a fresh lookup identity creates no claim or grant. A prior claim
+    // refuses this new request, rather than being taken over or replayed.
+    const original = await retainedLocalClaim(context, device, job);
+    if (original && ['work_dispatched', 'not_dispatched'].includes(original.stage))
+        throw new CliError('local_execution_recovery_required', original.stage === 'not_dispatched' ? 'This job was abandoned before launch. Repeat runner abandon to reconcile its original closing receipt.'
+            : 'This job already handed off to a container. Use runner cleanup and evidence sync for its original operation.');
+    const offered = await readLocalClaim(context, device, job, original?.request ?? randomUUID(), options.signal);
+    if (offered.job_id !== job || offered.world_id !== context.world || offered.device_key !== device
+        || !['validation', 'trial', 'world_session'].includes(offered.purpose)
+        || (!original && (offered.claim || offered.job_state !== 'queued'))
+        || (original && (!offered.claim || offered.claim.request_key !== original.request))) {
+        throw new CliError('local_assignment_changed', 'The saved assignment is not available for a first execution. Inspect its exact ownership.');
+    }
+    if (original && options.grant && options.grant !== original.grantRequest)
+        throw new CliError('local_assignment_changed', 'Resume uses the original permission. A different grant cannot replace it.');
+    let grantRequest = original?.grantRequest ?? options.grant;
+    if (!grantRequest && interactive)
+        grantRequest = await choose('Local permission', row => row.id === offered.grant_id && row.state === 'active' && row.device_key === device
+            && row.operations.includes(offered.purpose) ? { value: row.request_key, label: 'Use the saved permission', detail: `Expires ${row.expires_at}` } : null, options.signal, page => localPermissionPage(context, { device, page }, options.signal));
+    if (!grantRequest)
+        throw new CliError('local_grant_required', 'Provide --grant <permission-request-id> or choose the saved permission interactively.', 2);
+    const permission = await executionPermission(context, identifier(grantRequest), device, offered.purpose, options.signal);
+    if (permission.id !== offered.grant_id)
+        throw new CliError('local_assignment_changed', 'This permission does not own the saved assignment.');
+    output('local_execution_review', { job_id: job, world_id: context.world, device_key: device, purpose: offered.purpose,
+        evidence_upload: permission.document.consent.evidence_upload, limits: permission.document.limits }, `${offered.world_name}: ${purposeLabel(offered.purpose).toLowerCase()} on this device.\nJob: ${job}\n`
+        + (permission.document.consent.evidence_upload ? 'Evidence will be retained in this world’s cloud history.'
+            : 'Evidence sharing is off. Private files stay local; cloud readiness and scoring require shared evidence.')
+        + `\nBefore container handoff, resume with runner execute ${device} ${job} --world ${context.world}, or close with runner abandon ${device} ${job} --world ${context.world}.`);
+    if (!options.approve) {
+        if (!interactive)
+            throw new CliError('local_execution_confirmation_required', 'Use --execute-local to start this exact saved assignment.', 2);
+        if (!await selectOne('Start this saved local job?', [{ value: true, label: 'Run locally' },
+            { value: false, label: 'Cancel' }], options.signal))
+            throw new CliError('canceled', 'No job was claimed or executed.', 130);
+    }
+    const result = await withRunnerPresence(context, device, options.signal, output, async (active) => offered.purpose === 'trial'
+        ? await retainLocalTrial(context, device, job, grantRequest, active, output)
+        : offered.purpose === 'world_session' ? await retainLocalWorldSession(context, device, job, grantRequest, active)
+            : await retainLocalValidation(context, device, job, grantRequest, active));
+    output('local_execution_retained', result, 'Local execution and evidence handoff returned. The saved cloud receipt records its actual outcome.');
+}

package/dist/cli/src/runner-execution-deadline.js

@@ -0,0 +1,25 @@
+import { CliError } from './http.js';
+/** Original job authority, not a fresh window each time a pipe is opened. */
+export function localExecutionTimeout(configuration, closingGraceMs = 0) {
+    const control = configuration.control;
+    if (!Number.isSafeInteger(control.budget_seconds) || control.budget_seconds < 1
+        || !Number.isSafeInteger(closingGraceMs) || closingGraceMs < 0)
+        throw invalid();
+    const deadline = Date.parse(configuration.job_deadline_at);
+    if (!Number.isSafeInteger(deadline)
+        || control.schema === 'firedrill.trial.v1' && control.job_deadline_at !== configuration.job_deadline_at
+        || control.schema === 'firedrill.world-session.v1' && control.expires_at !== configuration.job_deadline_at)
+        throw invalid();
+    // Validation has no separate deadline field in its native control document.
+    // The shared, identity-checked delivery envelope still binds its original
+    // absolute window. Bootstrap, Docker setup or host sleep must not restart it.
+    const remaining = Math.min(control.budget_seconds * 1000, deadline - Date.now());
+    if (remaining <= 0)
+        throw new CliError('local_execution_window_ended', 'This execution window ended. Inspect the original run; the target was not restarted.');
+    // Native target supervision ends work before this optional host closing
+    // window. The extra time only permits its original evidence/cleanup readback.
+    return Math.min(2147483647, Math.ceil(remaining) + closingGraceMs);
+}
+function invalid() {
+    return new CliError('local_runtime_packet_invalid', 'The native execution deadline does not match the approved job. No target was started.');
+}