firedrill-run 2.2.4 → 3.0.0

package/dist/cli/src/runner-inputs.js

@@ -0,0 +1,308 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, mkdir, open, rename, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { api } from './auth.js';
+import { readLocalClaim } from './runner-claim-reads.js';
+import { localInputPacketMatches, localInputPendingMatches } from '../../contracts/local-input-contracts.js';
+import { privateDirectory } from './credentials.js';
+import { CliError } from './http.js';
+import { signRunnerRequest } from './runner-identity.js';
+import { identifier } from './world-contracts.js';
+import { processLifetimeLock } from './runner-process-lock.js';
+const CHUNK = 256 * 1024;
+const HASH = /^[a-f0-9]{64}$/;
+const IMAGE = /^[a-z][a-z0-9-]{0,62}-docker\.pkg\.dev\/[a-z][a-z0-9-]{4,62}\/[a-z][a-z0-9-]{0,62}\/[a-z][a-z0-9-]{0,62}@sha256:[a-f0-9]{64}$/;
+const FILES = { source: 'source.zip', candidate: 'candidates.zip', compiled: 'validation.zip', scenarios: 'scenarios.json', packages: 'packages.zip', dataset: 'world-dataset.json', checkpoint: 'initial-checkpoint.json', application_checkpoint: 'initial-application-checkpoint.json' };
+const MAXIMUM = { source: 128 * 1024 ** 2, candidate: 128 * 1024 ** 2, compiled: 128 * 1024 ** 2, scenarios: 64 * 1024 ** 2,
+    packages: 64 * 1024 ** 3, dataset: 64 * 1024 ** 2, checkpoint: 512 * 1024 ** 2, application_checkpoint: 8 * 1024 ** 2 }; // Streaming file guard, not a customer simulation quota.
+function canonical(value) {
+    if (Array.isArray(value))
+        return '[' + value.map(canonical).join(',') + ']';
+    if (value && typeof value === 'object') {
+        const row = value;
+        return '{' + Object.keys(row).sort().map(key => JSON.stringify(key) + ':' + canonical(row[key])).join(',') + '}';
+    }
+    const body = JSON.stringify(value);
+    if (body === undefined)
+        throw new CliError('local_input_invalid', 'The input manifest is incomplete.');
+    return body;
+}
+const sha = (value) => createHash('sha256').update(value).digest('hex');
+const invalid = () => new CliError('local_input_identity_changed', 'The execution inputs do not match their retained identities. No agent was started.');
+const prefix = (job) => `/api/workspaces/${identifier(job.workspace)}/local-runners/${identifier(job.device)}/jobs/${identifier(job.job)}/claims/${identifier(job.request)}`;
+/** Unlink only these private downloads, not original data, journals or backups. */
+export async function discardPrivateDataInputs(job) {
+    const parent = join(await privateDirectory(), 'execution-inputs');
+    const parentRow = await lstat(parent).catch(error => { if (error.code === 'ENOENT')
+        return null; throw error; });
+    if (!parentRow)
+        return;
+    if (!parentRow.isDirectory() || parentRow.isSymbolicLink() || parentRow.mode & 0o077 || process.getuid && parentRow.uid !== process.getuid())
+        throw invalid();
+    const directory = join(parent, [job.owner, job.workspace, job.job, job.lease].map(identifier).join('-'));
+    const folder = await lstat(directory).catch(error => { if (error.code === 'ENOENT')
+        return null; throw error; });
+    if (!folder)
+        return;
+    if (!folder.isDirectory() || folder.isSymbolicLink() || folder.mode & 0o077 || process.getuid && folder.uid !== process.getuid())
+        throw invalid();
+    for (const name of ['world-dataset.json', 'world-dataset.json.partial', 'initial-checkpoint.json', 'initial-checkpoint.json.partial', 'initial-application-checkpoint.json', 'initial-application-checkpoint.json.partial']) {
+        const path = join(directory, name);
+        const before = await lstat(path).catch(error => { if (error.code === 'ENOENT')
+            return null; throw error; });
+        if (!before)
+            continue;
+        if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.mode & 0o077
+            || before.size > (name.startsWith('initial-application') ? MAXIMUM.application_checkpoint : name.startsWith('initial-checkpoint') ? MAXIMUM.checkpoint : MAXIMUM.dataset)
+            || process.getuid && before.uid !== process.getuid())
+            throw invalid();
+        await unlink(path);
+    }
+}
+async function privateFolder(path) {
+    await mkdir(path, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST')
+        throw error; });
+    const row = await lstat(path);
+    if (!row.isDirectory() || row.isSymbolicLink() || row.mode & 0o077 || process.getuid && row.uid !== process.getuid()) {
+        throw new CliError('local_input_storage_unsafe', 'Execution input storage must be private and owned by this OS user.');
+    }
+}
+async function privateFile(path, writable = false) {
+    const before = await lstat(path);
+    if (!before.isFile() || before.isSymbolicLink() || before.mode & 0o077 || before.nlink !== 1
+        || process.getuid && before.uid !== process.getuid())
+        throw invalid();
+    const file = await open(path, (writable ? constants.O_RDWR : constants.O_RDONLY) | (constants.O_NOFOLLOW ?? 0));
+    const actual = await file.stat();
+    if (actual.dev !== before.dev || actual.ino !== before.ino || actual.size !== before.size) {
+        await file.close();
+        throw invalid();
+    }
+    return file;
+}
+function checkManifest(result, job, expected) {
+    const value = result.manifest;
+    if (!localInputPacketMatches(result, job) || !value || value.schema !== 'firedrill.local-input-manifest.v1'
+        || !HASH.test(result.manifest_sha256) || sha(canonical(value)) !== result.manifest_sha256
+        || expected && result.manifest_sha256 !== expected || value.job_id !== job.job || value.grant_id !== job.grant
+        || value.payload_digest !== job.payloadDigest || value.purpose !== job.purpose
+        || canonical(value.runtime) !== canonical(job.runtime) || !Array.isArray(value.entries)
+        || value.entries.length !== (job.purpose === 'world_session' ? 3 : 4)
+            + (['world_session', 'trial'].includes(job.purpose) && value.entries.some(row => row?.part === 'dataset') ? 1 : 0)
+            + (['world_session', 'trial'].includes(job.purpose) && value.entries.some(row => row?.part === 'checkpoint') ? 1 : 0)
+            + (['world_session', 'trial'].includes(job.purpose) && value.entries.some(row => row?.part === 'application_checkpoint') ? 1 : 0))
+        throw invalid();
+    if (typeof value.evidence_recovery_until !== 'string' || !Number.isSafeInteger(Date.parse(value.evidence_recovery_until))
+        || Date.parse(value.evidence_recovery_until) <= Date.now())
+        throw new CliError('local_evidence_retention_missing', 'The original inputs lack a current finite evidence policy. No local target was started. Inspect original recovery metadata or explicitly forget retained evidence after cleanup.');
+    const names = new Set();
+    const requiredParts = new Set(job.purpose === 'world_session' ? ['source', 'compiled', 'packages']
+        : ['source', job.purpose === 'trial' ? 'compiled' : 'candidate', 'packages', 'scenarios']);
+    if (['trial', 'world_session'].includes(job.purpose) && value.entries.some(row => row?.part === 'dataset'))
+        requiredParts.add('dataset');
+    if (['world_session', 'trial'].includes(job.purpose) && value.entries.some(row => row?.part === 'checkpoint')) {
+        if (requiredParts.has('dataset') || job.purpose === 'trial' && !value.entries.some(row => row?.part === 'application_checkpoint'))
+            throw invalid();
+        requiredParts.add('checkpoint');
+    }
+    if (['world_session', 'trial'].includes(job.purpose) && value.entries.some(row => row?.part === 'application_checkpoint')) {
+        if (job.purpose === 'trial' && requiredParts.has('dataset'))
+            throw invalid();
+        requiredParts.add('application_checkpoint');
+    }
+    for (const entry of value.entries) {
+        if (!entry || !requiredParts.has(entry.part) || !Object.hasOwn(FILES, entry.part) || names.has(entry.part) || !HASH.test(entry.sha256)
+            || !Number.isSafeInteger(entry.size) || entry.size <= 0 || entry.size > MAXIMUM[entry.part])
+            throw invalid();
+        names.add(entry.part);
+    }
+    return value;
+}
+async function readOnce(job, signal, part, offset = 0, length = 0, manifestSha256 = null) {
+    const auth = { actor: job.owner, workspace: job.workspace, signal };
+    const receipt = await readLocalClaim(job, job.device, job.job, job.request, signal);
+    if (receipt.job_id !== job.job || receipt.grant_id !== job.grant || receipt.device_key !== job.device
+        || receipt.payload_digest !== job.payloadDigest || receipt.purpose !== job.purpose || receipt.claim?.id !== job.lease
+        || receipt.claim.attempt_id !== job.attempt || receipt.claim.request_key !== job.request || receipt.claim.state !== 'claimed')
+        throw invalid();
+    const sentAt = Date.parse(receipt.server_time), expires = Date.parse(receipt.claim.expires_at);
+    if (!Number.isSafeInteger(sentAt) || !Number.isFinite(expires) || expires <= sentAt)
+        throw invalid();
+    const payload = { schema: 'firedrill.local-input-read.v1',
+        job_id: job.job, device_key: job.device, request_key: job.request, native_session_id: job.nativeSession,
+        lease_id: job.lease, part, offset, length, manifest_sha256: manifestSha256, read_key: randomUUID(), sent_at_ms: sentAt };
+    const signed = await signRunnerRequest(job, job.device, payload, signal);
+    // A read may be repeated after reconciliation, but it never renews a lease,
+    // allocates another attempt or replays a launch. No control token enters files.
+    const value = await api(prefix(job) + '/inputs', { ...auth, method: 'POST', body: signed });
+    if (!value || (value.schema === 'firedrill.local-input-pending.v1' ? part !== 'manifest' || !localInputPendingMatches(value, job)
+        : !localInputPacketMatches(value, job, { part, offset, length })))
+        throw invalid();
+    return value;
+}
+async function read(job, signal, part, offset = 0, length = 0, manifestSha256 = null) {
+    while (true) {
+        const result = await readOnce(job, signal, part, offset, length, manifestSha256);
+        if (result.schema !== 'firedrill.local-input-pending.v1')
+            return result;
+        if (part !== 'manifest' || result.job_id !== job.job || result.lease_id !== job.lease
+            || !['preparing_packages', 'preparing_starting_data'].includes(result.input_state) || !Number.isSafeInteger(result.retry_after_ms)
+            || result.retry_after_ms < 250 || result.retry_after_ms > 10000)
+            throw invalid();
+        // Wait only on the actual retained asynchronous input state. The existing
+        // keeper owns lease renewal; this read loop cannot claim/restart a job.
+        await sleep(result.retry_after_ms, undefined, { signal });
+    }
+}
+async function hashFile(path, maximum) {
+    const file = await privateFile(path);
+    try {
+        const size = (await file.stat()).size;
+        if (size > maximum)
+            throw invalid();
+        const digest = createHash('sha256'), buffer = Buffer.allocUnsafe(CHUNK);
+        let position = 0;
+        while (position < size) {
+            const { bytesRead } = await file.read(buffer, 0, Math.min(buffer.length, size - position), position);
+            if (!bytesRead)
+                throw invalid();
+            digest.update(buffer.subarray(0, bytesRead));
+            position += bytesRead;
+        }
+        if ((await file.stat()).size !== size)
+            throw invalid();
+        return { size, sha256: digest.digest('hex') };
+    }
+    finally {
+        await file.close();
+    }
+}
+async function retainManifest(directory, result) {
+    const path = join(directory, 'manifest.json');
+    const body = JSON.stringify({ schema: 'firedrill.local-retained-inputs.v1', manifest: result.manifest, sha256: result.manifest_sha256 });
+    try {
+        const file = await open(path, 'wx', 0o600);
+        try {
+            await file.writeFile(body);
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+    }
+    catch (error) {
+        if (error.code !== 'EEXIST')
+            throw error;
+    }
+    const file = await privateFile(path);
+    try {
+        if ((await file.stat()).size > 65536 || await file.readFile('utf8') !== body)
+            throw invalid();
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Internal executor input transfer. Not a standalone run or launch command. */
+export async function receiveLocalInputs(job, signal) {
+    for (const value of [job.owner, job.workspace, job.device, job.job, job.request, job.lease, job.attempt, job.nativeSession, job.grant])
+        identifier(value);
+    if (!HASH.test(job.payloadDigest) || !['validation', 'trial', 'world_session'].includes(job.purpose) || !job.runtime
+        || job.runtime.os !== 'linux' || !['amd64', 'arm64'].includes(job.runtime.arch) || !IMAGE.test(job.runtime.image)
+        || !/^[a-f0-9]{40}$/.test(job.runtime.source_revision) || job.runtime.profile !== 'firedrill.local-container.v1')
+        throw invalid();
+    const parent = join(await privateDirectory(), 'execution-inputs');
+    await privateFolder(parent);
+    const directory = join(parent, [job.owner, job.workspace, job.job, job.lease].join('-'));
+    await privateFolder(directory);
+    const lock = await processLifetimeLock(join(directory, 'transfer.lock'));
+    signal = AbortSignal.any([signal, lock.signal]);
+    let completed = false;
+    try {
+        lock.assertHeld();
+        signal.throwIfAborted();
+        const result = await read(job, signal, 'manifest');
+        const manifest = checkManifest(result, job);
+        if (result.part !== 'manifest' || result.offset !== 0)
+            throw invalid();
+        await retainManifest(directory, result);
+        const files = {};
+        for (const entry of manifest.entries) {
+            const path = join(directory, FILES[entry.part]), partial = path + '.partial';
+            const final = await lstat(path).catch(error => { if (error.code === 'ENOENT')
+                return null; throw error; });
+            if (final) {
+                const actual = await hashFile(path, entry.size);
+                if (actual.size !== entry.size || actual.sha256 !== entry.sha256)
+                    throw invalid();
+                files[entry.part] = path;
+                continue;
+            }
+            try {
+                const created = await open(partial, 'wx', 0o600);
+                await created.close();
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST')
+                    throw error;
+            }
+            const file = await privateFile(partial, true);
+            try {
+                let position = (await file.stat()).size;
+                if (position > entry.size)
+                    throw invalid();
+                while (position < entry.size) {
+                    signal.throwIfAborted();
+                    const length = Math.min(CHUNK, entry.size - position);
+                    const chunk = await read(job, signal, entry.part, position, length, result.manifest_sha256);
+                    checkManifest(chunk, job, result.manifest_sha256);
+                    if (chunk.part === 'manifest' || chunk.part !== entry.part || chunk.offset !== position || chunk.length !== length || typeof chunk.content_base64 !== 'string'
+                        || chunk.content_base64.length !== 4 * Math.ceil(length / 3) || !HASH.test(chunk.chunk_sha256 ?? ''))
+                        throw invalid();
+                    const bytes = Buffer.from(chunk.content_base64, 'base64');
+                    if (bytes.length !== length || bytes.toString('base64') !== chunk.content_base64 || sha(bytes) !== chunk.chunk_sha256)
+                        throw invalid();
+                    let written = 0;
+                    while (written < bytes.length) {
+                        const next = await file.write(bytes, written, bytes.length - written, position + written);
+                        if (!next.bytesWritten)
+                            throw invalid();
+                        written += next.bytesWritten;
+                    }
+                    await file.sync();
+                    position += length;
+                }
+            }
+            finally {
+                await file.close();
+            }
+            const actual = await hashFile(partial, entry.size);
+            if (actual.size !== entry.size || actual.sha256 !== entry.sha256)
+                throw invalid();
+            // Only fixed logical filenames are used. No extraction or project writes.
+            if (await lstat(path).then(() => true).catch(error => { if (error.code === 'ENOENT')
+                return false; throw error; }))
+                throw invalid();
+            await rename(partial, path);
+            files[entry.part] = path;
+        }
+        // Reject changed/withdrawn ownership after receiving the final bytes too.
+        const final = await read(job, signal, 'manifest');
+        checkManifest(final, job, result.manifest_sha256);
+        lock.assertHeld();
+        signal.throwIfAborted();
+        completed = true;
+        return { directory, manifest, manifestSha256: result.manifest_sha256, files };
+    }
+    finally {
+        try {
+            if (!completed)
+                await discardPrivateDataInputs(job);
+        }
+        finally {
+            await lock.close();
+        }
+    }
+}

package/dist/cli/src/runner-jobs.js

@@ -0,0 +1,49 @@
+import { localAssignmentsPage, readLocalClaim, changeLocalClaim } from './runner-claim-reads.js';
+import { CliError } from './http.js';
+import { selectedWorkspace } from './world-access.js';
+import { identifier } from './world-contracts.js';
+function describe(row) {
+    return `${row.job_id}  ${row.world_name}  ${row.purpose}  ${row.job_state}\n`
+        + (row.claim ? `Claim ${row.claim.request_key}: ${row.claim.state}. Cleanup ${row.claim.cleanup_recorded ? 'recorded' : 'not recorded'}.`
+            : 'No retained claim. A delayed ownership request may still arrive.')
+        + (row.input_preparation && ['pending', 'preparing', 'ready', 'failed'].includes(row.input_preparation.state)
+            ? `\nPackage inputs: ${row.input_preparation.state}.` + (row.input_preparation.state === 'failed' ? ' Review the source dependency locks. No target outcome is implied.' : '') : '');
+}
+export async function runnerJobs(action, device, job, request, options, output) {
+    device = identifier(device);
+    const scope = await selectedWorkspace({ workspace: options.workspace,
+        interactive: !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY,
+        openBrowser: options.openBrowser, signal: options.signal });
+    if (action === 'list') {
+        const page = options.page ?? '1';
+        if (!/^[1-9][0-9]*$/.test(page) || !Number.isSafeInteger(Number(page)))
+            throw new CliError('page_invalid', 'Use a positive page number.', 2);
+        const result = await localAssignmentsPage(scope, device, { page: Number(page), query: options.search }, options.signal);
+        output('local_job_directory', result, result.results.map(describe).join('\n\n') || 'No saved jobs assigned to this runner.');
+        return;
+    }
+    if (!job || !request)
+        throw new CliError('local_claim_identity_required', 'Use the exact device, job and claim request IDs.', 2);
+    job = identifier(job);
+    request = identifier(request);
+    let receipt;
+    if (action === 'withdraw') {
+        try {
+            receipt = await changeLocalClaim(scope, device, job, request, 'withdraw', undefined, options.signal);
+        }
+        catch {
+            // Reconcile the exact withdrawal after a lost reply. No new claim, job,
+            // attempt or resource is created by recovery.
+            receipt = await readLocalClaim(scope, device, job, request, options.signal);
+        }
+    }
+    else
+        receipt = await readLocalClaim(scope, device, job, request, options.signal);
+    if (receipt.job_id !== job || receipt.device_key !== device || receipt.claim && receipt.claim.request_key !== request
+        || !receipt.claim && receipt.request_key !== request)
+        throw new CliError('local_claim_receipt_invalid', 'The saved ownership receipt does not match this request.');
+    output('local_job_claim', receipt, describe(receipt));
+    if (action === 'withdraw' && receipt.claim?.state !== 'withdrawn') {
+        throw new CliError('local_claim_withdrawal_uncertain', 'Withdrawal is not confirmed. Inspect this exact claim before continuing.');
+    }
+}