firedrill-run 2.2.4 → 3.0.0

package/dist/cli/src/runner-ownership.js

@@ -0,0 +1,332 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, open, rename } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { readLocalClaim, changeLocalClaim } from './runner-claim-reads.js';
+import { privateDirectory } from './credentials.js';
+import { processLifetimeLock } from './runner-process-lock.js';
+import { CliError } from './http.js';
+import { executionPermission } from './runner-grants.js';
+import { receiveLocalInputs, discardPrivateDataInputs } from './runner-inputs.js';
+import { clearRuntimeConfiguration, receiveRuntimeConfiguration } from './runner-configuration.js';
+import { signRunnerRequest } from './runner-identity.js';
+import { identifier } from './world-contracts.js';
+function canonical(value) {
+    if (Array.isArray(value))
+        return '[' + value.map(canonical).join(',') + ']';
+    if (value && typeof value === 'object') {
+        const row = value;
+        return '{' + Object.keys(row).sort().map(key => JSON.stringify(key) + ':' + canonical(row[key])).join(',') + '}';
+    }
+    const body = JSON.stringify(value);
+    if (body === undefined)
+        throw new CliError('local_owner_journal_invalid', 'The retained ownership record is incomplete.');
+    return body;
+}
+const digest = (value) => createHash('sha256').update(canonical(value)).digest('hex');
+const invalid = () => new CliError('local_owner_identity_changed', 'Local ownership does not match the exact retained job. No replacement claim or launch was sent.');
+const pathFor = async (scope, job) => join(await privateDirectory(), `job-owner-${scope.owner}-${scope.workspace}-${identifier(job)}.json`);
+const writes = new Map();
+async function persist(value, initial = false) {
+    const path = await pathFor({ ...value, prefix: '' }, value.job);
+    const body = JSON.stringify(value);
+    // Input progress and the lease keeper share this journal. Preserve dispatch
+    // ordering even while a file sync is pending; never clear an uncertain pulse
+    // with an older concurrent stage snapshot.
+    const write = (writes.get(path) ?? Promise.resolve()).then(async () => {
+        if (initial) {
+            const file = await open(path, 'wx', 0o600);
+            try {
+                await file.writeFile(body);
+                await file.sync();
+            }
+            finally {
+                await file.close();
+            }
+            const directory = await open(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY | (constants.O_NOFOLLOW ?? 0));
+            try {
+                await directory.sync();
+            }
+            finally {
+                await directory.close();
+            }
+            return;
+        }
+        const temporary = path + '.' + randomUUID() + '.tmp';
+        const file = await open(temporary, 'wx', 0o600);
+        try {
+            await file.writeFile(body);
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+        await rename(temporary, path);
+        const directory = await open(dirname(path), constants.O_RDONLY | constants.O_DIRECTORY | (constants.O_NOFOLLOW ?? 0));
+        try {
+            await directory.sync();
+        }
+        finally {
+            await directory.close();
+        }
+    });
+    writes.set(path, write);
+    try {
+        await write;
+    }
+    finally {
+        if (writes.get(path) === write)
+            writes.delete(path);
+    }
+}
+async function load(context, job) {
+    const path = await pathFor(context, job);
+    const before = await lstat(path).catch(error => { if (error.code === 'ENOENT')
+        return null; throw error; });
+    if (!before)
+        return null;
+    if (!before.isFile() || before.isSymbolicLink() || before.mode & 0o077 || before.nlink !== 1 || before.size > 32768
+        || process.getuid && before.uid !== process.getuid())
+        throw invalid();
+    const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    try {
+        const actual = await file.stat();
+        if (actual.ino !== before.ino || actual.dev !== before.dev || actual.size !== before.size)
+            throw invalid();
+        const value = JSON.parse(await file.readFile('utf8'));
+        if (value.schema !== 'firedrill.local-job-owner.v1' || value.owner !== context.owner || value.workspace !== context.workspace
+            || value.world !== context.world || value.job !== job || !['claim_dispatched', 'claimed', 'inputs_received', 'work_dispatched', 'not_dispatched'].includes(value.stage)
+            || value.proof?.payload?.schema !== 'firedrill.local-job-claim.v1' || value.proof.payload.request_key !== value.request
+            || value.proof.payload.job_id !== job || value.proof.payload.device_key !== value.device || value.proof.payload.grant_id !== value.grant
+            || value.runtimeImage !== undefined && !/^[a-z][a-z0-9-]{0,62}-docker\.pkg\.dev\/[a-z][a-z0-9-]{4,62}\/[a-z][a-z0-9-]{0,62}\/[a-z][a-z0-9-]{0,62}@sha256:[a-f0-9]{64}$/.test(value.runtimeImage)
+            || typeof value.proof.signature !== 'string' || value.proof.signature.length !== 88)
+            throw invalid();
+        for (const id of [value.request, value.device, value.grant, value.grantRequest])
+            identifier(id);
+        if (value.lease !== null)
+            identifier(value.lease);
+        if (value.attempt !== null)
+            identifier(value.attempt);
+        return value;
+    }
+    catch {
+        throw invalid();
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Read only the original local intent. Never invent a replacement lookup key. */
+export async function retainedLocalClaim(context, device, job) {
+    const value = await load(context, identifier(job));
+    if (value && value.device !== identifier(device))
+        throw invalid();
+    return value;
+}
+/** A durable fence under the same lock as execution, not an expired-lease guess.
+ * The caller must also exclude a resource journal before accepting the fence.
+ */
+export async function withUnstartedLocalClaim(context, device, job, signal, work) {
+    job = identifier(job);
+    device = identifier(device);
+    const lock = await processLifetimeLock(join(await privateDirectory(), `job-lock-${context.owner}-${context.workspace}-${job}`));
+    try {
+        const active = AbortSignal.any([signal, lock.signal]);
+        lock.assertHeld();
+        active.throwIfAborted();
+        const value = await load(context, job);
+        if (!value || value.device !== device)
+            throw invalid();
+        if (value.stage === 'work_dispatched')
+            throw new CliError('local_resource_recovery_required', 'This job handed off to a container. Use runner cleanup and evidence sync for the original operation; it cannot be abandoned as unstarted.');
+        const receipt = await readLocalClaim(value, device, job, value.request, active);
+        match(value, receipt, false);
+        if (!receipt.claim?.attempt_id)
+            throw new CliError('local_claim_unconfirmed', 'The original claim has no confirmed attempt. Keep its journal and inspect the same request later; it was not repeated.');
+        value.lease = receipt.claim.id;
+        value.attempt = receipt.claim.attempt_id;
+        return await work(value, receipt, async () => { lock.assertHeld(); value.stage = 'not_dispatched'; await persist(value); }, active);
+    }
+    finally {
+        await lock.close();
+    }
+}
+function match(value, receipt, active = true) {
+    if (receipt.actor_id !== value.owner || receipt.workspace_id !== value.workspace || receipt.native_session_id !== value.proof.payload.native_session_id
+        || receipt.job_id !== value.job || receipt.world_id !== value.world || receipt.grant_id !== value.grant
+        || receipt.device_key !== value.device || receipt.payload_digest !== value.proof.payload.payload_digest)
+        throw invalid();
+    if (!receipt.claim) {
+        if (active || receipt.request_key !== value.request)
+            throw invalid();
+        return;
+    }
+    const claim = receipt.claim;
+    if (claim.request_key !== value.request || claim.request_digest !== digest(value.proof.payload)
+        || value.lease && claim.id !== value.lease || value.attempt && claim.attempt_id !== value.attempt)
+        throw invalid();
+    identifier(claim.id);
+    if (claim.attempt_id)
+        identifier(claim.attempt_id);
+    if (active && (claim.state !== 'claimed' || !claim.attempt_id || !Number.isSafeInteger(Date.parse(receipt.server_time))
+        || !Number.isFinite(Date.parse(claim.expires_at)) || Date.parse(claim.expires_at) <= Date.parse(receipt.server_time))) {
+        throw new CliError('local_claim_expired', 'This claim is no longer active. Use runner abandon <device> <job> for an unstarted claim, or runner cleanup for its original container. It was not replaced.');
+    }
+}
+async function pulse(value, signal) {
+    const receipt = await readLocalClaim(value, value.device, value.job, value.request, signal);
+    match(value, receipt);
+    if (!receipt.claim || !Number.isSafeInteger(receipt.claim.last_sequence) || receipt.claim.last_sequence < 0
+        || receipt.claim.last_sequence >= Number.MAX_SAFE_INTEGER)
+        throw invalid();
+    if (value.pending) {
+        const pending = value.pending.payload;
+        if (pending.schema !== 'firedrill.local-job-presence.v1' || pending.lease_id !== value.lease
+            || pending.request_key !== value.request || pending.job_id !== value.job || pending.device_key !== value.device)
+            throw invalid();
+        if (receipt.claim.last_sequence !== pending.sequence || receipt.claim.last_pulse_digest !== digest(pending)) {
+            throw new CliError('local_claim_pulse_uncertain', 'The previous ownership pulse is unresolved. No pulse or work was replayed.');
+        }
+        value.pending = null;
+        await persist(value);
+    }
+    const sent = Date.parse(receipt.server_time);
+    if (!Number.isSafeInteger(sent))
+        throw invalid();
+    const nativeSession = value.proof.payload.native_session_id;
+    if (typeof nativeSession !== 'string')
+        throw invalid();
+    const payload = { schema: 'firedrill.local-job-presence.v1',
+        job_id: value.job, device_key: value.device, request_key: value.request,
+        native_session_id: identifier(nativeSession), lease_id: identifier(value.lease),
+        sent_at_ms: sent, sequence: receipt.claim.last_sequence + 1, state: 'claimed' };
+    value.pending = await signRunnerRequest(value, value.device, payload, signal);
+    await persist(value);
+    const result = await changeLocalClaim(value, value.device, value.job, value.request, 'presence', value.pending, signal);
+    match(value, result);
+    if (!result.claim || result.claim.last_sequence !== value.pending.payload.sequence || result.claim.last_pulse_digest !== digest(value.pending.payload))
+        throw invalid();
+    value.pending = null;
+    await persist(value);
+}
+/** Internal executor seam: exact claim -> leased input transfer -> owned work.
+ * No CLI action is advertised until the container/model/evidence path is wired.
+ * The caller owns actual resource cleanup; a returned value is NOT its proof.
+ */
+export async function withLocalJobInputs(context, device, job, grantRequest, purpose, signal, work) {
+    device = identifier(device);
+    job = identifier(job);
+    grantRequest = identifier(grantRequest);
+    for (const id of [context.owner, context.workspace, context.world])
+        identifier(id);
+    const lock = await processLifetimeLock(join(await privateDirectory(), `job-lock-${context.owner}-${context.workspace}-${job}`));
+    signal = AbortSignal.any([signal, lock.signal]);
+    try {
+        lock.assertHeld();
+        signal.throwIfAborted();
+        const permission = await executionPermission(context, grantRequest, device, purpose, signal);
+        let value = await load(context, job);
+        if (value?.stage === 'work_dispatched' || value?.stage === 'not_dispatched') {
+            throw new CliError('local_execution_recovery_required', 'This job already dispatched owned work. Reconcile the original resources; the callback and target are not replayed.');
+        }
+        if (!value) {
+            const request = randomUUID();
+            const offered = await readLocalClaim(context, device, job, request, signal);
+            if (offered.claim || offered.job_id !== job || offered.world_id !== context.world || offered.device_key !== device
+                || offered.grant_id !== permission.id || offered.purpose !== purpose || !/^[a-f0-9]{64}$/.test(offered.payload_digest)
+                || !Number.isSafeInteger(Date.parse(offered.server_time)))
+                throw invalid();
+            const payload = { schema: 'firedrill.local-job-claim.v1', request_key: request,
+                job_id: job, device_key: device, native_session_id: permission.document.native_session_id,
+                grant_id: permission.id, payload_digest: offered.payload_digest, sent_at_ms: Date.parse(offered.server_time) };
+            const proof = await signRunnerRequest(context, device, payload, signal);
+            value = { schema: 'firedrill.local-job-owner.v1', owner: context.owner, workspace: context.workspace, world: context.world,
+                job, device, request, grantRequest, grant: permission.id, proof, runtimeImage: permission.document.runtime.image,
+                stage: 'claim_dispatched', lease: null, attempt: null, pending: null };
+            await persist(value, true);
+            // Exactly one dispatch. An ambiguous reply is recovered only by GET of
+            // this original request; never another PUT/new claim or target retry.
+            try {
+                match(value, await changeLocalClaim(value, device, job, request, 'claim', proof, signal));
+            }
+            catch {
+                const retained = await readLocalClaim(value, device, job, request, signal);
+                match(value, retained, false);
+                if (!retained.claim)
+                    throw new CliError('local_claim_unconfirmed', 'The saved claim is not confirmed. Keep this journal and inspect its exact request before continuing.');
+                match(value, retained);
+            }
+        }
+        if (value.device !== device || value.grant !== permission.id || value.grantRequest !== grantRequest
+            || value.runtimeImage !== undefined && value.runtimeImage !== permission.document.runtime.image
+            || value.proof.payload.native_session_id !== permission.document.native_session_id)
+            throw invalid();
+        const receipt = await readLocalClaim(value, value.device, job, value.request, signal);
+        match(value, receipt);
+        if (!receipt.claim?.attempt_id || receipt.purpose !== purpose)
+            throw invalid();
+        value.lease = receipt.claim.id;
+        value.attempt = receipt.claim.attempt_id;
+        value.stage = 'claimed';
+        await persist(value);
+        const common = { owner: value.owner, workspace: value.workspace, device, job, request: value.request,
+            lease: value.lease, attempt: value.attempt, nativeSession: permission.document.native_session_id,
+            grant: value.grant, payloadDigest: String(value.proof.payload.payload_digest), runtime: permission.document.runtime,
+            evidenceUpload: permission.document.consent.evidence_upload,
+            realAccountAccess: permission.document.consent.real_account_access,
+            catalogPackagesToDevice: permission.document.consent.catalog_packages_to_device,
+            world: context.world, configurationDigest: permission.document.basis.configuration_digest,
+            sourceSnapshot: permission.document.basis.source_snapshot_id, sourceRevision: permission.document.basis.source_revision,
+            limits: permission.document.limits,
+            resourceJournalRetained: async () => { lock.assertHeld(); combined.throwIfAborted(); value.stage = 'work_dispatched'; await persist(value); } };
+        let owner;
+        if (purpose === 'world_session') {
+            const session = receipt.world_session;
+            if (!session || Object.keys(session).sort().join(',') !== 'configuration_id,expires_at,input_sha256,session_id'
+                || identifier(session.session_id) !== session.session_id || identifier(session.configuration_id) !== session.configuration_id
+                || !/^[a-f0-9]{64}$/.test(session.input_sha256) || !Number.isSafeInteger(Date.parse(session.expires_at))
+                || Date.parse(session.expires_at) <= Date.parse(receipt.server_time) || !permission.document.consent.evidence_upload)
+                throw invalid();
+            owner = { ...common, purpose, session };
+        }
+        else
+            owner = { ...common, purpose };
+        const stop = new AbortController(), combined = AbortSignal.any([signal, stop.signal]);
+        let failed;
+        // Refresh once BEFORE work, then maintain original sequence ownership.
+        await pulse(value, signal);
+        const keeping = (async () => {
+            while (!combined.aborted) {
+                await sleep(10000, undefined, { signal: combined });
+                await pulse(value, combined);
+            }
+        })().catch(error => { if (!combined.aborted) {
+            failed = error;
+            stop.abort();
+        } });
+        let configuration = null;
+        try {
+            const inputs = await receiveLocalInputs(owner, combined);
+            value.stage = 'inputs_received';
+            await persist(value);
+            configuration = await receiveRuntimeConfiguration(owner, inputs, combined);
+            combined.throwIfAborted();
+            const result = await work(owner, inputs, configuration, combined);
+            if (failed)
+                throw failed;
+            lock.assertHeld();
+            combined.throwIfAborted();
+            return result;
+        }
+        finally {
+            stop.abort();
+            await keeping;
+            clearRuntimeConfiguration(configuration);
+            await discardPrivateDataInputs(owner);
+        }
+    }
+    finally {
+        await lock.close();
+    }
+}

package/dist/cli/src/runner-process-lock.js

@@ -0,0 +1,175 @@
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { controlPython } from './source-runtime.js';
+import { CliError } from './http.js';
+/** Fixed isolated stdlib only. The lock inode is never unlinked/replaced. The
+ * kernel releases flock when this broker exits; parent death closes its stdin.
+ * No PID-based takeover, stale timer, target import or provider operation. */
+const BROKER = String.raw `
+import fcntl, json, os, stat, sys
+path, nonce = sys.argv[1:]
+parent, name = os.path.dirname(path), os.path.basename(path)
+directory = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
+try:
+    base = os.fstat(directory)
+    if base.st_uid != os.getuid() or base.st_mode & 0o077:
+        raise ValueError()
+    fd = os.open(name, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600, dir_fd=directory)
+    try:
+        value = os.fstat(fd)
+        if not stat.S_ISREG(value.st_mode) or value.st_nlink != 1 or value.st_uid != os.getuid() or value.st_mode & 0o077:
+            raise ValueError()
+        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        current = os.stat(name, dir_fd=directory, follow_symlinks=False)
+        if (current.st_dev, current.st_ino) != (value.st_dev, value.st_ino):
+            raise ValueError()
+        body = json.dumps({'schema': 'firedrill.process-lock.v1', 'nonce': nonce, 'pid': os.getpid()}).encode()
+        os.ftruncate(fd, 0)
+        os.write(fd, body)
+        os.fsync(fd)
+        print(json.dumps({'state': 'locked', 'nonce': nonce}), flush=True)
+        if sys.stdin.buffer.read(1):
+            raise ValueError()
+    finally:
+        os.close(fd)
+finally:
+    os.close(directory)
+`;
+const unavailable = () => new CliError('local_operation_owned', 'This exact local operation is active, its legacy lock needs inspection, or the trusted offline lock helper is unavailable. No lock was stolen or cleanup inferred.');
+export async function processLifetimeLock(path) {
+    const controller = new AbortController(), nonce = randomUUID();
+    const runtime = await controlPython(AbortSignal.timeout(10000));
+    const child = spawn(runtime, ['-I', '-B', '-c', BROKER, path, nonce], {
+        shell: false, windowsHide: true, env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    let held = false, closing = false, ended = false, transportInvalid = false;
+    let failStartup = () => { };
+    let rejectTransport = () => { };
+    const transportFailure = new Promise((_resolve, reject) => { rejectTransport = reject; });
+    void transportFailure.catch(() => undefined);
+    const invalidateTransport = () => {
+        transportInvalid = true;
+        held = false;
+        controller.abort();
+        rejectTransport(unavailable());
+        failStartup();
+    };
+    // Pipe EPIPE/errors are stream events, not necessarily child-process events.
+    // Keep listeners through teardown and never expose transport/provider text.
+    child.stdin.on('error', invalidateTransport);
+    child.stdout.on('error', invalidateTransport);
+    child.stderr.on('error', invalidateTransport);
+    child.stderr.resume();
+    const exited = new Promise((resolve, reject) => {
+        child.once('error', () => { ended = true; held = false; controller.abort(); reject(unavailable()); });
+        child.once('close', code => {
+            ended = true;
+            held = false;
+            if (!closing || code !== 0 || transportInvalid) {
+                controller.abort();
+                reject(unavailable());
+            }
+            else
+                resolve();
+        });
+    });
+    void exited.catch(() => undefined);
+    let closePromise;
+    const stop = () => {
+        if (closePromise)
+            return closePromise;
+        closing = true;
+        closePromise = (async () => {
+            let kill, deadline;
+            try {
+                const timeout = new Promise((_resolve, reject) => {
+                    kill = setTimeout(() => child.kill('SIGKILL'), 2000);
+                    deadline = setTimeout(() => reject(unavailable()), 3000);
+                });
+                try {
+                    child.stdin.end();
+                }
+                catch {
+                    invalidateTransport();
+                }
+                await Promise.race([exited, transportFailure, timeout]);
+            }
+            finally {
+                if (kill)
+                    clearTimeout(kill);
+                if (deadline)
+                    clearTimeout(deadline);
+                if (transportInvalid && !ended)
+                    child.kill('SIGKILL');
+            }
+        })();
+        return closePromise;
+    };
+    try {
+        await new Promise((resolve, reject) => {
+            let body = '', settled = false;
+            const finish = (error) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                if (error)
+                    reject(error);
+                else
+                    resolve();
+            };
+            const timer = setTimeout(() => finish(unavailable()), 10000);
+            failStartup = () => finish(unavailable());
+            if (transportInvalid)
+                return finish(unavailable());
+            child.stdout.on('data', (part) => {
+                if (transportInvalid)
+                    return finish(unavailable());
+                body += part.toString('utf8');
+                if (body.length > 1024)
+                    return finish(unavailable());
+                if (!body.endsWith('\n'))
+                    return;
+                try {
+                    const value = JSON.parse(body);
+                    if (!value || Object.keys(value).sort().join(',') !== 'nonce,state' || value.state !== 'locked' || value.nonce !== nonce)
+                        throw new Error();
+                    held = true;
+                    finish();
+                }
+                catch {
+                    finish(unavailable());
+                }
+            });
+            child.once('error', () => finish(unavailable()));
+            child.once('close', () => { if (!held)
+                finish(unavailable()); });
+        });
+    }
+    catch (error) {
+        child.kill('SIGTERM');
+        await stop().catch(() => undefined);
+        throw error;
+    }
+    return { signal: controller.signal, assertHeld: () => { if (!held || ended || transportInvalid)
+            throw unavailable(); }, close: stop };
+}
+/** Release-only coordination. Reacquiring never authorizes replaying the work
+ * whose immutable original journal is inspected inside the callback. A legacy
+ * directory or unexpected file fails the native owner's existing validation.
+ */
+export async function withProcessLifetimeLock(path, signal, work) {
+    signal.throwIfAborted();
+    const lock = await processLifetimeLock(path), active = AbortSignal.any([signal, lock.signal]);
+    try {
+        lock.assertHeld();
+        active.throwIfAborted();
+        const value = await work(active, lock.signal);
+        lock.assertHeld();
+        active.throwIfAborted();
+        return value;
+    }
+    finally {
+        await lock.close();
+    }
+}