firedrill-run 2.2.4 → 3.0.0

package/dist/cli/src/runner-trial-live.js

@@ -0,0 +1,393 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, mkdir, open, rename, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { api } from './auth.js';
+import { privateDirectory } from './credentials.js';
+import { CliError } from './http.js';
+import { runnerRequestSigner } from './runner-identity.js';
+import { identifier } from './world-contracts.js';
+import { localConfigurationIdentityMatches } from '../../contracts/local-configuration-contracts.js';
+import { localTrialLiveReceiptMatches } from '../../contracts/local-trial-live-contracts.js';
+import { assertEvidencePrivacy, evidenceNeedles } from './runner-evidence-privacy.js';
+import { inspectTrialArtifact, inspectTrialEvent } from './runner-trial-protocol.js';
+import { synchronizeTrialEvidence, trialTransferKey } from './runner-trial-transfer.js';
+import { assertEvidenceRetentionCurrent, evidenceRetentionSignal } from './runner-evidence-retention.js';
+const messages = { 'trial.preparing': 'Preparing the drill world',
+    'trial.world_unavailable': 'The drill world became unavailable', 'runner.started': 'Agent invocation observed',
+    'dependency.request': 'Service request recorded', 'dependency.response': 'Service response recorded',
+    'task.progress': 'Background task activity recorded',
+    'driver.ready': 'Input adapter ready', 'driver.ingress_observed': 'Input delivery observed',
+    'driver.action_started': 'Input action started', 'driver.action_finished': 'Input action finished',
+    'conversation.progress': 'Conversation activity recorded', 'voice.progress': 'Voice activity recorded',
+    'browser.capture_progress': 'Browser capture recorded', 'desktop.capture_progress': 'Desktop capture recorded',
+    'world.continuity.started': 'World window started', 'world.agent.observed': 'Original process observation recorded',
+    'world.interaction.observed': 'Agent input activity recorded',
+    'world.event.intent': 'World event pending', 'world.event.observed': 'World event result recorded',
+    'world.control.observed': 'World control result recorded', 'world.continuity.finished': 'World window finished',
+    'world.time.started': 'World schedule started', 'world.time.advanced': 'World schedule advanced',
+    'world.time.finished': 'World schedule stopped', 'trial.elapsed': 'Drill timing recorded' };
+const hash = (body) => createHash('sha256').update(body).digest('hex');
+const invalid = () => new CliError('local_trial_live_invalid', 'Live activity could not be retained safely. The exact execution was not restarted.');
+async function syncDirectory(directory) {
+    const file = await open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0));
+    try {
+        await file.sync();
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Private disk queue plus one caller-owned upload pump. Neither a new runner
+ * daemon nor execution authority. An uncertain POST gets one exact GET, never a
+ * write replay. Final collection/closing still validates the entire journal. */
+export async function createLiveTrialRetainer(owner, manifest, configuration, retention, signal, output) {
+    signal = evidenceRetentionSignal(retention, signal);
+    const control = configuration.control;
+    if (owner.purpose !== 'trial' || control.schema !== 'firedrill.trial.v1' || !localConfigurationIdentityMatches(configuration, owner)
+        || !/^[a-f0-9]{64}$/.test(manifest) || configuration.manifest_sha256 !== manifest
+        || control.job_id !== owner.job || control.operation_id !== owner.lease)
+        throw invalid();
+    const directory = join(await privateDirectory(), `trial-live-${identifier(owner.owner)}-${identifier(owner.workspace)}-${identifier(owner.lease)}`);
+    await mkdir(directory, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST')
+        throw error; });
+    const row = await lstat(directory);
+    if (!row.isDirectory() || row.isSymbolicLink() || row.mode & 0o077 || process.getuid && row.uid !== process.getuid())
+        throw invalid();
+    const blocked = evidenceNeedles(configuration);
+    const journal = { schema: 'firedrill.local-trial-live-journal.v2', job_id: owner.job,
+        attempt_id: owner.attempt, lease_id: owner.lease, execution_id: control.execution_id, manifest_sha256: manifest, retention,
+        artifact_count: 0, cloud_artifact_count: 0, last_cloud_artifact: null, artifact_bytes: 0,
+        queued: 0, cursor: 0, pending_transfer: null,
+        events: 0, cloud_events: 0, pending: null, state: owner.evidenceUpload ? 'listening' : 'local_only', problem: null,
+        provenance: 'customer_device', cloud_completion_recorded: false };
+    let writing = Promise.resolve(), enqueueing = Promise.resolve(), pump, finished = false, paused = !owner.evidenceUpload;
+    const persist = (initial = false) => {
+        assertEvidenceRetentionCurrent(retention);
+        const body = Buffer.from(JSON.stringify(journal));
+        assertEvidencePrivacy(body, blocked);
+        const next = writing.then(async () => {
+            assertEvidenceRetentionCurrent(retention);
+            const target = join(directory, 'receipt.json'), temporary = initial ? target : target + '.' + randomUUID() + '.tmp';
+            const file = await open(temporary, 'wx', 0o600);
+            try {
+                await file.writeFile(body);
+                await file.sync();
+            }
+            finally {
+                await file.close();
+            }
+            if (!initial)
+                await rename(temporary, target);
+            await syncDirectory(directory);
+        });
+        writing = next;
+        return next;
+    };
+    const privateFile = async (name, maximum) => {
+        assertEvidenceRetentionCurrent(retention);
+        const path = join(directory, name), before = await lstat(path);
+        if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.mode & 0o077 || before.size > maximum
+            || process.getuid && before.uid !== process.getuid())
+            throw invalid();
+        const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+        const actual = await file.stat();
+        if (actual.ino !== before.ino || actual.dev !== before.dev || actual.size !== before.size) {
+            await file.close();
+            throw invalid();
+        }
+        return file;
+    };
+    const read = async (sequence) => {
+        const file = await privateFile(`event-${sequence}.json`, 2 * 1024 ** 2);
+        try {
+            const actual = await file.stat();
+            const bytes = await file.readFile(), after = await file.stat();
+            if (after.size !== actual.size || after.mtimeMs !== actual.mtimeMs || after.ctimeMs !== actual.ctimeMs)
+                throw invalid();
+            return bytes;
+        }
+        finally {
+            await file.close();
+        }
+    };
+    const readMetadata = async (name, maximum) => {
+        const file = await privateFile(name, maximum);
+        try {
+            const before = await file.stat(), bytes = await file.readFile(), after = await file.stat();
+            if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
+                throw invalid();
+            return JSON.parse(bytes.toString('utf8'));
+        }
+        finally {
+            await file.close();
+        }
+    };
+    const enqueue = (item) => {
+        const next = enqueueing.then(async () => {
+            if (!Number.isSafeInteger(journal.queued + 1))
+                throw invalid();
+            const file = await open(join(directory, `queue-${journal.queued}.json`), 'wx', 0o600);
+            try {
+                await file.writeFile(JSON.stringify(item));
+                await file.sync();
+            }
+            finally {
+                await file.close();
+            }
+            await syncDirectory(directory);
+            journal.queued++;
+            await persist();
+        });
+        enqueueing = next;
+        return next;
+    };
+    const queued = async (position) => {
+        const value = await readMetadata(`queue-${position}.json`, 256);
+        if (!value || typeof value !== 'object' || Array.isArray(value))
+            throw invalid();
+        const item = value;
+        if (item.kind === 'event' && Object.keys(item).sort().join(',') === 'kind,sequence'
+            && Number.isSafeInteger(item.sequence) && item.sequence === journal.cloud_events && Number(item.sequence) < journal.events) {
+            return { kind: 'event', sequence: Number(item.sequence) };
+        }
+        if (item.kind === 'artifact' && Object.keys(item).sort().join(',') === 'kind,position'
+            && Number.isSafeInteger(item.position) && item.position === journal.cloud_artifact_count && Number(item.position) < journal.artifact_count) {
+            return { kind: 'artifact', position: Number(item.position) };
+        }
+        throw invalid();
+    };
+    const match = (value, sequence, body) => {
+        if (!localTrialLiveReceiptMatches(value, owner, { manifestSha256: manifest, executionId: control.execution_id,
+            eventId: createHash('sha256').update(`trial:${owner.attempt}:${sequence}`).digest('hex'),
+            sequence, bodySha256: hash(body), bodySize: body.length }))
+            throw invalid();
+    };
+    const upload = async (sequence, bytes) => {
+        assertEvidenceRetentionCurrent(retention);
+        const active = AbortSignal.any([signal, AbortSignal.timeout(10000)]);
+        const auth = { actor: owner.owner, workspace: owner.workspace, signal: active };
+        const path = `/api/workspaces/${identifier(owner.workspace)}/local-runners/${identifier(owner.device)}/jobs/${identifier(owner.job)}/claims/${identifier(owner.request)}/trial-events/${sequence}`;
+        let receipt = await api(path, auth);
+        match(receipt, sequence, bytes);
+        if (receipt.state === 'retained')
+            return;
+        const signer = await runnerRequestSigner(owner, owner.device, active);
+        journal.pending = { sequence, sha256: hash(bytes), size: bytes.length };
+        journal.state = 'cloud_pending';
+        await persist(); // Exact bytes and write intent are durable before dispatch.
+        const payload = { schema: 'firedrill.local-trial-live-write.v1', job_id: owner.job,
+            attempt_id: owner.attempt, lease_id: owner.lease, request_key: owner.request, device_key: owner.device,
+            native_session_id: owner.nativeSession, manifest_sha256: manifest, sequence,
+            sent_at_ms: Date.parse(receipt.server_time), event_json: bytes.toString('ascii') };
+        const proof = await signer(payload);
+        assertEvidenceRetentionCurrent(retention);
+        try {
+            receipt = await api(path, { ...auth, method: 'POST', body: proof });
+        }
+        catch (error) {
+            if (!(error instanceof CliError) || !['connection_uncertain', 'response_unavailable'].includes(error.code))
+                throw error;
+            receipt = await api(path, { ...auth, signal: AbortSignal.any([signal, AbortSignal.timeout(5000)]) });
+        }
+        match(receipt, sequence, bytes);
+        if (receipt.state !== 'retained')
+            throw invalid();
+    };
+    const startPump = () => {
+        if (finished || paused || pump)
+            return;
+        pump = (async () => {
+            try {
+                while (!finished && journal.cursor < journal.queued) {
+                    signal.throwIfAborted();
+                    const item = await queued(journal.cursor);
+                    if (item.kind === 'artifact') {
+                        const description = await readMetadata(`${item.position}.bin.json`, 8192);
+                        inspectTrialArtifact(description, control.execution_id, control.scenario_key);
+                        if (description.id !== item.position + '.bin' || description.kind === 'result')
+                            throw invalid();
+                        const metadata = description, file = await privateFile(metadata.id, 512 * 1024 ** 2);
+                        try {
+                            const before = await file.stat();
+                            if (before.size !== metadata.size)
+                                throw invalid();
+                            const key = trialTransferKey(owner.lease, 'live-artifact:' + metadata.id);
+                            const retained = await synchronizeTrialEvidence(owner, manifest, key, { kind: 'trial_live_artifact', receipt: { size: metadata.size, sha256: metadata.sha256 }, metadata }, async (offset, length) => {
+                                assertEvidenceRetentionCurrent(retention);
+                                const bytes = Buffer.alloc(length);
+                                let position = 0;
+                                while (position < length) {
+                                    const next = await file.read(bytes, position, length - position, offset + position);
+                                    if (!next.bytesRead)
+                                        throw invalid();
+                                    position += next.bytesRead;
+                                }
+                                const after = await file.stat();
+                                if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
+                                    throw invalid();
+                                assertEvidencePrivacy(bytes, blocked);
+                                return bytes;
+                            }, AbortSignal.any([signal, AbortSignal.timeout(60000)]), async (value) => { journal.pending_transfer = value; journal.state = 'cloud_pending'; await persist(); });
+                            journal.last_cloud_artifact = { id: metadata.id, receipt: retained };
+                            journal.cloud_artifact_count++;
+                            journal.cursor++;
+                            journal.state = 'listening';
+                            await persist();
+                            output?.('local_trial_file', { job_id: owner.job, execution_id: control.execution_id, kind: metadata.kind,
+                                artifact_id: retained.id, cloud_retained: true, provenance: 'customer_device', cloud_completion_recorded: false }, `${owner.job.slice(0, 8)}: Evidence file saved.`);
+                        }
+                        finally {
+                            await file.close();
+                        }
+                        continue;
+                    }
+                    const sequence = item.sequence, bytes = await read(sequence);
+                    if (bytes.length > 256 * 1024) {
+                        journal.problem = 'live_envelope_capacity';
+                        throw invalid();
+                    }
+                    await upload(sequence, bytes);
+                    journal.cloud_events++;
+                    journal.cursor++;
+                    journal.pending = null;
+                    journal.state = 'listening';
+                    await persist();
+                    const event = inspectTrialEvent(JSON.parse(bytes.toString('ascii')), control.execution_id, sequence);
+                    if (event.kind === 'trial.execution_receipt')
+                        throw invalid();
+                    output?.('local_trial_activity', { job_id: owner.job, execution_id: control.execution_id,
+                        sequence, kind: event.kind, observed_at: event.at, cloud_retained: true, provenance: 'customer_device' }, `${owner.job.slice(0, 8)}: ${messages[event.kind] ?? 'Drill activity recorded'}.`);
+                }
+            }
+            catch {
+                // Do not restart/retry target or live writes on a transport problem.
+                // Complete evidence can still use the existing post-exit sync path.
+                paused = true;
+                journal.state = 'cloud_paused';
+                journal.problem ??= 'live_transfer_unconfirmed';
+                await persist();
+                output?.('local_trial_live_paused', { job_id: owner.job, execution_id: control.execution_id,
+                    events_locally_retained: journal.events, events_cloud_retained: journal.cloud_events,
+                    problem: journal.problem, cloud_completion_recorded: false }, `${owner.job.slice(0, 8)}: Live cloud updates paused. Local evidence collection continues.`);
+            }
+        })().finally(() => {
+            pump = undefined;
+            if (!finished && !paused && journal.cursor < journal.queued)
+                startPump();
+        });
+        void pump.catch(() => undefined); // Disk errors are awaited again at finish.
+    };
+    try {
+        await persist(true);
+    }
+    catch (error) {
+        for (const value of blocked)
+            value.fill(0);
+        throw error;
+    }
+    return { accept: async (body, event) => {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(retention);
+            if (finished || event.sequence !== journal.events || event.execution_id !== control.execution_id
+                || !body.length || body.length > 2 * 1024 ** 2 || body.some(value => value > 127))
+                throw invalid();
+            const original = inspectTrialEvent(JSON.parse(body.toString('ascii')), control.execution_id, journal.events);
+            if (original.kind !== event.kind || original.at !== event.at || original.kind === 'trial.execution_receipt')
+                throw invalid();
+            assertEvidencePrivacy(body, blocked);
+            const file = await open(join(directory, `event-${event.sequence}.json`), 'wx', 0o600);
+            try {
+                await file.writeFile(body);
+                await file.sync();
+            }
+            finally {
+                await file.close();
+            }
+            await syncDirectory(directory);
+            journal.events++;
+            await enqueue({ kind: 'event', sequence: event.sequence });
+            startPump();
+        }, artifact: async (metadata) => {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(retention);
+            inspectTrialArtifact(metadata, control.execution_id, control.scenario_key);
+            if (finished || metadata.kind === 'result' || metadata.execution_id !== control.execution_id
+                || metadata.scenario_key !== control.scenario_key || metadata.id !== journal.artifact_count + '.bin'
+                || journal.artifact_bytes + metadata.size > 512 * 1024 ** 2)
+                throw invalid();
+            metadata = { ...metadata };
+            const partial = join(directory, metadata.id + '.partial'), file = await open(partial, 'wx', 0o600);
+            const overlap = blocked.reduce((maximum, value) => Math.max(maximum, value.length), 512), digest = createHash('sha256');
+            let tail = Buffer.alloc(0), position = 0, closed = false, committed = false;
+            const close = async () => { if (!closed) {
+                closed = true;
+                await file.close();
+            } tail.fill(0); };
+            return { write: async (bytes) => {
+                    signal.throwIfAborted();
+                    assertEvidenceRetentionCurrent(retention);
+                    if (closed || !bytes.length || bytes.length > 16384 || position + bytes.length > metadata.size)
+                        throw invalid();
+                    const scan = Buffer.concat([tail, bytes]);
+                    assertEvidencePrivacy(scan, blocked);
+                    tail.fill(0);
+                    tail = Buffer.from(scan.subarray(Math.max(0, scan.length - overlap + 1)));
+                    digest.update(bytes);
+                    let written = 0;
+                    while (written < bytes.length) {
+                        const next = await file.write(bytes, written, bytes.length - written);
+                        if (!next.bytesWritten)
+                            throw invalid();
+                        written += next.bytesWritten;
+                    }
+                    position += bytes.length;
+                }, commit: async () => {
+                    assertEvidenceRetentionCurrent(retention);
+                    if (closed || position !== metadata.size || digest.digest('hex') !== metadata.sha256)
+                        throw invalid();
+                    await file.sync();
+                    await close();
+                    await rename(partial, join(directory, metadata.id));
+                    await syncDirectory(directory);
+                    const description = Buffer.from(JSON.stringify(metadata));
+                    assertEvidencePrivacy(description, blocked);
+                    const record = await open(join(directory, metadata.id + '.json'), 'wx', 0o600);
+                    try {
+                        await record.writeFile(description);
+                        await record.sync();
+                    }
+                    finally {
+                        await record.close();
+                    }
+                    await syncDirectory(directory);
+                    const artifactPosition = journal.artifact_count;
+                    journal.artifact_count++;
+                    journal.artifact_bytes += metadata.size;
+                    await enqueue({ kind: 'artifact', position: artifactPosition });
+                    committed = true;
+                    startPump();
+                }, abort: async () => { await close(); if (!committed)
+                    await unlink(partial).catch(() => undefined); } };
+        }, finish: async () => {
+            finished = true;
+            try {
+                // Do not drain a backlog here: closing must not wait for bulk uploads.
+                // Await only the bounded in-flight exchange; the original final journal
+                // sync retains every remaining event after cleanup/closing as before.
+                await enqueueing;
+                if (pump)
+                    await pump;
+                journal.state = paused ? owner.evidenceUpload ? 'cloud_paused' : 'local_only' : 'closed';
+                await persist();
+                return { events_locally_retained: journal.events, events_cloud_retained: journal.cloud_events,
+                    files_locally_retained: journal.artifact_count, files_cloud_retained: journal.cloud_artifact_count,
+                    state: journal.state, problem: journal.problem, provenance: 'customer_device', cloud_completion_recorded: false };
+            }
+            finally {
+                for (const value of blocked)
+                    value.fill(0);
+                blocked.length = 0;
+            }
+        } };
+}