firedrill-run 2.2.4 → 3.0.0

package/dist/cli/src/runner-trial-journal.js

@@ -0,0 +1,358 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, mkdir, open, rename, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { privateDirectory } from './credentials.js';
+import { CliError } from './http.js';
+import { identifier } from './world-contracts.js';
+import { executeLocalTrial } from './runner-container.js';
+import { inspectTrialReport, trialReportBytes, trialStatus } from './runner-trial-protocol.js';
+import { withLocalJobInputs } from './runner-ownership.js';
+import { assertEvidencePrivacy, evidenceNeedles } from './runner-evidence-privacy.js';
+import { createLiveTrialRetainer } from './runner-trial-live.js';
+import { acquireLocalEvidence, assertEvidenceRetentionCurrent, evidenceRetentionSignal, frozenEvidenceRetention, retentionMatches } from './runner-evidence-retention.js';
+const hash = (body) => createHash('sha256').update(body).digest('hex');
+const fail = () => new CliError('local_trial_journal_invalid', 'The private drill journal is incomplete. No target was restarted or cloud result claimed.');
+async function folder(owner, create = true) {
+    const directory = join(await privateDirectory(), `trial-${identifier(owner.owner)}-${identifier(owner.workspace)}-${identifier(owner.lease)}`);
+    if (create)
+        await mkdir(directory, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST')
+            throw error; });
+    const row = await lstat(directory);
+    if (!row.isDirectory() || row.isSymbolicLink() || row.mode & 0o077 || process.getuid && row.uid !== process.getuid())
+        throw fail();
+    return directory;
+}
+async function syncDirectory(directory) {
+    const file = await open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0));
+    try {
+        await file.sync();
+    }
+    finally {
+        await file.close();
+    }
+}
+async function write(directory, name, body, replace = false) {
+    const path = join(directory, name), pending = replace ? path + '.' + randomUUID() + '.tmp' : path;
+    const file = await open(pending, 'wx', 0o600);
+    try {
+        await file.writeFile(body);
+        await file.sync();
+    }
+    finally {
+        await file.close();
+    }
+    if (replace)
+        await rename(pending, path);
+    await syncDirectory(directory);
+}
+async function retainArtifact(directory, metadata, reader, blocked, retention, signal) {
+    assertEvidenceRetentionCurrent(retention);
+    const exported = await reader.artifact(metadata), partial = join(directory, metadata.id + '.partial');
+    const file = await open(partial, 'wx', 0o600), digest = createHash('sha256');
+    const overlap = blocked.reduce((maximum, value) => Math.max(maximum, value.length), 512);
+    let position = 0, tail = Buffer.alloc(0), accepted = false;
+    try {
+        while (position < metadata.size) {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(retention);
+            const bytes = await exported.read(position, Math.min(16384, metadata.size - position));
+            assertEvidenceRetentionCurrent(retention);
+            if (!bytes.length || bytes.length > 16384 || position + bytes.length > metadata.size)
+                throw fail();
+            const scan = Buffer.concat([tail, bytes]);
+            assertEvidencePrivacy(scan, blocked);
+            tail.fill(0);
+            tail = Buffer.from(scan.subarray(Math.max(0, scan.length - overlap + 1)));
+            digest.update(bytes);
+            let written = 0;
+            while (written < bytes.length) {
+                const result = await file.write(bytes, written, bytes.length - written);
+                if (!result.bytesWritten)
+                    throw fail();
+                written += result.bytesWritten;
+            }
+            position += bytes.length;
+        }
+        if (position !== metadata.size || digest.digest('hex') !== metadata.sha256)
+            throw fail();
+        await file.sync();
+        accepted = true;
+    }
+    finally {
+        await file.close();
+        tail.fill(0);
+        if (!accepted)
+            await unlink(partial).catch(() => undefined);
+    }
+    // Exact metadata was already retained in the immutable report page. Only a
+    // successful hash-checked rename publishes its bytes to a future sync reader.
+    await rename(partial, join(directory, metadata.id));
+    await syncDirectory(directory);
+    await write(directory, metadata.id + '.json', Buffer.from(JSON.stringify(metadata)));
+}
+async function collect(owner, manifest, configuration, observation, reader, retention, signal) {
+    signal = evidenceRetentionSignal(retention, signal);
+    const control = configuration.control;
+    if (owner.purpose !== 'trial' || control.schema !== 'firedrill.trial.v1')
+        throw fail();
+    const directory = await folder(owner), blocked = evidenceNeedles(configuration);
+    const journal = { schema: 'firedrill.local-trial-journal.v1', owner, manifest_sha256: manifest,
+        execution_id: control.execution_id, scenario_key: control.scenario_key, observation, stage: 'collecting', retention,
+        pages: 0, event_offset: 0, artifact_offset: 0, artifacts_retained: 0, page_chain_sha256: hash(''), collected_at: null,
+        cloud_completion_recorded: false, provenance: 'customer_device' };
+    const persist = (initial = false) => {
+        assertEvidenceRetentionCurrent(retention);
+        const body = Buffer.from(JSON.stringify(journal));
+        assertEvidencePrivacy(body, blocked);
+        return write(directory, 'receipt.json', body, !initial);
+    };
+    try {
+        await persist(true); // Existing/uncertain attempt is never overwritten.
+        for (;;) {
+            signal.throwIfAborted();
+            assertEvidenceRetentionCurrent(retention);
+            const report = await reader.report(journal.event_offset, journal.artifact_offset);
+            assertEvidenceRetentionCurrent(retention);
+            if (JSON.stringify(report.status) !== JSON.stringify(observation.status))
+                throw fail();
+            const body = trialReportBytes(report);
+            assertEvidencePrivacy(body, blocked);
+            const digest = hash(body), page = journal.pages;
+            await write(directory, `page-${page}.json`, body);
+            journal.pages++;
+            journal.event_offset = report.offset;
+            journal.artifact_offset = report.artifact_offset;
+            journal.page_chain_sha256 = hash(journal.page_chain_sha256 + ':' + page + ':' + digest);
+            await persist(); // Keep events even if the next artifact/pipe is lost.
+            for (const metadata of report.artifacts) {
+                await retainArtifact(directory, metadata, reader, blocked, retention, signal);
+                journal.artifacts_retained++;
+                await persist();
+            }
+            if (!report.events_more && !report.artifacts_more)
+                break;
+        }
+        journal.stage = 'local_collected';
+        journal.collected_at = new Date().toISOString();
+        await persist();
+        return { schema: 'firedrill.local-trial-collection.v1', operation_id: owner.lease, execution_id: control.execution_id,
+            state: journal.stage, events: journal.event_offset, artifacts: journal.artifacts_retained,
+            provenance: 'customer_device', cloud_completion_recorded: false };
+    }
+    finally {
+        for (const value of blocked)
+            value.fill(0);
+        blocked.length = 0;
+    }
+}
+/** Uses the same claim/keeper/engine and exact cleanup path, never a new engine.
+ * Campaign admission and the explicit assignment command own the user intent.
+ */
+export async function retainLocalTrial(context, device, job, request, signal, output) {
+    let operation, executionError;
+    let result;
+    let live;
+    try {
+        result = await withLocalJobInputs(context, device, job, request, 'trial', signal, async (owner, inputs, configuration, active) => {
+            operation = owner.lease;
+            if (owner.purpose !== 'trial')
+                throw fail();
+            const evidence = await acquireLocalEvidence(owner, owner.lease, frozenEvidenceRetention(owner, inputs.manifestSha256, inputs.manifest.evidence_recovery_until));
+            try {
+                active = AbortSignal.any([active, evidence.signal]);
+                const stream = await createLiveTrialRetainer(owner, inputs.manifestSha256, configuration, evidence.retention, active, output);
+                let collected;
+                try {
+                    collected = await executeLocalTrial(owner, inputs, configuration, (owned, observation, reader, collecting) => collect(owned, inputs.manifestSha256, configuration, observation, reader, evidence.retention, collecting), active, stream.accept, stream.artifact);
+                }
+                catch (error) {
+                    await stream.finish().catch(() => undefined);
+                    throw error;
+                }
+                live = await stream.finish();
+                return collected;
+            }
+            finally {
+                await evidence.close();
+            }
+        });
+    }
+    catch (error) {
+        executionError = error;
+    }
+    const { syncLocalTrialCompletion } = await import('./runner-trial-completion.js');
+    if (operation) {
+        try {
+            await syncLocalTrialCompletion(context, operation, AbortSignal.timeout(30000));
+        }
+        catch (error) {
+            if (!executionError)
+                throw error;
+        }
+    }
+    if (executionError) {
+        if (operation) {
+            // Original evidence consent still governs bounded partial bookkeeping.
+            // This transfer cannot start/renew the failed or interrupted execution.
+            try {
+                const { syncLocalTrialEvidence } = await import('./runner-trial-sync.js');
+                await syncLocalTrialEvidence(context, operation, AbortSignal.timeout(30000), output);
+            }
+            catch { /* Exact private journal remains. */ }
+        }
+        throw executionError;
+    }
+    if (!result || !operation)
+        throw fail();
+    const { syncLocalTrialEvidence } = await import('./runner-trial-sync.js');
+    const evidence = await syncLocalTrialEvidence(context, operation, signal, output);
+    const closing = await syncLocalTrialCompletion(context, operation, signal, { wait: true });
+    return { ...result, live, evidence, closing };
+}
+async function privateFile(path, maximum) {
+    const before = await lstat(path);
+    if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.mode & 0o077 || before.size > maximum
+        || process.getuid && before.uid !== process.getuid())
+        throw fail();
+    const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    const actual = await file.stat();
+    if (actual.ino !== before.ino || actual.dev !== before.dev || actual.size !== before.size) {
+        await file.close();
+        throw fail();
+    }
+    return file;
+}
+async function privateJson(path, maximum) {
+    const file = await privateFile(path, maximum);
+    try {
+        const before = await file.stat(), body = await file.readFile('utf8'), after = await file.stat();
+        if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
+            throw fail();
+        return JSON.parse(body);
+    }
+    catch {
+        throw fail();
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Immutable local snapshot for subsequent explicit sync. Never opens Docker,
+ * renews a lease, runs an agent or assumes the cloud retained this evidence.
+ * A consumer must exhaust pages before committing an aggregate: the final chain
+ * check is not permission to treat a partially consumed snapshot as complete.
+ */
+export async function localTrialCollection(scope, operation, retention) {
+    const directory = await folder({ ...scope, lease: identifier(operation) }, false);
+    const value = await privateJson(join(directory, 'receipt.json'), 65536);
+    if (!value || value.schema !== 'firedrill.local-trial-journal.v1' || !value.owner || value.owner.owner !== scope.owner
+        || value.owner.workspace !== scope.workspace || value.owner.lease !== operation || value.owner.purpose !== 'trial'
+        || !['collecting', 'local_collected'].includes(value.stage) || value.provenance !== 'customer_device'
+        || value.cloud_completion_recorded !== false || !/^[a-f0-9]{64}$/.test(value.manifest_sha256)
+        || !/^[a-f0-9]{64}$/.test(value.page_chain_sha256) || typeof value.scenario_key !== 'string'
+        || !value.scenario_key.length || value.scenario_key.length > 255
+        || [value.pages, value.event_offset, value.artifact_offset, value.artifacts_retained].some(item => !Number.isSafeInteger(item) || item < 0)
+        || value.artifacts_retained > value.artifact_offset || value.event_offset > value.pages || value.artifact_offset > value.pages
+        || value.stage === 'local_collected' && (value.artifacts_retained !== value.artifact_offset
+            || typeof value.collected_at !== 'string' || !Number.isFinite(Date.parse(value.collected_at))))
+        throw fail();
+    for (const id of [value.execution_id, value.owner.job, value.owner.attempt, value.owner.device, value.owner.request])
+        identifier(id);
+    const observation = value.observation;
+    if (!observation || observation.operation_id !== operation || observation.provenance !== 'customer_device'
+        || !/^[a-f0-9]{64}$/.test(observation.container_id) || !observation.launch || observation.launch.launch_id !== operation
+        || !['exited', 'launch_failed'].includes(observation.launch.state)
+        || observation.launch.state === 'exited' && (!Number.isSafeInteger(observation.launch.exit_code)
+            || observation.launch.exit_code < -128 || observation.launch.exit_code > 255))
+        throw fail();
+    trialStatus(observation.status);
+    const raw = () => {
+        if (!retention || !retentionMatches(value.retention, retention) || value.manifest_sha256 !== retention.manifest_sha256)
+            throw new CliError('local_evidence_retention_missing', 'This drill is available for metadata inspection only. Raw evidence needs its original finite device retention policy.');
+        assertEvidenceRetentionCurrent(retention);
+    };
+    return { journal: value,
+        pages: async function* (signal) {
+            raw();
+            let after = 0, artifactAfter = 0, chain = hash(''), last = null;
+            for (let position = 0; position < value.pages; position++) {
+                signal.throwIfAborted();
+                raw();
+                const file = await privateFile(join(directory, `page-${position}.json`), 4 * 1024 ** 2);
+                let bytes;
+                try {
+                    const before = await file.stat();
+                    bytes = await file.readFile();
+                    const after = await file.stat();
+                    if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
+                        throw fail();
+                }
+                finally {
+                    await file.close();
+                }
+                let decoded;
+                try {
+                    decoded = JSON.parse(bytes.toString('utf8'));
+                }
+                catch {
+                    throw fail();
+                }
+                last = inspectTrialReport(decoded, operation, value.execution_id, value.scenario_key, observation.launch, after, artifactAfter);
+                if (JSON.stringify(last.status) !== JSON.stringify(observation.status))
+                    throw fail();
+                after = last.offset;
+                artifactAfter = last.artifact_offset;
+                chain = hash(chain + ':' + position + ':' + hash(bytes));
+                yield { position, size: bytes.length, sha256: hash(bytes), bytes, report: last };
+            }
+            if (chain !== value.page_chain_sha256 || after !== value.event_offset || artifactAfter !== value.artifact_offset
+                || value.stage === 'local_collected' && (!last || last.events_more || last.artifacts_more))
+                throw fail();
+        },
+        artifact: async (metadata, consume, signal) => {
+            raw();
+            if (!/^(0|[1-9][0-9]{0,9})\.bin$/.test(metadata.id) || metadata.execution_id !== value.execution_id
+                || metadata.scenario_key !== value.scenario_key)
+                throw fail();
+            const retained = await privateJson(join(directory, metadata.id + '.json'), 8192);
+            if (!retained || Object.keys(retained).sort().join(',') !== Object.keys(metadata).sort().join(',')
+                || Object.keys(metadata).some(key => retained[key] !== metadata[key])
+                || !Number.isSafeInteger(metadata.size) || metadata.size < 0 || metadata.size > 512 * 1024 ** 2)
+                throw fail();
+            const file = await privateFile(join(directory, metadata.id), 512 * 1024 ** 2);
+            try {
+                const before = await file.stat(), digest = createHash('sha256');
+                for await (const body of file.createReadStream({ autoClose: false, start: 0 })) {
+                    signal.throwIfAborted();
+                    raw();
+                    digest.update(body);
+                }
+                if (before.size !== metadata.size || digest.digest('hex') !== metadata.sha256)
+                    throw fail();
+                await consume(async (offset, length) => {
+                    signal.throwIfAborted();
+                    raw();
+                    if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(length) || length < 1 || length > 1024 ** 2
+                        || offset + length > metadata.size)
+                        throw fail();
+                    const bytes = Buffer.alloc(length);
+                    let position = 0;
+                    while (position < length) {
+                        const part = await file.read(bytes, position, length - position, offset + position);
+                        if (!part.bytesRead)
+                            throw fail();
+                        position += part.bytesRead;
+                    }
+                    const after = await file.stat();
+                    if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs)
+                        throw fail();
+                    return bytes;
+                });
+            }
+            finally {
+                await file.close();
+            }
+        },
+    };
+}