firedrill-run 2.2.5 → 3.0.0

package/dist/cli/src/source-connection.js

@@ -0,0 +1,451 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, open, realpath, rename, unlink } from 'node:fs/promises';
+import { basename, dirname, join, resolve } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { Readable } from 'node:stream';
+import { api } from './auth.js';
+import { privateDirectory } from './credentials.js';
+import { CliError, releaseClosed } from './http.js';
+import { localSource } from './local-source.js';
+import { selectOne, terminalText } from './prompts.js';
+import { identifier } from './world-contracts.js';
+import { selectedWorkspace } from './world-access.js';
+import { withBootstrapProcessLock } from './bootstrap-process-lock.js';
+import { localSourceConnectionMatches, localSourceLookupMatches, localSourceOriginalMatches, localSourceUploadMatches, localSourcePageMatches, nativeMetadataCanonical } from '../../contracts/local-value-source-contracts.js';
+const prefix = (scope, key) => `/api/workspaces/${identifier(scope.workspace)}/local-sources/${identifier(key)}`;
+const auth = (scope, signal) => ({ actor: scope.owner, workspace: scope.workspace, signal });
+const projectKey = (root) => createHash('sha256').update(root).digest('hex');
+const requestDigest = (body) => createHash('sha256').update(nativeMetadataCanonical(body)).digest('hex');
+const invalidReceipt = () => new CliError('source_receipt_invalid', 'The source response did not match this account, workspace and original capture. No source was substituted or target started.');
+function original(intent, value) {
+    if (value.state === 'not_retained' || !localSourceOriginalMatches(value, intent, intent.key, intent.body, requestDigest(intent.body)))
+        throw invalidReceipt();
+}
+export async function ownedSourceMetadataFile(path, maximum) {
+    const previous = await lstat(path);
+    if (!previous.isFile() || previous.isSymbolicLink() || previous.nlink !== 1 || previous.size > maximum
+        || process.getuid && previous.uid !== process.getuid() || process.platform !== 'win32' && (previous.mode & 0o077) !== 0) {
+        throw new CliError('source_journal_unsafe', 'Use private source metadata owned by this OS user. No file was transferred.');
+    }
+    const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    const actual = await file.stat();
+    if (actual.ino !== previous.ino || actual.dev !== previous.dev || actual.size !== previous.size
+        || actual.nlink !== 1 || actual.mtimeMs !== previous.mtimeMs || actual.ctimeMs !== previous.ctimeMs) {
+        await file.close();
+        throw new CliError('source_journal_changed', 'The local source metadata changed. No file was transferred.');
+    }
+    return file;
+}
+const ownedFile = ownedSourceMetadataFile;
+export async function readSourceMetadata(file, maximum) {
+    const before = await file.stat();
+    if (!before.isFile() || before.nlink !== 1 || before.size > maximum)
+        throw new CliError('source_journal_unsafe', 'The original source metadata exceeds its private read capacity.');
+    const bytes = Buffer.alloc(before.size + 1);
+    let size = 0;
+    while (size < bytes.length) {
+        const part = await file.read(bytes, size, bytes.length - size, size);
+        if (!part.bytesRead)
+            break;
+        size += part.bytesRead;
+    }
+    const after = await file.stat();
+    if (size !== before.size || after.size !== before.size || after.nlink !== 1
+        || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) {
+        throw new CliError('source_journal_changed', 'The original source metadata changed while it was read. Nothing was transferred.');
+    }
+    return JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(bytes.subarray(0, size)));
+}
+export async function writeSourceMetadata(path, value) {
+    const temporary = path + '.' + randomUUID() + '.tmp';
+    const file = await open(temporary, 'wx', 0o600);
+    try {
+        await file.writeFile(JSON.stringify(value));
+        await file.sync();
+    }
+    finally {
+        await file.close();
+    }
+    try {
+        await rename(temporary, path);
+    }
+    catch (error) {
+        await unlink(temporary).catch(() => undefined);
+        throw error;
+    }
+    if (process.platform !== 'win32') {
+        const directory = await open(dirname(path), constants.O_RDONLY | (constants.O_DIRECTORY ?? 0));
+        try {
+            await directory.sync();
+        }
+        finally {
+            await directory.close();
+        }
+    }
+}
+const writePrivate = writeSourceMetadata;
+async function journalPath(scope, key) {
+    return join(await privateDirectory(), `source-connect-${identifier(scope.owner)}-${identifier(scope.workspace)}-${identifier(key)}.json`);
+}
+async function journal(scope, key) {
+    const file = await ownedFile(await journalPath(scope, key), 16384);
+    try {
+        const value = await readSourceMetadata(file, 16384);
+        if (value.schema !== 'firedrill.source-connect-intent.v1' || value.key !== identifier(key)
+            || value.owner !== scope.owner || value.workspace !== scope.workspace || !/^[a-f0-9]{64}$/.test(value.project_key)
+            || value.body?.schema !== 'firedrill.local-source-connect.v1' || !/^[a-f0-9]{64}$/.test(value.body.archive?.sha256)
+            || !/^[a-f0-9]{64}$/.test(value.body.capture?.content_digest)
+            || value.body.consent?.policy !== 'firedrill.local-source-transfer.v1'
+            || value.body.consent.source_upload !== true || value.body.consent.source_processing !== true
+            || !['request_dispatched', 'registered', 'upload_dispatched', 'uploaded', 'complete_requested'].includes(value.stage))
+            throw new Error();
+        return value;
+    }
+    catch {
+        throw new CliError('source_journal_invalid', 'The source request journal could not be read safely. It was not discarded.');
+    }
+    finally {
+        await file.close();
+    }
+}
+async function requestLock(scope, key, signal, work) {
+    const directory = join(await privateDirectory(), `source-lock-${identifier(scope.owner)}-${identifier(scope.workspace)}-${identifier(key)}`);
+    return withBootstrapProcessLock(directory, signal, work);
+}
+function readable(value) {
+    const key = `Source request ${value.request_key}`;
+    if (value.state === 'not_retained')
+        return `${key}: no retained receipt yet. This does not prove the original request cannot arrive.`;
+    if (value.state === 'connected')
+        return value.defer_world
+            ? `${key}: component source connected.\nAgent: ${value.agent_id}\nNo world, build or simulation has been started.`
+            : `${key}: source connected.\nWorld: ${value.world_id}\nNo build or simulation has been started.`;
+    if (value.state === 'awaiting_upload')
+        return `${key}: waiting for the approved source archive.`;
+    if (value.state === 'verifying')
+        return `${key}: source processing is accepted and still in progress.`;
+    if (value.state === 'upload_missing')
+        return `${key}: upload is not visible yet and may still arrive. Use firedrill connect resume ${value.request_key} to check processing, or connect upload ${value.request_key} to explicitly attempt the same archive.`;
+    if (value.state === 'withdrawn')
+        return `${key}: withdrawn. A delayed request with this key cannot create source.`;
+    return `${key}: ${value.state}. ${value.problem ?? 'Inspect this request before continuing.'}`;
+}
+async function lookup(scope, key, signal) {
+    const result = await api(prefix(scope, key), auth(scope, signal));
+    if (!localSourceLookupMatches(result, scope, key))
+        throw invalidReceipt();
+    return result;
+}
+async function bindProject(intent, receipt) {
+    original(intent, receipt);
+    if (receipt.state !== 'connected' || !receipt.world_id || !receipt.source_id || !receipt.agent_id
+        || receipt.capture?.content_digest !== intent.body.capture.content_digest
+        || receipt.archive?.sha256 !== intent.body.archive.sha256)
+        throw new CliError('source_receipt_invalid', 'The connected source did not match the selected capture.');
+    await writePrivate(join(await privateDirectory(), `project-${intent.owner}-${intent.project_key}.json`), {
+        schema: 'firedrill.project-binding.v1', owner: intent.owner, workspace: intent.workspace, world: identifier(receipt.world_id),
+        agent: identifier(receipt.agent_id), source: identifier(receipt.source_id), content_digest: intent.body.capture.content_digest
+    });
+}
+export async function bindSavedProject(scope, root, value) {
+    const physical = await realpath(resolve(root));
+    if (!/^[a-f0-9]{64}$/.test(value.content_digest))
+        throw new CliError('source_receipt_invalid', 'The source digest cannot be confirmed.');
+    await writePrivate(join(await privateDirectory(), `project-${identifier(scope.owner)}-${projectKey(physical)}.json`), {
+        schema: 'firedrill.project-binding.v1', owner: scope.owner, workspace: scope.workspace, world: identifier(value.world),
+        agent: identifier(value.agent), source: identifier(value.source), content_digest: value.content_digest,
+        ...(value.batch_key ? { batch_key: identifier(value.batch_key) } : {})
+    });
+}
+export async function savedProject(scope, root) {
+    const physical = await realpath(resolve(root));
+    const path = join(await privateDirectory(), `project-${identifier(scope.owner)}-${projectKey(physical)}.json`);
+    let file;
+    try {
+        file = await ownedFile(path, 4096);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return null;
+        throw error;
+    }
+    try {
+        const value = await readSourceMetadata(file, 4096);
+        if (value.schema !== 'firedrill.project-binding.v1' || value.owner !== scope.owner || typeof value.world !== 'string')
+            throw new Error();
+        if (value.workspace !== scope.workspace)
+            return null;
+        return { world: identifier(value.world), source: value.source === undefined ? null : identifier(String(value.source)),
+            batch_key: value.batch_key === undefined ? null : identifier(String(value.batch_key)) };
+    }
+    catch {
+        throw new CliError('project_binding_invalid', 'The saved project binding needs attention. Choose an explicit world; it was not discarded.');
+    }
+    finally {
+        await file.close();
+    }
+}
+async function archiveFile(intent) {
+    const directory = await privateDirectory();
+    if (dirname(intent.snapshot) !== directory || !/^source-[A-Za-z0-9_-]+$/.test(basename(intent.snapshot)))
+        throw new CliError('source_snapshot_invalid', 'Choose the private captured source for this request.');
+    const info = await lstat(intent.snapshot);
+    if (!info.isDirectory() || info.isSymbolicLink() || process.getuid && info.uid !== process.getuid()
+        || process.platform !== 'win32' && (info.mode & 0o077) !== 0)
+        throw new CliError('source_snapshot_invalid', 'The source snapshot is not private.');
+    const file = await ownedFile(join(intent.snapshot, 'source.tar.gz'), 5 * 1024 ** 3);
+    try {
+        const digest = createHash('sha256');
+        const stream = file.createReadStream({ autoClose: false });
+        for await (const chunk of stream)
+            digest.update(chunk);
+        const actual = await file.stat();
+        if (actual.size !== intent.body.archive.size || digest.digest('hex') !== intent.body.archive.sha256)
+            throw new Error();
+        return file;
+    }
+    catch {
+        await file.close();
+        throw new CliError('source_snapshot_changed', 'The archive no longer matches the approved capture. No file was transferred.');
+    }
+}
+async function upload(intent, signal) {
+    const file = await archiveFile(intent);
+    try {
+        const access = await api(`${prefix(intent, intent.key)}/upload-access`, { ...auth(intent, signal), method: 'POST', body: {} });
+        let destination;
+        try {
+            destination = new URL(access.upload_url);
+            if (!localSourceUploadMatches(access, intent, intent.key, intent.body, requestDigest(intent.body)) || destination.origin !== 'https://storage.googleapis.com'
+                || destination.username || destination.password || destination.hash || access.archive.sha256 !== intent.body.archive.sha256
+                || access.archive.size !== intent.body.archive.size)
+                throw new Error();
+        }
+        catch {
+            throw new CliError('source_upload_access_invalid', 'The scoped upload response was not accepted. No file was transferred.');
+        }
+        intent.stage = 'upload_dispatched';
+        await writePrivate(await journalPath(intent, intent.key), intent);
+        signal.throwIfAborted();
+        try {
+            const response = await fetch(destination, { method: 'PUT', headers: { ...access.headers, 'Content-Length': String(intent.body.archive.size) },
+                body: Readable.toWeb(file.createReadStream({ autoClose: false, start: 0 })), duplex: 'half', redirect: 'error', credentials: 'omit',
+                signal: AbortSignal.any([signal, AbortSignal.timeout(12 * 60 * 1000)]) });
+            await response.body?.cancel();
+            // 412 can mean the write-once object already exists after a lost reply.
+            // Completion verifies exact GCS bytes; it is not treated as upload success.
+            if (![200, 201, 412].includes(response.status))
+                throw new CliError('source_upload_uncertain', 'The source upload was not confirmed. Inspect or complete this exact request before choosing to upload it again.');
+            if (response.status !== 412) {
+                intent.stage = 'uploaded';
+                await writePrivate(await journalPath(intent, intent.key), intent);
+            }
+        }
+        catch (error) {
+            if (signal.aborted)
+                throw error;
+            throw error instanceof CliError ? error : new CliError('source_upload_uncertain', 'The upload reply was not received. Use connect resume for the exact request; source is not uploaded again automatically.');
+        }
+    }
+    finally {
+        await file.close();
+    }
+}
+async function complete(intent, signal, output) {
+    signal.throwIfAborted();
+    intent.stage = 'complete_requested';
+    await writePrivate(await journalPath(intent, intent.key), intent);
+    let value;
+    try {
+        value = await api(`${prefix(intent, intent.key)}/complete`, { ...auth(intent, signal), method: 'POST', body: {} });
+        original(intent, value);
+        value = await lookup(intent, intent.key, signal);
+    }
+    catch (error) {
+        if (signal.aborted || releaseClosed(error))
+            throw error;
+        value = await lookup(intent, intent.key, signal);
+    }
+    let last = '';
+    const deadline = performance.now() + 30000;
+    while (true) {
+        original(intent, value);
+        if (last !== value.state) {
+            output('source_connection', value, readable(value));
+            last = value.state;
+        }
+        if (value.state !== 'verifying' || performance.now() >= deadline)
+            break;
+        await sleep(1500, undefined, { signal });
+        value = await lookup(intent, intent.key, signal);
+    }
+    if (value.state === 'connected' && !value.defer_world)
+        await bindProject(intent, value);
+    if (['refused', 'rejected'].includes(value.state))
+        throw new CliError('source_connection_refused', value.problem ?? 'Source connection was refused.');
+    return value;
+}
+export async function connectSource(options, output) {
+    const interactive = !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY;
+    if (!interactive && !options.upload)
+        throw new CliError('source_transfer_consent_required', 'Use --upload-source to explicitly approve selected source upload and cloud source processing. No capture was transferred.', 2);
+    const root = await realpath(resolve(options.root ?? process.cwd()));
+    const capture = await localSource('snapshot', root, options.signal, interactive);
+    output('source_capture', { files: capture.files.length, bytes: capture.expanded_bytes, content_digest: capture.content_digest }, `${capture.files.length} selected files, ${capture.expanded_bytes} bytes. Environment files and values excluded.\n${capture.disclosure}`);
+    const approved = options.upload || await selectOne('Source use', [
+        { value: true, label: 'Upload selected source for cloud preparation', detail: 'Store this capture in the workspace and let Firedrill process its source. Does not start a target or simulation.' },
+        { value: false, label: 'Keep this capture local', detail: 'Do not connect or transfer source.' },
+    ], options.signal);
+    if (!approved) {
+        output('source_local', { uploaded: false, snapshot: capture.local_snapshot_path }, 'Capture kept locally. No cloud connection was created.');
+        return null;
+    }
+    if (!capture.archive || !capture.local_snapshot_path)
+        throw new CliError('source_snapshot_invalid', 'The private source capture is incomplete.');
+    const scope = await selectedWorkspace({ workspace: options.workspace, expectedOwner: options.expectedOwner,
+        interactive, signal: options.signal, openBrowser: options.openBrowser });
+    if (options.expectedOwner && scope.owner !== options.expectedOwner)
+        throw new CliError('native_account_changed', 'The terminal account changed. Reopen this project before transferring source.', 2);
+    const name = (options.name ?? basename(root)).trim();
+    if (!name || name.length > 120 || /[\u0000-\u001f\u007f]/.test(name))
+        throw new CliError('agent_name_invalid', 'Use --name with an agent name of up to 120 characters.', 2);
+    const intent = capturedSourceIntent(scope, root, name, capture);
+    return connectPreparedSource(intent, options.signal, output);
+}
+/** Reuse the immutable captured bytes, not a new scan after partial transfer. */
+export function capturedSourceIntent(scope, root, name, capture, options = {}) {
+    if (!capture.archive || !capture.local_snapshot_path)
+        throw new CliError('source_snapshot_invalid', 'The private source capture is incomplete.');
+    return { ...scope, schema: 'firedrill.source-connect-intent.v1', key: options.key ? identifier(options.key) : randomUUID(), snapshot: capture.local_snapshot_path,
+        project_key: projectKey(root), stage: 'request_dispatched', created_at: new Date().toISOString(), body: {
+            schema: 'firedrill.local-source-connect.v1', name, archive: { size: capture.archive.size, sha256: capture.archive.sha256 },
+            capture: { schema: 'firedrill.local-source.v1', content_digest: capture.content_digest,
+                base_commit: capture.repository?.base_commit ?? null, project_root: capture.repository?.project_root ?? '',
+                file_count: capture.files.length, expanded_bytes: capture.expanded_bytes },
+            consent: { policy: 'firedrill.local-source-transfer.v1', source_upload: true, source_processing: true },
+            ...(options.deferWorld ? { defer_world: true } : {}),
+            ...(options.componentAgent ? { component_agent_id: identifier(options.componentAgent) } : {})
+        } };
+}
+export async function connectPreparedSource(intent, signal, output) {
+    return requestLock(intent, intent.key, signal, async (active) => {
+        signal = active;
+        // A recoverable metadata-only intent precedes the first network mutation.
+        await writePrivate(await journalPath(intent, intent.key), intent);
+        output('source_request', { request_key: intent.key, state: 'not_retained' }, `Source request ${intent.key}.`);
+        let receipt;
+        try {
+            receipt = await api(prefix(intent, intent.key), { ...auth(intent, signal), method: 'PUT', body: intent.body });
+        }
+        catch (error) {
+            if (signal.aborted || releaseClosed(error))
+                throw error;
+            receipt = await lookup(intent, intent.key, signal);
+            if (receipt.state === 'not_retained')
+                throw new CliError('source_request_uncertain', `Inspect or withdraw source request ${intent.key}. It was not resubmitted. No source upload was attempted.`);
+        }
+        original(intent, receipt);
+        receipt = await lookup(intent, intent.key, signal);
+        original(intent, receipt);
+        if (receipt.state !== 'awaiting_upload')
+            throw new CliError('source_request_changed', 'The saved source request changed. Inspect its exact key.');
+        intent.stage = 'registered';
+        await writePrivate(await journalPath(intent, intent.key), intent);
+        await upload(intent, signal);
+        return complete(intent, signal, output);
+    });
+}
+export async function retainPreparedSource(intent) {
+    await writePrivate(await journalPath(intent, intent.key), intent);
+}
+export async function resumePreparedSource(scope, key, signal, output) {
+    return requestLock(scope, key, signal, async (active) => {
+        const intent = await journal(scope, key);
+        const value = await lookup(scope, key, active);
+        original(intent, value);
+        if (value.state === 'connected') {
+            if (!value.defer_world)
+                await bindProject(intent, value);
+            output('source_connection', value, readable(value));
+            return value;
+        }
+        if (!['awaiting_upload', 'upload_missing', 'verifying'].includes(value.state)) {
+            output('source_connection', value, readable(value));
+            return value;
+        }
+        // Only a first upload that never reached its dispatch boundary is allowed
+        // automatically. Uncertain writes require the explicit same-key upload
+        // command, not recapture or implicit retries by a batch coordinator.
+        if (value.state === 'awaiting_upload' && ['request_dispatched', 'registered'].includes(intent.stage))
+            await upload(intent, active);
+        return complete(intent, active, output);
+    });
+}
+export async function sourceRequests(options, output) {
+    const scope = await selectedWorkspace({ workspace: options.workspace, interactive: !options.json && !!process.stdin.isTTY,
+        signal: options.signal, openBrowser: options.openBrowser });
+    if (options.action === 'list') {
+        const search = (options.search ?? '').trim(), status = options.status ?? 'all', after = options.after ?? '';
+        if (search.length > 200 || /[\u0000-\u001f\u007f]/.test(search)
+            || !['all', 'awaiting_upload', 'verifying', 'upload_missing', 'connected', 'withdrawn', 'refused', 'rejected'].includes(status)
+            || after.length > 2048 || after && !/^[A-Za-z0-9_.:-]+$/.test(after)) {
+            throw new CliError('source_history_filters_invalid', 'Use the available source request status, search and returned continuation.', 2);
+        }
+        const query = new URLSearchParams({ q: search, state: status });
+        if (after)
+            query.set('after', after);
+        const value = await api(`/api/workspaces/${identifier(scope.workspace)}/local-sources?${query}`, auth(scope, options.signal));
+        if (!localSourcePageMatches(value, scope, { query: search, state: status, after }))
+            throw invalidReceipt();
+        const lines = value.results.map(one => `${one.name || 'Withdrawn source request'}\n${readable(one)}`);
+        lines.push(`${value.results.length} retained source requests on this page.`);
+        if (value.next_after)
+            lines.push(`Continue with --after '${value.next_after}' and the same --search and --status filters.`);
+        lines.push('This is cloud-retained history. A pending local request may not appear yet. Use connect status, resume, upload or withdraw with its original request key; no local journal was discarded.');
+        output('source_connection_page', value, lines.join('\n'));
+        return;
+    }
+    const key = identifier(options.key ?? '');
+    if (options.action === 'status') {
+        const value = await lookup(scope, key, options.signal);
+        output('source_connection', value, readable(value));
+        return;
+    }
+    if (options.action === 'withdraw') {
+        const value = await api(prefix(scope, key), { ...auth(scope, options.signal), method: 'DELETE' });
+        const retained = await lookup(scope, key, options.signal);
+        if (!localSourceConnectionMatches(value, scope, key) || value.state !== 'withdrawn' || retained.state !== 'withdrawn'
+            || nativeMetadataCanonical(value) !== nativeMetadataCanonical(retained))
+            throw invalidReceipt();
+        output('source_connection', retained, readable(retained));
+        return;
+    }
+    await requestLock(scope, key, options.signal, async (active) => {
+        options = { ...options, signal: active };
+        const intent = await journal(scope, key);
+        const value = await lookup(scope, key, options.signal);
+        if (value.state === 'connected') {
+            if (!value.defer_world)
+                await bindProject(intent, value);
+            output('source_connection', value, readable(value));
+            return;
+        }
+        if (value.state === 'not_retained')
+            throw new CliError('source_request_uncertain', 'The original request may still arrive. Withdraw this exact key before making a new connection; it was not resubmitted.');
+        if (!['awaiting_upload', 'upload_missing', 'verifying'].includes(value.state)) {
+            output('source_connection', value, readable(value));
+            return;
+        }
+        original(intent, value);
+        if (options.action === 'upload') {
+            if (value.state === 'verifying')
+                throw new CliError('source_processing_active', 'Inspect or resume source processing. An accepted upload is not repeated.');
+            // This command is an explicit same-archive write attempt, never a hidden
+            // resume retry or a different source/target action.
+            await upload(intent, options.signal);
+        }
+        await complete(intent, options.signal, output);
+    });
+}