firedrill-run 2.2.5 → 3.0.1

package/dist/cli/src/source-batch.js

@@ -0,0 +1,322 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { open, realpath } from 'node:fs/promises';
+import { basename, dirname, join, resolve } from 'node:path';
+import { createInterface } from 'node:readline/promises';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { performance } from 'node:perf_hooks';
+import { z } from 'zod';
+import { api } from './auth.js';
+import { privateDirectory } from './credentials.js';
+import { CliError, releaseClosed } from './http.js';
+import { localSource } from './local-source.js';
+import { selectOne, terminalText } from './prompts.js';
+import { identifier } from './world-contracts.js';
+import { selectedWorkspace } from './world-access.js';
+import { withBootstrapProcessLock } from './bootstrap-process-lock.js';
+import { capturedSourceIntent, connectPreparedSource, retainPreparedSource, resumePreparedSource, bindSavedProject, ownedSourceMetadataFile, readSourceMetadata, writeSourceMetadata } from './source-connection.js';
+import { nativeMetadataCanonical, localSourceOriginalMatches } from '../../contracts/local-value-source-contracts.js';
+import { readSourceSet, readSourceSetWithdrawal, sourceSetWithdrawalSchema } from '../../contracts/source-set-contracts.js';
+import { rememberSourceSelection } from './source-selection-intents.js';
+const selectionSchema = z.object({ name: z.string().trim().min(1).max(120).optional(),
+    roots: z.array(z.object({ alias: z.string().regex(/^[a-z][a-z0-9_-]{0,63}$/), path: z.string().min(1).max(4096) }).strict()).min(1).max(128),
+    graph: z.record(z.string(), z.unknown()).optional() }).strict();
+const digest = (value) => createHash('sha256').update(nativeMetadataCanonical(value)).digest('hex');
+const invalid = () => new CliError('source_batch_receipt_invalid', 'The source selection no longer matches this account, workspace or original capture. Nothing was substituted.');
+const pending = (batch, step) => new CliError('source_batch_pending', `${step} Keep this selection and resume with firedrill connect multi-resume ${batch.key}. No capture, upload or execution will be repeated automatically.`);
+const auth = (scope, signal) => ({ actor: scope.owner, workspace: scope.workspace, signal });
+async function pathFor(scope, key) {
+    return join(await privateDirectory(), `source-batch-${identifier(scope.owner)}-${identifier(scope.workspace)}-${identifier(key)}.json`);
+}
+async function save(batch) { await writeSourceMetadata(await pathFor(batch, batch.key), batch); }
+async function load(scope, key) {
+    const file = await ownedSourceMetadataFile(await pathFor(scope, key), 768 * 1024);
+    try {
+        const value = await readSourceMetadata(file, 768 * 1024);
+        if (value.schema !== 'firedrill.local-source-batch.v1' || value.key !== identifier(key)
+            || value.owner !== scope.owner || value.workspace !== scope.workspace || !Array.isArray(value.roots)
+            || value.roots.length < 1 || value.roots.length > 128 || typeof value.name !== 'string' || !value.name || value.name.length > 120
+            || ![value.compose_key, value.world_key].every(item => identifier(item) === item)
+            || ![value.agent_id, value.expected_source_id, value.source_id, value.world_id].every(item => item === null || identifier(item) === item)
+            || ![value.compose_dispatched, value.world_dispatched].every(item => typeof item === 'boolean')
+            || new Set(value.roots.map(item => item.alias)).size !== value.roots.length
+            || new Set(value.roots.map(item => item.path)).size !== value.roots.length)
+            throw invalid();
+        for (const root of value.roots) {
+            const intent = root.intent;
+            if (!/^[a-z][a-z0-9_-]{0,63}$/.test(root.alias) || !root.path || typeof root.dispatched !== 'boolean'
+                || root.source_id !== null && identifier(root.source_id) !== root.source_id
+                || intent.schema !== 'firedrill.source-connect-intent.v1' || intent.owner !== scope.owner || intent.workspace !== scope.workspace
+                || identifier(intent.key) !== intent.key || intent.body.defer_world !== true
+                || intent.project_key !== createHash('sha256').update(root.path).digest('hex')
+                || intent.body.component_agent_id !== undefined && intent.body.component_agent_id !== value.agent_id
+                || !/^[a-f0-9]{64}$/.test(intent.body.archive.sha256) || !/^[a-f0-9]{64}$/.test(intent.body.capture.content_digest))
+                throw invalid();
+        }
+        return value;
+    }
+    catch {
+        throw new CliError('source_batch_journal_invalid', 'The original private source selection needs attention. It was not discarded, recaptured or uploaded.');
+    }
+    finally {
+        await file.close();
+    }
+}
+async function readSelection(path) {
+    const file = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    try {
+        const stat = await file.stat();
+        if (!stat.isFile() || stat.size > 128 * 1024)
+            throw new Error();
+        const bytes = Buffer.alloc(stat.size + 1);
+        const result = await file.read(bytes, 0, bytes.length, 0);
+        if (result.bytesRead !== stat.size)
+            throw new Error();
+        const selection = selectionSchema.parse(JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(bytes.subarray(0, result.bytesRead))));
+        // Paths are relative only to this explicitly selected declaration file.
+        return { ...selection, roots: selection.roots.map(root => ({ ...root, path: resolve(dirname(path), root.path) })) };
+    }
+    catch {
+        throw new CliError('source_batch_input_invalid', 'Use a JSON file with roots [{alias,path}], an optional name and optional service graph. No credentials or source files belong in this file.', 2);
+    }
+    finally {
+        await file.close();
+    }
+}
+async function interactiveSelection(signal) {
+    const roots = [];
+    const prompt = createInterface({ input: process.stdin, output: process.stderr, terminal: false });
+    try {
+        while (roots.length < 128) {
+            const path = (await prompt.question(roots.length ? 'Add source folder (blank to continue): ' : 'Agent source folder (blank for current folder): ', { signal })).trim();
+            if (!path && roots.length)
+                break;
+            const selected = resolve(path || process.cwd());
+            const proposed = basename(selected).toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 48);
+            const defaultAlias = /^[a-z]/.test(proposed) && !roots.some(root => root.alias === proposed) ? proposed : `component-${roots.length + 1}`;
+            const alias = (await prompt.question(`Component name (${defaultAlias}): `, { signal })).trim() || defaultAlias;
+            if (!/^[a-z][a-z0-9_-]{0,63}$/.test(alias) || roots.some(root => root.alias === alias))
+                throw new CliError('source_alias_invalid', 'Use distinct short component names beginning with a letter.', 2);
+            roots.push({ alias, path: selected });
+        }
+        return { roots };
+    }
+    finally {
+        prompt.close();
+    }
+}
+function checkedWorld(value, batch) {
+    if (!value || value.schema !== 'firedrill.world-creation.v1' || value.actor_id !== batch.owner
+        || value.workspace_id !== batch.workspace || value.agent_id !== batch.agent_id || value.source_id !== batch.source_id
+        || value.request_key !== batch.world_key || !['not_received', 'accepted', 'withdrawn'].includes(value.state)
+        || typeof value.resource_available !== 'boolean' || value.world_id !== null && identifier(value.world_id) !== value.world_id
+        || value.resource_available && (value.state !== 'accepted' || !value.world_id || value.detail_url !== `/worlds/${value.world_id}`))
+        throw invalid();
+    return value;
+}
+async function advance(batch, signal, output) {
+    const context = auth(batch, signal);
+    for (let index = 0; index < batch.roots.length; index++) {
+        const root = batch.roots[index];
+        if (root.source_id)
+            continue;
+        if (index > 0 && !batch.agent_id)
+            throw invalid();
+        if (!root.dispatched && index > 0)
+            root.intent.body.component_agent_id = batch.agent_id;
+        let receipt;
+        if (root.dispatched)
+            receipt = await resumePreparedSource(batch, root.intent.key, signal, output);
+        else {
+            await retainPreparedSource(root.intent);
+            root.dispatched = true;
+            await save(batch);
+            receipt = await connectPreparedSource(root.intent, signal, output);
+        }
+        if (receipt.state === 'not_retained' || !localSourceOriginalMatches(receipt, batch, root.intent.key, root.intent.body, digest(root.intent.body)))
+            throw invalid();
+        if (receipt.state !== 'connected')
+            throw pending(batch, `Source component ${root.alias} is ${receipt.state}.`);
+        const row = receipt;
+        if (!row.defer_world || row.world_id !== null || !row.agent_id || !row.source_id
+            || batch.agent_id && row.agent_id !== batch.agent_id)
+            throw invalid();
+        if (index === 0) {
+            batch.agent_id = identifier(row.agent_id);
+            batch.expected_source_id = identifier(row.source_id);
+        }
+        root.source_id = identifier(row.source_id);
+        await save(batch);
+    }
+    const base = `/api/workspaces/${batch.workspace}/agents/${batch.agent_id}`;
+    const selection = { expected_source_id: batch.expected_source_id, consent_to_transfer: true,
+        roots: batch.roots.map(root => ({ alias: root.alias, source_id: root.source_id })), ...(batch.graph ? { graph: batch.graph } : {}) };
+    await rememberSourceSelection(batch, batch.agent_id, batch.compose_key, selection);
+    const composition = (value) => {
+        if (sourceSetWithdrawalSchema.safeParse(value).success) {
+            readSourceSetWithdrawal(value, batch.workspace, batch.agent_id, batch.owner, batch.compose_key, digest({ ...selection, graph: selection.graph ?? {} }));
+            throw new CliError('source_set_withdrawn', 'This original source selection was withdrawn. No replacement composition or world was created.');
+        }
+        return readSourceSet(value, batch.workspace, batch.agent_id, batch.compose_key);
+    };
+    let raw = await api(`${base}/source-sets/${batch.compose_key}`, context);
+    if (raw.state === 'not_received') {
+        z.object({ state: z.literal('not_received'), request_key: z.literal(batch.compose_key) }).strict().parse(raw);
+        if (batch.compose_dispatched)
+            throw pending(batch, `The original source-composition request has no retained receipt yet. Inspect sources status ${batch.compose_key} --agent ${batch.agent_id}; explicitly retry the same sources use --request-key or withdraw that original selection.`);
+        batch.compose_dispatched = true;
+        await save(batch);
+        output('source_set_request', { agent_id: batch.agent_id, request_key: batch.compose_key, batch_key: batch.key }, 'Combining the approved source captures.');
+        try {
+            raw = await api(`${base}/source-sets`, { ...context, method: 'POST', idempotencyKey: batch.compose_key, body: selection });
+        }
+        catch (error) {
+            if (signal.aborted || releaseClosed(error))
+                throw error;
+            raw = await api(`${base}/source-sets/${batch.compose_key}`, context);
+            if (raw.state === 'not_received')
+                throw pending(batch, 'The original composition may still arrive.');
+        }
+    }
+    let composed = composition(raw);
+    const deadline = performance.now() + 60000;
+    while (true) {
+        if (composed.actor_id !== batch.owner || composed.previous_source_id !== batch.expected_source_id
+            || composed.request_sha256 !== digest({ ...selection, graph: selection.graph ?? {} })
+            || composed.roots.length !== selection.roots.length || selection.roots.some(root => !composed.roots.some(actual => actual.alias === root.alias && actual.source_id === root.source_id)))
+            throw invalid();
+        if (composed.state === 'ready')
+            break;
+        if (['failed', 'withdrawn'].includes(composed.state))
+            throw new CliError('source_batch_composition_failed', composed.failure?.message ?? 'This source selection could not be composed. Review its saved request.');
+        if (performance.now() >= deadline)
+            throw pending(batch, 'Source composition is still in progress.');
+        await sleep(1500, undefined, { signal });
+        composed = composition(await api(`${base}/source-sets/${batch.compose_key}`, context));
+    }
+    if (batch.source_id && batch.source_id !== composed.source_id)
+        throw invalid();
+    const sourceId = composed.source_id;
+    batch.source_id = sourceId;
+    await save(batch);
+    const body = { actor_id: batch.owner, source_id: sourceId };
+    const status = `${base}/world-requests/${batch.world_key}?${new URLSearchParams(body)}`;
+    let world = checkedWorld(await api(status, context), batch);
+    if (world.state === 'not_received') {
+        if (batch.world_dispatched)
+            throw pending(batch, 'The original world request may still arrive.');
+        batch.world_dispatched = true;
+        await save(batch);
+        try {
+            world = checkedWorld(await api(`${base}/worlds`, { ...context, method: 'POST', idempotencyKey: batch.world_key, body }), batch);
+        }
+        catch (error) {
+            if (signal.aborted || releaseClosed(error))
+                throw error;
+            world = checkedWorld(await api(status, context), batch);
+            if (world.state === 'not_received')
+                throw pending(batch, 'The original world request is not yet confirmed.');
+        }
+    }
+    if (world.state !== 'accepted' || !world.resource_available || !world.world_id)
+        throw new CliError('source_batch_world_unavailable', 'The original combined-source world is no longer available. No replacement was created.');
+    if (batch.world_id && batch.world_id !== world.world_id)
+        throw invalid();
+    batch.world_id = world.world_id;
+    await save(batch);
+    for (const root of batch.roots)
+        await bindSavedProject(batch, root.path, { world: batch.world_id,
+            agent: batch.agent_id, source: batch.source_id, content_digest: root.intent.body.capture.content_digest, batch_key: batch.key });
+    const result = { state: 'connected', request_key: batch.key, world_id: batch.world_id,
+        agent_id: batch.agent_id, source_id: batch.source_id, roots: batch.roots.length };
+    output('source_batch_connected', result, `${batch.roots.length} source components connected to one agent and one world.\nWorld: ${batch.world_id}\nThe world can use the combined sources. No build or simulation started.`);
+    return result;
+}
+/** Read only the exact original private selection and its existing receipts.
+ * Never advance, capture, upload or compose while checking execution authority.
+ */
+export async function originalLocalSourceBatch(scope, key, world, source, sourceSet, signal) {
+    const batch = await load(scope, identifier(key));
+    if (batch.world_id !== world || batch.source_id !== source || !batch.agent_id || !batch.expected_source_id
+        || !batch.world_dispatched || !batch.compose_dispatched || batch.roots.length !== sourceSet.root_count
+        || batch.roots.some(root => !root.dispatched || !root.source_id))
+        throw invalid();
+    const selection = { expected_source_id: batch.expected_source_id, consent_to_transfer: true,
+        roots: batch.roots.map(root => ({ alias: root.alias, source_id: root.source_id })), ...(batch.graph ? { graph: batch.graph } : {}) };
+    const composed = readSourceSet(await api(`/api/workspaces/${scope.workspace}/agents/${batch.agent_id}/source-sets/${batch.compose_key}`, auth(scope, signal)), scope.workspace, batch.agent_id, batch.compose_key);
+    if (composed.state !== 'ready' || composed.id !== sourceSet.id || composed.actor_id !== scope.owner || composed.source_id !== source
+        || composed.previous_source_id !== batch.expected_source_id || composed.source_set_sha256 !== sourceSet.source_set_sha256
+        || composed.graph_sha256 !== sourceSet.graph_sha256 || digest(composed.graph) !== sourceSet.graph_sha256
+        || composed.request_sha256 !== digest({ ...selection, graph: selection.graph ?? {} })
+        || composed.roots.length !== batch.roots.length || batch.roots.some(root => !composed.roots.some(row => row.alias === root.alias && row.source_id === root.source_id)))
+        throw invalid();
+    const roots = [];
+    for (const root of batch.roots) {
+        signal.throwIfAborted();
+        const receipt = await api(`/api/workspaces/${scope.workspace}/local-sources/${root.intent.key}`, auth(scope, signal));
+        if (!localSourceOriginalMatches(receipt, scope, root.intent.key, root.intent.body, digest(root.intent.body))
+            || receipt.state !== 'connected' || receipt.source_id !== root.source_id || receipt.agent_id !== batch.agent_id
+            || !receipt.defer_world || receipt.world_id !== null)
+            throw invalid();
+        roots.push({ alias: root.alias, path: root.path, source_id: root.source_id, content_digest: root.intent.body.capture.content_digest });
+    }
+    return { key: batch.key, roots: roots.sort((left, right) => left.alias < right.alias ? -1 : left.alias > right.alias ? 1 : 0), composition: composed };
+}
+export async function connectSources(options, output, provided) {
+    const interactive = !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY;
+    if (!interactive && ((!options.file && !provided) || !options.upload))
+        throw new CliError('source_transfer_consent_required', 'Use connect multi --file <sources.json> --upload-source to approve exactly the listed source roots.', 2);
+    const selection = selectionSchema.parse(provided ?? (options.file ? await readSelection(resolve(options.file)) : await interactiveSelection(options.signal)));
+    const scope = await selectedWorkspace({ workspace: options.workspace, expectedOwner: options.expectedOwner,
+        interactive, signal: options.signal, openBrowser: options.openBrowser });
+    const roots = [], physical = new Set();
+    for (const selected of selection.roots) {
+        const path = await realpath(resolve(selected.path));
+        if (physical.has(path) || roots.some(root => root.alias === selected.alias))
+            throw new CliError('source_selection_duplicate', 'Choose distinct physical folders and component names.', 2);
+        physical.add(path);
+        const capture = await localSource('snapshot', path, options.signal, interactive);
+        output('source_component_capture', { alias: selected.alias, files: capture.files.length, bytes: capture.expanded_bytes,
+            content_digest: capture.content_digest, base_commit: capture.repository?.base_commit ?? null }, `${terminalText(selected.alias)}: ${capture.files.length} selected files, ${capture.expanded_bytes} bytes. Environment files and values excluded.`);
+        roots.push({ alias: selected.alias, path, intent: capturedSourceIntent(scope, path, (options.name ?? selection.name ?? basename(path)).trim(), capture, { deferWorld: true }), dispatched: false, source_id: null });
+    }
+    const approved = options.upload || interactive && await selectOne('Connect these sources as one agent?', [
+        { value: false, label: 'Keep captures local' }, { value: true, label: `Upload ${roots.length} selected components`,
+            detail: 'Cloud source processing only. Credentials stay excluded. No agent execution, build or simulation starts.' }
+    ], options.signal);
+    if (!approved) {
+        output('source_batch_local', { uploaded: false, roots: roots.length }, 'Source captures kept locally. No cloud resources created.');
+        return null;
+    }
+    const name = (options.name ?? selection.name ?? basename(roots[0].path)).trim();
+    if (!name || name.length > 120 || /[\u0000-\u001f\u007f]/.test(name))
+        throw new CliError('agent_name_invalid', 'Choose an agent name up to 120 characters without terminal controls.', 2);
+    roots[0].intent.body.name = name;
+    const batch = { ...scope, schema: 'firedrill.local-source-batch.v1', key: randomUUID(), name, roots,
+        ...(selection.graph ? { graph: selection.graph } : {}), agent_id: null, expected_source_id: null,
+        compose_key: randomUUID(), compose_dispatched: false, source_id: null, world_key: randomUUID(), world_dispatched: false,
+        world_id: null, created_at: new Date().toISOString() };
+    await save(batch);
+    output('source_batch_request', { request_key: batch.key, roots: roots.length }, `Source selection ${batch.key}. Recovery: firedrill connect multi-resume ${batch.key}.`);
+    return withBootstrapProcessLock(join(await privateDirectory(), `source-batch-lock-${batch.key}`), options.signal, signal => advance(batch, signal, output));
+}
+export async function sourceBatchCommand(args, options, output) {
+    const action = args[0] ?? 'multi';
+    if (action === 'multi' && args.length <= 1)
+        return connectSources(options, output);
+    if (!['multi-resume', 'multi-status'].includes(action) || args.length !== 2 || options.file || options.upload || options.name)
+        throw new CliError('arguments_invalid', 'Use connect multi [--file sources.json], multi-status <key> or multi-resume <key>. Saved selections cannot change roots or consent.', 2);
+    const scope = await selectedWorkspace({ workspace: options.workspace, expectedOwner: options.expectedOwner,
+        interactive: !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY, signal: options.signal, openBrowser: options.openBrowser });
+    const key = identifier(args[1]);
+    return withBootstrapProcessLock(join(await privateDirectory(), `source-batch-lock-${key}`), options.signal, async (signal) => {
+        const batch = await load(scope, key);
+        if (action === 'multi-status') {
+            output('source_batch_status', { request_key: batch.key, roots: batch.roots.map(root => ({ alias: root.alias,
+                    request_key: root.intent.key, source_id: root.source_id, dispatched: root.dispatched })), agent_id: batch.agent_id,
+                source_id: batch.source_id, world_id: batch.world_id }, 'Retained local source selection. Use multi-resume to reconcile original receipts; no source was recaptured or uploaded.');
+            return null;
+        }
+        return advance(batch, signal, output);
+    });
+}