firedrill-run 2.2.5 → 3.0.1

package/dist/cli/src/evaluation-authoring.js

@@ -0,0 +1,315 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, open, unlink } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { createInterface } from 'node:readline/promises';
+import { api } from './auth.js';
+import { privateDirectory } from './credentials.js';
+import { CliError } from './http.js';
+import { selectMany, selectOne } from './prompts.js';
+import { selectPolicies } from './company-policies.js';
+import { identifier } from './world-contracts.js';
+import { selectedWorkspace, selectedWorld } from './world-access.js';
+import { assessmentRequestText, inspectAssessmentRequest } from './assessment-requests.js';
+import { inspectPriorityRequest, priorityRequestText } from './priority-requests.js';
+import { assessmentAcceptanceMatches } from '../../contracts/assessment-contracts.js';
+import { recommendationBatchMatches, recommendationAcceptanceMatches } from '../../contracts/recommendation-contracts.js';
+const CAPACITY = 256 * 1024;
+function interactive(options) { return !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY; }
+function paging(options) {
+    if (options.page !== undefined && !/^[1-9][0-9]{0,8}$/.test(options.page) || (options.search?.length ?? 0) > 200) {
+        throw new CliError('arguments_invalid', 'Choose valid paging and search options.', 2);
+    }
+    return `page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`;
+}
+function object(value) { return !!value && typeof value === 'object' && !Array.isArray(value); }
+function fingerprint(value) { return createHash('sha256').update(JSON.stringify(value)).digest('hex'); }
+async function readJson(path, privateFile = false) {
+    const absolute = resolve(path), before = await lstat(absolute);
+    if (!before.isFile() || before.isSymbolicLink() || before.size < 2 || before.size > CAPACITY
+        || privateFile && (process.getuid && before.uid !== process.getuid() || process.platform !== 'win32' && !!(before.mode & 0o077))) {
+        throw new CliError('evaluation_file_invalid', 'Choose a regular, bounded JSON file. Recovery records must remain private to this OS user.', 2);
+    }
+    const file = await open(absolute, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    try {
+        const actual = await file.stat();
+        if (!actual.isFile() || actual.dev !== before.dev || actual.ino !== before.ino || actual.size !== before.size || actual.mtimeMs !== before.mtimeMs
+            || privateFile && (process.getuid && actual.uid !== process.getuid() || process.platform !== 'win32' && !!(actual.mode & 0o077)))
+            throw new Error();
+        const buffer = Buffer.alloc(actual.size);
+        let offset = 0;
+        while (offset < buffer.length) {
+            const part = await file.read(buffer, offset, buffer.length - offset, offset);
+            if (!part.bytesRead)
+                throw new Error();
+            offset += part.bytesRead;
+        }
+        const after = await file.stat();
+        if (after.size !== actual.size || after.mtimeMs !== actual.mtimeMs || after.ctimeMs !== actual.ctimeMs)
+            throw new Error();
+        const data = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(buffer), (_key, value) => {
+            if (typeof value === 'number' && !Number.isFinite(value))
+                throw new Error();
+            return value;
+        });
+        if (!object(data))
+            throw new Error();
+        return data;
+    }
+    catch {
+        throw new CliError('evaluation_file_invalid', 'The JSON file changed or could not be read. No assessment or criterion was requested.', 2);
+    }
+    finally {
+        await file.close();
+    }
+}
+async function journalPath(scope, key) {
+    return join(await privateDirectory(), `evaluation-${identifier(scope.owner)}-${identifier(scope.workspace)}-${identifier(key)}.json`);
+}
+function validPath(kind, path, workspace) {
+    const id = '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}';
+    return new RegExp(`^/api/workspaces/${identifier(workspace)}/${kind === 'criterion'
+        ? `worlds/${id}/evaluations(?:/${id})?` : kind === 'priority' ? `worlds/${id}/recommendation-batches` : `campaigns/${id}/trials/${id}/assessments`}$`).test(path);
+}
+async function retained(scope, key, kind) {
+    const value = await readJson(await journalPath(scope, key), true);
+    if (value.schema !== 'firedrill.native-evaluation-intent.v1' || value.owner !== scope.owner || value.workspace !== scope.workspace
+        || value.key !== identifier(key) || value.kind !== kind || typeof value.path !== 'string' || !validPath(kind, value.path, scope.workspace)
+        || !object(value.body) || value.digest !== fingerprint(value.body))
+        throw new CliError('evaluation_request_invalid', 'The retained evaluation request cannot be confirmed. It was not submitted.');
+    return value;
+}
+async function submit(intent, scope, options, output) {
+    output('evaluation_request', { request_key: intent.key, kind: intent.kind, body_sha256: intent.digest }, `Request: ${intent.key}\nIf the reply is interrupted, use firedrill ${intent.kind === 'criterion' ? 'criteria' : intent.kind === 'priority' ? 'priority' : 'assess'} resume --workspace ${scope.workspace} --request-key ${intent.key}.`);
+    if (intent.kind === 'criterion') {
+        const saved = await api(intent.path, { actor: scope.owner, workspace: scope.workspace,
+            signal: options.signal, method: 'POST', idempotencyKey: intent.key, body: intent.body });
+        const receipt = await inspectCriterion(intent, scope, options, output, saved);
+        if (receipt.resource_available)
+            output('criterion_saved', { criterion: saved, request: receipt }, `${saved.name} · Revision ${saved.revision}\n${saved.id}\nCriterion saved. Existing plans and results are unchanged.`);
+        return;
+    }
+    if (intent.kind === 'priority') {
+        const world = identifier(intent.path.slice(0, -'/recommendation-batches'.length).split('/').at(-1));
+        const turn = identifier(typeof intent.body.turn_id === 'string' ? intent.body.turn_id : '');
+        const result = await api(intent.path, { actor: scope.owner, workspace: scope.workspace, signal: options.signal, method: 'POST', idempotencyKey: intent.key, body: intent.body });
+        if (!recommendationBatchMatches(result, { workspace: scope.workspace, world, turn, actor: scope.owner }))
+            throw new CliError('recommendation_request_invalid', 'Priority acceptance could not be confirmed. Inspect the original saved request.');
+        await inspectPriority(intent, scope, options, output, result);
+        return;
+    }
+    const result = await api(intent.path, { actor: scope.owner, workspace: scope.workspace, signal: options.signal, method: 'POST', idempotencyKey: intent.key, body: intent.body });
+    const prefix = intent.path.slice(0, -'/assessments'.length);
+    const receipt = await inspectAssessmentRequest(prefix, intent.key, scope, options);
+    if (!assessmentAcceptanceMatches(receipt, result, { actor: scope.owner, workspace: scope.workspace,
+        campaign: prefix.split('/').at(-3), execution: prefix.split('/').at(-1) }, intent.key)) {
+        throw new CliError('assessment_request_invalid', 'The original assessment acceptance could not be confirmed. Resume this saved request; no evaluation was repeated.');
+    }
+    output('result', result, `${result.name} · ${result.state}\n${result.id}\nAssessment requested for saved evidence. The agent was not run again.`);
+}
+async function inspectCriterion(intent, scope, options, output, saved) {
+    const prefix = intent.path.split('/evaluations')[0], world = prefix.split('/').at(-1);
+    const value = await api(`${prefix}/evaluation-request?request_key=${encodeURIComponent(intent.key)}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
+    if (value.schema !== 'firedrill.evaluation-change-receipt.v1' || value.actor_id !== scope.owner || value.workspace_id !== scope.workspace
+        || value.world_id !== world || value.request_key !== intent.key || !['accepted', 'withdrawn', 'not_received'].includes(value.state)
+        || typeof value.resource_available !== 'boolean' || value.resource_available && (value.state !== 'accepted' || !value.item_id)
+        || saved && (saved.world_id !== world || value.state !== 'accepted' || !value.resource_available
+            || value.item_id !== saved.id || value.revision !== saved.revision))
+        throw new CliError('evaluation_request_invalid', 'The original criterion receipt could not be confirmed. No save was repeated.');
+    if (!saved)
+        output('result', value, `Request: ${intent.key}\n${value.state === 'accepted'
+            ? value.resource_available ? `${value.name} · Revision ${value.revision}\n${value.item_id}\nCriterion saved. No save was repeated.` : 'The change was saved, but its criterion is no longer available.'
+            : value.state === 'withdrawn' ? 'The unsaved request was canceled. No save was repeated.'
+                : `No acceptance is recorded yet. The original request may still arrive.\nCancel only that unsaved request: firedrill criteria withdraw-request ${intent.key} --world ${world} --workspace ${scope.workspace}`}`);
+    return value;
+}
+async function inspectAssessment(intent, scope, options, output) {
+    const prefix = intent.path.slice(0, -'/assessments'.length);
+    const value = await inspectAssessmentRequest(prefix, intent.key, scope, options);
+    output('result', value, assessmentRequestText(value));
+}
+async function inspectPriority(intent, scope, options, output, accepted) {
+    const prefix = intent.path.slice(0, -'/recommendation-batches'.length), world = identifier(prefix.split('/').at(-1));
+    const turn = identifier(typeof intent.body.turn_id === 'string' ? intent.body.turn_id : '');
+    const value = await inspectPriorityRequest({ ...scope, world, prefix }, turn, intent.key, options.signal);
+    if (accepted !== undefined && !recommendationAcceptanceMatches(value, accepted, { actor: scope.owner, workspace: scope.workspace, world, turn }, intent.key)) {
+        throw new CliError('recommendation_request_invalid', 'The original priority acceptance could not be confirmed. Resume this saved request; no evaluation was repeated.');
+    }
+    output('result', value, priorityRequestText(value) + (value.resource_available
+        ? `\nView: firedrill priority show ${value.batch_id} --workspace ${scope.workspace} --world ${world}` : ''));
+}
+export async function remember(kind, path, body, scope, options, output) {
+    const key = options.request ? identifier(options.request) : randomUUID();
+    if (!validPath(kind, path, scope.workspace))
+        throw new CliError('evaluation_request_invalid', 'The evaluation destination could not be confirmed.');
+    const intent = { schema: 'firedrill.native-evaluation-intent.v1', key, owner: scope.owner, workspace: scope.workspace,
+        kind, path, body, digest: fingerprint(body) };
+    const encoded = JSON.stringify(intent);
+    if (Buffer.byteLength(encoded) > CAPACITY)
+        throw new CliError('evaluation_input_capacity', 'Use a more focused criterion. No request was submitted.', 2);
+    try {
+        const file = await open(await journalPath(scope, key), 'wx', 0o600);
+        try {
+            await file.writeFile(encoded);
+            await file.sync();
+        }
+        finally {
+            await file.close();
+        }
+    }
+    catch (error) {
+        if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'EEXIST')
+            throw error;
+        const previous = await retained(scope, key, kind);
+        if (previous.path !== path || previous.digest !== intent.digest)
+            throw new CliError('evaluation_request_conflict', 'This request key belongs to different inputs. Resume its exact saved request instead.', 2);
+        if (kind === 'criterion')
+            return inspectCriterion(previous, scope, options, output);
+        if (kind === 'priority')
+            return inspectPriority(previous, scope, options, output);
+        return inspectAssessment(previous, scope, options, output);
+    }
+    await submit(intent, scope, options, output);
+}
+async function line(label, maximum, signal, optional = false) {
+    const prompt = createInterface({ input: process.stdin, output: process.stderr, terminal: false });
+    try {
+        const value = (await prompt.question(`${label}: `, { signal })).trim();
+        if ((!optional && !value) || value.length > maximum || /[\u0000-\u001f\u007f]/u.test(value))
+            throw new CliError('criterion_input_invalid', 'Use non-empty text within the displayed field capacity. No criterion was saved.', 2);
+        return value;
+    }
+    finally {
+        prompt.close();
+    }
+}
+async function rubric(scope, agent, signal) {
+    const name = await line('Criterion name (up to 240 characters)', 240, signal);
+    const instructions = await line('Behavior to assess (up to 16,000 characters)', 16000, signal);
+    const count = await selectOne('Number of anchored score levels', Array.from({ length: 9 }, (_, index) => ({ value: index + 2, label: String(index + 2) })), signal);
+    const levels = [];
+    for (let score = 0; score < count; score++)
+        levels.push(await line(`Score ${score} description (up to 4,000 characters)`, 4000, signal));
+    if (new Set(levels).size !== levels.length)
+        throw new CliError('criterion_input_invalid', 'Use distinct score descriptions.', 2);
+    const direction = await selectOne('Better outcome', [{ value: 'higher', label: 'Higher score' }, { value: 'lower', label: 'Lower score' }], signal);
+    const sources = await selectMany('Required retained evidence', [{ value: 'conversation', label: 'Conversation' }, { value: 'actions', label: 'Tool actions' }, { value: 'state', label: 'Observed controlled-service state from this drill' }], signal, ['conversation']);
+    if (!sources.length)
+        throw new CliError('criterion_evidence_required', 'Choose at least one evidence source.', 2);
+    const applicability = await line('When this applies (optional, up to 8,000 characters)', 8000, signal, true);
+    const policies = await selectPolicies(scope, agent, signal);
+    return { name, instructions, levels, direction, sources, applicability, ...(policies.length ? { company_policies: policies } : {}) };
+}
+async function approval(kind, options) {
+    const accepted = options.approve || interactive(options) && await selectOne(kind === 'assessment'
+        ? 'Use selected saved drill evidence and policy passages for AI scoring and cited explanations? Keep a private recovery record for this request.'
+        : 'Save this criterion and keep an exact private local recovery copy? This does not run or evaluate the agent.', [{ value: false, label: 'Cancel' }, { value: true, label: kind === 'assessment' ? 'Evaluate saved evidence' : 'Save criterion' }], options.signal);
+    if (!accepted)
+        throw new CliError('evaluation_consent_required', `Nothing was submitted. Use --approve-${kind} only after reviewing these inputs and local recovery retention.`, 2);
+}
+export async function recovery(kind, args, scope, options, output) {
+    if (args.length !== 1 || !options.request || options.file || options.page || options.search || options.world)
+        throw new CliError('arguments_invalid', 'Use resume or forget with --request-key and --workspace only.', 2);
+    const intent = await retained(scope, options.request, kind);
+    if (args[0] === 'forget') {
+        const accepted = options.approve || interactive(options) && await selectOne('Remove only this private local recovery copy? This does not cancel or revoke a server request.', [{ value: false, label: 'Keep recovery copy' }, { value: true, label: 'Remove local copy' }], options.signal);
+        if (!accepted)
+            throw new CliError('evaluation_consent_required', 'The local recovery copy was kept.', 2);
+        await unlink(await journalPath(scope, intent.key));
+        output('result', { request_key: intent.key, local_copy_removed: true }, 'Local recovery copy removed. Any saved server criterion or assessment is unchanged.');
+    }
+    else if (kind === 'criterion')
+        await inspectCriterion(intent, scope, options, output);
+    else if (kind === 'priority')
+        await inspectPriority(intent, scope, options, output);
+    else
+        await inspectAssessment(intent, scope, options, output);
+}
+export async function criteriaCommand(args, options, output) {
+    paging(options);
+    const action = args[0] ?? 'list';
+    if (!['list', 'show', 'history', 'save', 'resume', 'forget'].includes(action) || args.length > 2)
+        throw new CliError('arguments_invalid', 'Use criteria list, show|history <id>, save [id], resume or forget.', 2);
+    if (['resume', 'forget'].includes(action))
+        return recovery('criterion', args, await selectedWorkspace({ ...options, interactive: interactive(options) }), options, output);
+    if (action !== 'save' && (options.file || options.request || options.approve) || action === 'list' && args.length > 1
+        || ['show', 'save'].includes(action) && (options.page || options.search) || action === 'history' && options.search) {
+        throw new CliError('arguments_invalid', 'Use only the options for this criterion action.', 2);
+    }
+    const context = await selectedWorld({ ...options, interactive: interactive(options) });
+    const auth = { actor: context.owner, workspace: context.workspace, signal: options.signal };
+    const prefix = `${context.prefix}/evaluations`;
+    if (action === 'list' || action === 'history') {
+        const data = await api(`${prefix}${action === 'history' ? `/${identifier(args[1] ?? '')}/history` : ''}?${paging(options)}`, auth);
+        output('result', data, [data.results.map(row => `${row.name} · Revision ${row.revision} · ${row.state}\n${row.id}`).join('\n\n') || 'No matching criteria.',
+            ...(data.next ? [`Next page: --page ${data.page + 1}`] : [])].join('\n'));
+        return;
+    }
+    if (action === 'show') {
+        const data = await api(`${prefix}/${identifier(args[1] ?? '')}`, auth);
+        output('result', data, `${data.name} · Revision ${data.revision} · ${data.state}\n${data.rubric ? JSON.stringify(data.rubric, null, 2) : 'This native check is inspected through its world contract.'}`);
+        return;
+    }
+    const item = args[1] ? identifier(args[1]) : undefined;
+    let body;
+    if (options.file)
+        body = await readJson(options.file);
+    else {
+        if (!interactive(options) || item)
+            throw new CliError('criterion_file_required', 'Use --file with the exact criterion body to edit a criterion or automate creation. Include world_revision, basis_revision, rubric, scenario_keys, enabled and archived.', 2);
+        const data = await api(prefix, auth);
+        if (!data.can_write)
+            throw new CliError('write_access_required', 'Write access is required to save criteria.', 2);
+        const definition = await rubric(context, identifier(data.agent_id), options.signal);
+        const enabled = await selectOne('Use this criterion in future simulation plans?', [{ value: false, label: 'Save disabled' }, { value: true, label: 'Enable for all scenarios' }], options.signal);
+        const change = { world_revision: data.world_revision, basis_revision: null, rubric: definition, scenario_keys: [], enabled, archived: false };
+        body = change;
+    }
+    const fields = ['world_revision', 'basis_revision', 'rubric', 'scenario_keys', 'enabled', 'archived'];
+    if (Object.keys(body).length !== fields.length || fields.some(key => !(key in body)))
+        throw new CliError('criterion_file_invalid', 'Use the complete criterion body: world_revision, basis_revision, rubric, scenario_keys, enabled and archived. No request was submitted.', 2);
+    await approval('criterion', options);
+    await remember('criterion', `${prefix}${item ? `/${item}` : ''}`, body, context, options, output);
+}
+export async function assessCommand(args, options, output) {
+    paging(options);
+    const scope = await selectedWorkspace({ ...options, interactive: interactive(options) });
+    if (['resume', 'forget'].includes(args[0]))
+        return recovery('assessment', args, scope, options, output);
+    if (!args.length || args.length > 2 || options.world || options.page || options.search)
+        throw new CliError('arguments_invalid', 'Use assess <campaign-id> [drill-id] with --file <rubric.json>, or run in a terminal for guided input.', 2);
+    const campaign = identifier(args[0]), prefix = `/api/workspaces/${scope.workspace}/campaigns/${campaign}`;
+    const auth = { actor: scope.owner, workspace: scope.workspace, signal: options.signal };
+    let trial = args[1] ? identifier(args[1]) : undefined;
+    if (!trial && interactive(options)) {
+        let number = 1;
+        while (!trial) {
+            const data = await api(`${prefix}/trials?page=${number}`, auth);
+            const choices = data.results.map(row => ({ value: row.id, label: `Drill ${row.position + 1}: ${row.scenario_name}`, detail: row.state }));
+            if (number > 1)
+                choices.unshift({ value: 'previous', label: 'Previous page', detail: '' });
+            if (data.next)
+                choices.push({ value: 'next', label: 'Next page', detail: '' });
+            if (!choices.length)
+                throw new CliError('trial_required', 'This simulation has no retained drills.');
+            const picked = await selectOne('Saved drill', choices, options.signal);
+            if (picked === 'next')
+                number++;
+            else if (picked === 'previous')
+                number--;
+            else
+                trial = identifier(picked);
+        }
+    }
+    if (!trial)
+        throw new CliError('trial_required', 'Choose a saved drill ID. This command does not run a simulation.', 2);
+    const path = `${prefix}/trials/${trial}/assessments`;
+    const data = await api(path, auth);
+    if (!data.can_assess)
+        throw new CliError('assessment_unavailable', 'Evaluation permission, retained evidence or evaluator configuration is unavailable. No assessment was requested.');
+    const definition = options.file ? await readJson(options.file) : interactive(options) ? await rubric(scope, identifier(data.agent_id), options.signal) : null;
+    if (!definition)
+        throw new CliError('rubric_file_required', 'Pass --file <rubric.json> containing the reviewed criterion.', 2);
+    await approval('assessment', options);
+    await remember('assessment', path, { rubric: definition, share_evidence: true, share_explanation: true }, scope, options, output);
+}