@relayflows/sdk 2.0.42 → 2.0.43

package/dist/flow-eval-local.d.ts

@@ -0,0 +1,54 @@
+import type { FlowEvalExecutor, FlowEvalRun } from './flow-eval-report.js';
+import type { RunExecution } from './cli/run.js';
+export interface LocalFlowEvalExecutorOptions {
+    /** Daemon data directory. Defaults to `.relayflowd`. */
+    dataDir?: string;
+    /** Attach a local agent/LLM worker, as `flows run --local-agent`. */
+    localAgent?: boolean;
+    agentCapacity?: number;
+    /** `{ spawn: false }` refuses instead of starting a daemon, as `flows run --no-spawn`. */
+    daemon?: {
+        spawn?: boolean;
+    };
+    /** The `flows` CLI entry point. Defaults to this package's own `dist/cli.js`. */
+    cliPath?: string;
+    /**
+     * Working directory for each run. Defaults to the snapshot directory holding
+     * the flow, so a relative read (`readFileSync('./prompt.txt')`) resolves
+     * inside the sealed copy — and fails there if the file is not part of the
+     * version — instead of reading the caller's mutable working tree. Setting it
+     * deliberately reintroduces that dependency.
+     */
+    cwd?: string;
+}
+/**
+ * Execute each case as a fresh local run, in its own `flows run` process.
+ *
+ * A child process per case is deliberate. Authored source is loaded with
+ * `import()`, and an ES module is cached by URL for the life of a process: an
+ * in-process run would keep executing the FIRST version of a `.flow.ts` it saw,
+ * while the report named the edited bytes on disk. A gate must run what it
+ * judges, so each case gets a process that has never seen the flow before.
+ *
+ * Authored `.flow.ts` receives the case input (`{}` when the case has none);
+ * declarative YAML/JSON specs take no input, so a case that declares one is
+ * refused rather than silently run without it.
+ */
+export declare function localFlowEvalExecutor(options?: LocalFlowEvalExecutorOptions): FlowEvalExecutor;
+/** Map a local `RunExecution` onto the eval vocabulary. Exported for tests. */
+export declare function localRunOutcome(execution: RunExecution): FlowEvalRun;
+/**
+ * Spend from the local journals.
+ *
+ * Summed from each step's own `step.completed` budget, never from a run's
+ * `budget_total`: an authored child run starts from the flow's `prior_spend`,
+ * so its total already includes every earlier child, and adding totals would
+ * count the same dollars repeatedly.
+ *
+ * An authored root runs each step as a child run with its own journal. The
+ * children are found in the root's `authored-steps` stream, which is written
+ * as each child opens and so survives a failed body, and in the root step's
+ * success output (`journalSteps`).
+ */
+export declare function localRunSpend(runId: string, dataDir: string): Promise<Pick<FlowEvalRun, 'costUsd' | 'tokensIn' | 'tokensOut' | 'steps'>>;
+//# sourceMappingURL=flow-eval-local.d.ts.map

package/dist/flow-eval-local.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-local.d.ts","sourceRoot":"","sources":["../src/flow-eval-local.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAgB,MAAM,uBAAuB,CAAC;AAEzF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEjD,MAAM,WAAW,4BAA4B;IAC3C,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0FAA0F;IAC1F,MAAM,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IAC7B,iFAAiF;IACjF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,4BAAiC,GAAG,gBAAgB,CAgClG;AAkCD,+EAA+E;AAC/E,wBAAgB,eAAe,CAAC,SAAS,EAAE,YAAY,GAAG,WAAW,CAMpE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC,CAAC,CAqB9I"}

package/dist/flow-eval-local.js

@@ -0,0 +1,220 @@
+// The local executor for `evaluateFlow`: one fresh `flows run` per case
+// against relayflowd, and the case's spend read back from the journals.
+// Memoization (`--reuse-from`) is never used: an evaluation must judge the
+// candidate's own step results, not a prior version's.
+import { spawn } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { canonicalize } from './canonical.js';
+import { isAuthoredFlowPath } from './direct-input.js';
+import { walkJournal } from './journal-reader.js';
+import { runReportOutcome } from './flow-eval-outcome.js';
+/**
+ * Execute each case as a fresh local run, in its own `flows run` process.
+ *
+ * A child process per case is deliberate. Authored source is loaded with
+ * `import()`, and an ES module is cached by URL for the life of a process: an
+ * in-process run would keep executing the FIRST version of a `.flow.ts` it saw,
+ * while the report named the edited bytes on disk. A gate must run what it
+ * judges, so each case gets a process that has never seen the flow before.
+ *
+ * Authored `.flow.ts` receives the case input (`{}` when the case has none);
+ * declarative YAML/JSON specs take no input, so a case that declares one is
+ * refused rather than silently run without it.
+ */
+export function localFlowEvalExecutor(options = {}) {
+    const dataDir = resolve(options.dataDir ?? '.relayflowd');
+    return async ({ flow, input, signal }) => {
+        if (!('path' in flow) || typeof flow.path !== 'string') {
+            throw new Error('The local executor runs flow files; write the spec to a file or use a custom executor.');
+        }
+        const authored = isAuthoredFlowPath(flow.path);
+        if (!authored && input !== undefined)
+            throw new Error('Declarative flows take no input; remove `input` from the case.');
+        const scratch = authored ? await mkdtemp(join(tmpdir(), 'flows-eval-')) : undefined;
+        try {
+            const args = [options.cliPath ?? defaultCliPath(), 'run', '--json', '--no-observer-link', '--data-dir', dataDir];
+            if (options.daemon?.spawn === false)
+                args.push('--no-spawn');
+            if (options.localAgent)
+                args.push('--local-agent');
+            if (options.agentCapacity !== undefined)
+                args.push('--agent-capacity', String(options.agentCapacity));
+            args.push(resolve(flow.path));
+            if (scratch !== undefined) {
+                // A file, never inline: an inline argument that happens to name a file
+                // in the working directory would be read as that file.
+                const inputPath = join(scratch, 'input.json');
+                await writeFile(inputPath, canonicalize(input ?? {}));
+                args.push('--input', inputPath);
+            }
+            const execution = await runCliProcess(args, options.cwd ?? dirname(resolve(flow.path)), signal);
+            const run = localRunOutcome(execution);
+            if (run.runId !== undefined && run.completionReason !== 'refused') {
+                Object.assign(run, await localRunSpend(run.runId, dataDir));
+            }
+            return run;
+        }
+        finally {
+            if (scratch !== undefined)
+                await rm(scratch, { recursive: true, force: true });
+        }
+    };
+}
+function defaultCliPath() {
+    // Built: this file sits beside cli.js in dist/. From source (tests), use the build.
+    const sibling = fileURLToPath(new URL('./cli.js', import.meta.url));
+    return existsSync(sibling) ? sibling : fileURLToPath(new URL('../dist/cli.js', import.meta.url));
+}
+function runCliProcess(args, cwd, signal) {
+    return new Promise((resolvePromise, reject) => {
+        const child = spawn(process.execPath, args, {
+            ...(cwd === undefined ? {} : { cwd }), stdio: ['ignore', 'pipe', 'pipe'],
+            ...(signal === undefined ? {} : { signal }),
+        });
+        let stdout = '';
+        let stderr = '';
+        child.stdout.setEncoding('utf8').on('data', (chunk) => { stdout += chunk; });
+        child.stderr.setEncoding('utf8').on('data', (chunk) => { stderr = (stderr + chunk).slice(-4_000); });
+        child.once('error', reject);
+        child.once('close', (code) => {
+            const line = stdout.trim().split('\n').reverse().find(candidate => candidate.startsWith('{'));
+            let report;
+            try {
+                report = line === undefined ? undefined : JSON.parse(line);
+            }
+            catch { /* reported below */ }
+            if (report === undefined || typeof report.ok !== 'boolean' || !Array.isArray(report.diagnostics)) {
+                reject(new Error(`flows run exited ${code} without a JSON report: ${stderr.trim() || stdout.trim() || '(no output)'}`));
+                return;
+            }
+            resolvePromise({ exitCode: (code ?? 1), report });
+        });
+    });
+}
+/** Map a local `RunExecution` onto the eval vocabulary. Exported for tests. */
+export function localRunOutcome(execution) {
+    const { report } = execution;
+    // The root, not the failing child: a failed authored run reports the child
+    // that failed as `runId`, and spend must be read from the whole run.
+    const runId = report.rootRunId ?? report.runId;
+    return { ...(runId === undefined ? {} : { runId }), ...runReportOutcome(report, execution.exitCode) };
+}
+/**
+ * Spend from the local journals.
+ *
+ * Summed from each step's own `step.completed` budget, never from a run's
+ * `budget_total`: an authored child run starts from the flow's `prior_spend`,
+ * so its total already includes every earlier child, and adding totals would
+ * count the same dollars repeatedly.
+ *
+ * An authored root runs each step as a child run with its own journal. The
+ * children are found in the root's `authored-steps` stream, which is written
+ * as each child opens and so survives a failed body, and in the root step's
+ * success output (`journalSteps`).
+ */
+export async function localRunSpend(runId, dataDir) {
+    try {
+        const spend = { dollars: 0, tokensIn: 0, tokensOut: 0 };
+        const steps = [];
+        const visited = new Set();
+        const queue = [{ runId, depth: 0 }];
+        while (queue.length > 0) {
+            const next = queue.shift();
+            if (visited.has(next.runId))
+                continue;
+            visited.add(next.runId);
+            const read = await readJournal(next.runId, dataDir);
+            addSpend(spend, read.spend);
+            steps.push(...read.steps);
+            // Bounded like `f.dispatch` (depth <= 3).
+            if (next.depth < 3)
+                for (const child of read.children)
+                    queue.push({ runId: child, depth: next.depth + 1 });
+        }
+        return { costUsd: spend.dollars === null ? null : roundUsd(spend.dollars), tokensIn: spend.tokensIn, tokensOut: spend.tokensOut, steps };
+    }
+    catch {
+        // Spend is evidence, not the verdict: an unreadable journal leaves cost unknown.
+        return { costUsd: null, tokensIn: null, tokensOut: null };
+    }
+}
+const AUTHORED_STEP_RECORD = 'relayflows.authored-step.v1';
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u;
+async function readJournal(runId, dataDir) {
+    const spend = { dollars: 0, tokensIn: 0, tokensOut: 0 };
+    const steps = new Map();
+    const children = new Set();
+    for await (const event of walkJournal(runId, dataDir)) {
+        const payload = (event.payload !== null && typeof event.payload === 'object' ? event.payload : {});
+        if (event.entry_type === 'step.attempt.started' && event.step_id !== null) {
+            const step = steps.get(event.step_id) ?? { id: event.step_id, status: 'running' };
+            step.startedAt ??= event.at_ms;
+            steps.set(event.step_id, step);
+        }
+        else if (event.entry_type === 'step.completed' && event.step_id !== null) {
+            const budget = readBudget(payload['budget']);
+            const step = steps.get(event.step_id) ?? { id: event.step_id, status: 'done' };
+            step.status = typeof payload['completionReason'] === 'string' ? payload['completionReason'] : 'done';
+            if (step.startedAt !== undefined)
+                step.durationMs = event.at_ms - step.startedAt;
+            if (budget !== undefined) {
+                step.costUsd = budget.dollars;
+                addSpend(spend, budget);
+            }
+            steps.set(event.step_id, step);
+            for (const child of journalStepRunIds(payload['output']))
+                children.add(child);
+        }
+        else if (event.entry_type === 'stream.appended') {
+            for (const child of authoredStepRunIds(payload))
+                children.add(child);
+        }
+    }
+    children.delete(runId);
+    return { spend, steps: [...steps.values()].map(({ startedAt: _startedAt, ...step }) => step), children: [...children] };
+}
+function journalStepRunIds(output) {
+    if (output === null || typeof output !== 'object')
+        return [];
+    const journalSteps = output['journalSteps'];
+    if (!Array.isArray(journalSteps))
+        return [];
+    return journalSteps.flatMap(step => {
+        const id = step !== null && typeof step === 'object' ? step['runId'] : undefined;
+        return typeof id === 'string' && RUN_ID.test(id) ? [id] : [];
+    });
+}
+/** Child run ids from an `authored-steps` index record, wherever the stream envelope nests it. */
+function authoredStepRunIds(value, depth = 0) {
+    if (value === null || typeof value !== 'object' || depth > 4)
+        return [];
+    const record = value;
+    if (record['index'] === AUTHORED_STEP_RECORD) {
+        return typeof record['runId'] === 'string' && RUN_ID.test(record['runId']) ? [record['runId']] : [];
+    }
+    return Object.values(record).flatMap(nested => authoredStepRunIds(nested, depth + 1));
+}
+function addSpend(into, add) {
+    into.tokensIn += add.tokensIn;
+    into.tokensOut += add.tokensOut;
+    into.dollars = into.dollars === null || add.dollars === null ? null : into.dollars + add.dollars;
+}
+function roundUsd(value) {
+    return Math.round(value * 1e6) / 1e6;
+}
+function readBudget(value) {
+    if (value === null || typeof value !== 'object')
+        return undefined;
+    const budget = value;
+    const dollars = typeof budget['dollars'] === 'string' ? Number(budget['dollars']) : NaN;
+    return {
+        dollars: budget['dollars_unmetered'] === true || !Number.isFinite(dollars) || dollars < 0 ? null : dollars,
+        tokensIn: typeof budget['tokens_in'] === 'number' ? budget['tokens_in'] : 0,
+        tokensOut: typeof budget['tokens_out'] === 'number' ? budget['tokens_out'] : 0,
+    };
+}
+//# sourceMappingURL=flow-eval-local.js.map

package/dist/flow-eval-local.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-local.js","sourceRoot":"","sources":["../src/flow-eval-local.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,wEAAwE;AACxE,2EAA2E;AAC3E,uDAAuD;AAEvD,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAuB1D;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAAwC,EAAE;IAC9E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,CAAC;IAC1D,OAAO,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;QACvC,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAC5G,CAAC;QACD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;QACxH,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACpF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,cAAc,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,oBAAoB,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;YACjH,IAAI,OAAO,CAAC,MAAM,EAAE,KAAK,KAAK,KAAK;gBAAE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC7D,IAAI,OAAO,CAAC,UAAU;gBAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACnD,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS;gBAAE,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;YACtG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,uEAAuE;gBACvE,uDAAuD;gBACvD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;gBAC9C,MAAM,SAAS,CAAC,SAAS,EAAE,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;gBACtD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAClC,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YAChG,MAAM,GAAG,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;gBAClE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,aAAa,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9D,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;gBAAS,CAAC;YACT,IAAI,OAAO,KAAK,SAAS;gBAAE,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc;IACrB,oFAAoF;IACpF,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACnG,CAAC;AAED,SAAS,aAAa,CAAC,IAAc,EAAE,GAAuB,EAAE,MAA+B;IAC7F,OAAO,IAAI,OAAO,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC1C,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACxE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC5C,CAAC,CAAC;QACH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7G,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9F,IAAI,MAA0C,CAAC;YAC/C,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2B,CAAC;YACvF,CAAC;YAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAC;YAChC,IAAI,MAAM,KAAK,SAAS,IAAI,OAAO,MAAM,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBACjG,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,IAAI,2BAA2B,MAAM,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,EAAE,IAAI,aAAa,EAAE,CAAC,CAAC,CAAC;gBACxH,OAAO;YACT,CAAC;YACD,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,IAAI,IAAI,CAAC,CAA6B,EAAE,MAAM,EAAE,CAAC,CAAC;QAChF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,eAAe,CAAC,SAAuB;IACrD,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAC7B,2EAA2E;IAC3E,qEAAqE;IACrE,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC;IAC/C,OAAO,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,gBAAgB,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;AACxG,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAa,EAAE,OAAe;IAChE,IAAI,CAAC;QACH,MAAM,KAAK,GAAU,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAmB,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,KAAK,GAA4C,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC5B,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAS;YACtC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACpD,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,0CAA0C;YAC1C,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC;gBAAE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ;oBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7G,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC;IAC3I,CAAC;IAAC,MAAM,CAAC;QACP,iFAAiF;QACjF,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC5D,CAAC;AACH,CAAC;AAID,MAAM,oBAAoB,GAAG,6BAA6B,CAAC;AAC3D,MAAM,MAAM,GAAG,8BAA8B,CAAC;AAE9C,KAAK,UAAU,WAAW,CAAC,KAAa,EAAE,OAAe;IACvD,MAAM,KAAK,GAAU,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAiD,CAAC;IACvE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAA4B,CAAC;QAC9H,IAAI,KAAK,CAAC,UAAU,KAAK,sBAAsB,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC1E,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;YAClF,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,KAAK,CAAC;YAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,KAAK,gBAAgB,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAC3E,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YAC/E,IAAI,CAAC,MAAM,GAAG,OAAO,OAAO,CAAC,kBAAkB,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACrG,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;gBAAE,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACjF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC1B,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAC/B,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,KAAK,CAAC,UAAU,KAAK,iBAAiB,EAAE,CAAC;YAClD,KAAK,MAAM,KAAK,IAAI,kBAAkB,CAAC,OAAO,CAAC;gBAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IACD,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC;AAC1H,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAe;IACxC,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAC7D,MAAM,YAAY,GAAI,MAAkC,CAAC,cAAc,CAAC,CAAC;IACzE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5C,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QACjC,MAAM,EAAE,GAAG,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAE,IAAgC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9G,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/D,CAAC,CAAC,CAAC;AACL,CAAC;AAED,kGAAkG;AAClG,SAAS,kBAAkB,CAAC,KAAc,EAAE,KAAK,GAAG,CAAC;IACnD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACxE,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,oBAAoB,EAAE,CAAC;QAC7C,OAAO,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACtG,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAW,EAAE,GAAU;IACvC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC;IAC9B,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC;IAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;AACnG,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,MAAM,OAAO,GAAG,OAAO,MAAM,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACxF,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,mBAAmB,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO;QAC1G,QAAQ,EAAE,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,SAAS,EAAE,OAAO,MAAM,CAAC,YAAY,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;KAC/E,CAAC;AACJ,CAAC"}

package/dist/flow-eval-outcome.d.ts

@@ -0,0 +1,22 @@
+import type { FlowEvalRun } from './flow-eval-report.js';
+/** The fields of a `RunReport` the verdict reads. */
+export interface RunReportVerdict {
+    ok?: unknown;
+    status?: unknown;
+    completionReason?: unknown;
+    completionDetail?: unknown;
+    parkCause?: unknown;
+    diagnostics?: unknown;
+}
+/**
+ * Map a run report onto the eval vocabulary.
+ *
+ * - `declined`: the report says `success` with a `run_declined` diagnostic —
+ *   the body chose not to act. Scoring it as `success` would let a flow that
+ *   did nothing pass a suite expecting it to act.
+ * - `worker_unavailable` vs `needs_human`: both park (exit 3); only the second
+ *   reached a human wait.
+ * - `refused`: preflight refused the run (exit 2); nothing executed.
+ */
+export declare function runReportOutcome(report: RunReportVerdict, exitCode?: number): Pick<FlowEvalRun, 'completionReason' | 'completionDetail'>;
+//# sourceMappingURL=flow-eval-outcome.d.ts.map

package/dist/flow-eval-outcome.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-outcome.d.ts","sourceRoot":"","sources":["../src/flow-eval-outcome.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,EAAE,CAAC,EAAE,OAAO,CAAC;IACb,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,gBAAgB,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,kBAAkB,GAAG,kBAAkB,CAAC,CAexI"}

package/dist/flow-eval-outcome.js

@@ -0,0 +1,37 @@
+// One vocabulary for a finished run's verdict, whoever ran it: a local
+// `flows run` report and a hosted run's stored result are the same RunReport
+// shape, and must map to the same completion reason, or a suite would pass
+// locally and fail on Cloud for the same flow behavior.
+/**
+ * Map a run report onto the eval vocabulary.
+ *
+ * - `declined`: the report says `success` with a `run_declined` diagnostic —
+ *   the body chose not to act. Scoring it as `success` would let a flow that
+ *   did nothing pass a suite expecting it to act.
+ * - `worker_unavailable` vs `needs_human`: both park (exit 3); only the second
+ *   reached a human wait.
+ * - `refused`: preflight refused the run (exit 2); nothing executed.
+ */
+export function runReportOutcome(report, exitCode) {
+    const diagnostics = Array.isArray(report.diagnostics)
+        ? report.diagnostics.filter((d) => d !== null && typeof d === 'object') : [];
+    const message = diagnostics.find(d => ['failure', 'refusal', 'parked'].includes(d['severity']))?.['message'];
+    const detail = typeof report.completionDetail === 'string' ? report.completionDetail
+        : typeof message === 'string' ? message : undefined;
+    let completionReason;
+    if (exitCode === 2)
+        completionReason = 'refused';
+    else if (report.status === 'parked' || exitCode === 3) {
+        completionReason = report.parkCause === 'worker_unavailable' ? 'worker_unavailable' : 'needs_human';
+    }
+    else if (report.ok === true && diagnostics.some(d => d['kind'] === 'run_declined'))
+        completionReason = 'declined';
+    else if (report.ok === true)
+        completionReason = 'success';
+    else if (typeof report.completionReason === 'string')
+        completionReason = report.completionReason;
+    else
+        completionReason = report.status === 'suspended' ? 'suspended' : 'failed';
+    return { completionReason, ...(detail === undefined ? {} : { completionDetail: detail }) };
+}
+//# sourceMappingURL=flow-eval-outcome.js.map

package/dist/flow-eval-outcome.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-outcome.js","sourceRoot":"","sources":["../src/flow-eval-outcome.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,6EAA6E;AAC7E,2EAA2E;AAC3E,wDAAwD;AAcxD;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAwB,EAAE,QAAiB;IAC1E,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC;QACnD,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAgC,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7G,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAW,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IACvH,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB;QAClF,CAAC,CAAC,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACtD,IAAI,gBAAwB,CAAC;IAC7B,IAAI,QAAQ,KAAK,CAAC;QAAE,gBAAgB,GAAG,SAAS,CAAC;SAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACtD,gBAAgB,GAAG,MAAM,CAAC,SAAS,KAAK,oBAAoB,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,aAAa,CAAC;IACtG,CAAC;SAAM,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,cAAc,CAAC;QAAE,gBAAgB,GAAG,UAAU,CAAC;SAC/G,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI;QAAE,gBAAgB,GAAG,SAAS,CAAC;SACrD,IAAI,OAAO,MAAM,CAAC,gBAAgB,KAAK,QAAQ;QAAE,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,CAAC;;QAC5F,gBAAgB,GAAG,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC/E,OAAO,EAAE,gBAAgB,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AAC7F,CAAC"}

package/dist/flow-eval-report.d.ts

@@ -0,0 +1,106 @@
+import type { FlowEvalCase } from './flow-eval-suite.js';
+import type { JsonValue } from './json-value.js';
+import type { FlowEvalTarget } from './flow-eval-version.js';
+export declare const FLOW_EVAL_REPORT_SCHEMA_VERSION = 1;
+/** One step of an executed case, as far as the executor can say. */
+export interface FlowEvalStep {
+    id: string;
+    status: string;
+    durationMs?: number | null;
+    costUsd?: number | null;
+}
+/** What an executor reports for one execution. */
+export interface FlowEvalRun {
+    runId?: string;
+    /**
+     * `success`, `declined`, `step_failed`, `budget_exceeded`, `canceled`,
+     * `needs_human`, `worker_unavailable` (parked with nothing attached to run
+     * it), or `refused` (preflight refused the run).
+     */
+    completionReason: string;
+    completionDetail?: string;
+    /** `null` when the executor cannot attest a dollar cost. */
+    costUsd?: number | null;
+    tokensIn?: number | null;
+    tokensOut?: number | null;
+    steps?: FlowEvalStep[];
+    /** Executor-supplied numeric metrics, merged into the case's metrics. */
+    metrics?: Record<string, number>;
+}
+export interface FlowEvalExecutionRequest {
+    /** The sealed snapshot to execute (a path inside it, or the in-memory spec), never the working tree. */
+    flow: FlowEvalTarget;
+    caseId: string;
+    /** Absent when the case declares no input. */
+    input?: JsonValue;
+    signal?: AbortSignal;
+}
+/** Executes the flow once. Throwing marks the case `error`, never `pass`. */
+export type FlowEvalExecutor = (request: FlowEvalExecutionRequest) => Promise<FlowEvalRun>;
+export interface FlowEvalScore {
+    /** Numeric value recorded under the scorer's name in the case's metrics. */
+    value?: number;
+    /** False fails the case. Absent means the scorer only measures. */
+    pass?: boolean;
+    detail?: string;
+}
+/** A caller-defined metric over one executed case. */
+export interface FlowEvalScorer {
+    name: string;
+    score(run: FlowEvalRun, testCase: FlowEvalCase): FlowEvalScore | Promise<FlowEvalScore>;
+}
+export type FlowEvalOutcome = 'pass' | 'fail' | 'error';
+export interface FlowEvalCaseResult {
+    id: string;
+    outcome: FlowEvalOutcome;
+    runId?: string;
+    completionReason?: string;
+    latencyMs: number;
+    costUsd: number | null;
+    tokensIn: number | null;
+    tokensOut: number | null;
+    metrics: Record<string, number>;
+    steps?: FlowEvalStep[];
+    /** Why the case did not pass; empty when it did. */
+    failures: string[];
+}
+export interface FlowEvalReport {
+    schemaVersion: typeof FLOW_EVAL_REPORT_SCHEMA_VERSION;
+    kind: 'flows.eval.report';
+    /** `files`: the local sources the version covers, relative to the entry's directory. */
+    flow: {
+        path?: string;
+        version: string;
+        files: string[];
+    };
+    suite: {
+        name: string;
+        sha256: string;
+        cases: number;
+    };
+    executor: string;
+    startedAt: string;
+    finishedAt: string;
+    cases: FlowEvalCaseResult[];
+    summary: {
+        total: number;
+        passed: number;
+        failed: number;
+        errored: number;
+        passRate: number;
+        /** `null` when any case's cost is unknown. */
+        totalCostUsd: number | null;
+        latencyMs: {
+            p50: number;
+            p95: number;
+            max: number;
+        };
+    };
+    /** The deploy decision: pass only when every threshold and regression check holds. */
+    gate: {
+        pass: boolean;
+        reasons: string[];
+        regressions: string[];
+    };
+}
+//# sourceMappingURL=flow-eval-report.d.ts.map

package/dist/flow-eval-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-report.d.ts","sourceRoot":"","sources":["../src/flow-eval-report.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE7D,eAAO,MAAM,+BAA+B,IAAI,CAAC;AAEjD,oEAAoE;AACpE,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,kDAAkD;AAClD,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC;IACvB,yEAAyE;IACzE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,wBAAwB;IACvC,wGAAwG;IACxG,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,6EAA6E;AAC7E,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,EAAE,wBAAwB,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAE3F,MAAM,WAAW,aAAa;IAC5B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,sDAAsD;AACtD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,GAAG,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,GAAG,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACzF;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAExD,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,eAAe,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC;IACvB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,OAAO,+BAA+B,CAAC;IACtD,IAAI,EAAE,mBAAmB,CAAC;IAC1B,wFAAwF;IACxF,IAAI,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC1D,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACvD,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,kBAAkB,EAAE,CAAC;IAC5B,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,8CAA8C;QAC9C,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,SAAS,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;KACtD,CAAC;IACF,sFAAsF;IACtF,IAAI,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;CACnE"}

package/dist/flow-eval-report.js

@@ -0,0 +1,4 @@
+// The data a flow evaluation exchanges: what an executor returns for one run,
+// and the report the evaluation produces. Types only.
+export const FLOW_EVAL_REPORT_SCHEMA_VERSION = 1;
+//# sourceMappingURL=flow-eval-report.js.map

package/dist/flow-eval-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-report.js","sourceRoot":"","sources":["../src/flow-eval-report.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,sDAAsD;AAMtD,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,CAAC"}

package/dist/flow-eval-suite.d.ts

@@ -0,0 +1,46 @@
+import { type JsonValue } from './json-value.js';
+export declare const MAX_FLOW_EVAL_CASES = 1000;
+/** What a case must observe to pass. Every field present must hold. */
+export interface FlowEvalExpectation {
+    /**
+     * The run's outcome. Defaults to `success`. `declined` is a successful run
+     * whose body chose not to act; `needs_human` is a parked run.
+     */
+    completionReason?: string;
+    /** Substrings the run's completion detail must contain. */
+    detailIncludes?: string[];
+    /** A regular expression (JavaScript syntax) the completion detail must match. */
+    detailMatches?: string;
+    /** Upper bound on this case's wall-clock latency. */
+    maxLatencyMs?: number;
+    /** Upper bound on this case's reported cost; an unknown cost fails it. */
+    maxCostUsd?: number;
+}
+export interface FlowEvalCase {
+    /** Stable identity across suite revisions; unique within the suite. */
+    id: string;
+    /** The run's input. Required by authored `.flow.ts`; refused by declarative specs on Cloud. */
+    input?: JsonValue;
+    expect?: FlowEvalExpectation;
+}
+export interface FlowEvalThresholds {
+    /** Fraction of cases that must pass, 0..1. Defaults to 1: every case. */
+    minPassRate?: number;
+    /** Ceiling on the summed cost of every case; an unknown cost fails it. */
+    maxTotalCostUsd?: number;
+    /** Ceiling on the 95th-percentile case latency. */
+    maxP95LatencyMs?: number;
+}
+export interface FlowEvalSuite {
+    name: string;
+    cases: FlowEvalCase[];
+    thresholds?: FlowEvalThresholds;
+}
+/** Validate untrusted suite data (parsed JSON) into a frozen suite. */
+export declare function parseFlowEvalSuite(value: unknown): FlowEvalSuite;
+/** Read and validate a suite file (JSON). */
+export declare function loadFlowEvalSuite(path: string): Promise<FlowEvalSuite>;
+/** The suite's identity: sha256 of its canonical JSON. */
+export declare function flowEvalSuiteSha256(suite: FlowEvalSuite): string;
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+//# sourceMappingURL=flow-eval-suite.d.ts.map

package/dist/flow-eval-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-suite.d.ts","sourceRoot":"","sources":["../src/flow-eval-suite.ts"],"names":[],"mappings":"AAOA,OAAO,EAA8C,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI7F,eAAO,MAAM,mBAAmB,OAAQ,CAAC;AAKzC,uEAAuE;AACvE,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,iFAAiF;IACjF,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAC3B,uEAAuE;IACvE,EAAE,EAAE,MAAM,CAAC;IACX,+FAA+F;IAC/F,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,MAAM,CAAC,EAAE,mBAAmB,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IACjC,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0EAA0E;IAC1E,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED,uEAAuE;AACvE,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,aAAa,CAiDhE;AAED,6CAA6C;AAC7C,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAc5E;AAED,0DAA0D;AAC1D,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CAEhE;AAsCD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAEzE"}