@relayflows/sdk 2.0.42 → 2.0.43

package/dist/flow-eval-suite.js

@@ -0,0 +1,144 @@
+// The frozen input set of a flow evaluation: its shape, strict parsing of
+// untrusted suite JSON, and its identity hash.
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { canonicalize } from './canonical.js';
+import { FlowEvalError } from './flow-eval-error.js';
+import { snapshotJsonValue } from './json-value.js';
+import { MAX_DIRECT_INPUT_BYTES } from './direct-input.js';
+import { deepFreeze } from './flow-eval-version.js';
+export const MAX_FLOW_EVAL_CASES = 1_000;
+/** One case's input: no larger than `flows run --input` accepts. */
+const INPUT_LIMITS = Object.freeze({ maxDepth: 64, maxNodes: 262_144, maxBytes: MAX_DIRECT_INPUT_BYTES });
+/** Validate untrusted suite data (parsed JSON) into a frozen suite. */
+export function parseFlowEvalSuite(value) {
+    const fail = (message) => { throw new FlowEvalError('invalid_suite', message); };
+    if (!isRecord(value))
+        fail('Suite must be a JSON object.');
+    const raw = value;
+    allowOnly(raw, ['name', 'cases', 'thresholds'], 'suite', fail);
+    if (typeof raw.name !== 'string' || raw.name.trim() === '')
+        fail('Suite needs a nonempty `name`.');
+    if (!Array.isArray(raw.cases) || raw.cases.length === 0)
+        fail('Suite needs a nonempty `cases` array.');
+    const rawCases = raw.cases;
+    if (rawCases.length > MAX_FLOW_EVAL_CASES)
+        fail(`Suite has ${rawCases.length} cases; the limit is ${MAX_FLOW_EVAL_CASES}.`);
+    const seen = new Set();
+    const cases = rawCases.map((entry, index) => {
+        const at = `cases[${index}]`;
+        if (!isRecord(entry))
+            fail(`${at} must be an object.`);
+        const item = entry;
+        allowOnly(item, ['id', 'input', 'expect'], at, fail);
+        if (typeof item.id !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/u.test(item.id)) {
+            fail(`${at}.id must be 1-128 characters of letters, digits, '.', '_', ':' or '-'.`);
+        }
+        const id = item.id;
+        if (seen.has(id))
+            fail(`${at}.id "${id}" is not unique.`);
+        seen.add(id);
+        const testCase = { id };
+        if (Object.prototype.hasOwnProperty.call(item, 'input')) {
+            try {
+                testCase.input = snapshotJsonValue(item.input, `${at}.input`, INPUT_LIMITS);
+            }
+            catch (error) {
+                fail(`${at}.input is not JSON data: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        if (item.expect !== undefined)
+            testCase.expect = parseExpectation(item.expect, `${at}.expect`, fail);
+        return testCase;
+    });
+    const suite = { name: raw.name, cases };
+    if (raw.thresholds !== undefined) {
+        if (!isRecord(raw.thresholds))
+            fail('`thresholds` must be an object.');
+        const t = raw.thresholds;
+        allowOnly(t, ['minPassRate', 'maxTotalCostUsd', 'maxP95LatencyMs'], 'thresholds', fail);
+        const thresholds = {};
+        if (t.minPassRate !== undefined) {
+            if (typeof t.minPassRate !== 'number' || !(t.minPassRate >= 0 && t.minPassRate <= 1))
+                fail('`thresholds.minPassRate` must be a number from 0 to 1.');
+            thresholds.minPassRate = t.minPassRate;
+        }
+        if (t.maxTotalCostUsd !== undefined)
+            thresholds.maxTotalCostUsd = nonNegative(t.maxTotalCostUsd, 'thresholds.maxTotalCostUsd', fail);
+        if (t.maxP95LatencyMs !== undefined)
+            thresholds.maxP95LatencyMs = nonNegative(t.maxP95LatencyMs, 'thresholds.maxP95LatencyMs', fail);
+        suite.thresholds = thresholds;
+    }
+    // Frozen in fact, not just by name: a scorer receives these cases, and must
+    // not be able to rename one or loosen an expectation for the cases after it.
+    return deepFreeze(suite);
+}
+/** Read and validate a suite file (JSON). */
+export async function loadFlowEvalSuite(path) {
+    let text;
+    try {
+        text = await readFile(path, 'utf8');
+    }
+    catch {
+        throw new FlowEvalError('invalid_suite', `Suite file "${path}" is not readable.`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        throw new FlowEvalError('invalid_suite', `Suite file "${path}" is not valid JSON.`);
+    }
+    return parseFlowEvalSuite(parsed);
+}
+/** The suite's identity: sha256 of its canonical JSON. */
+export function flowEvalSuiteSha256(suite) {
+    return `sha256:${createHash('sha256').update(canonicalize(suite)).digest('hex')}`;
+}
+function parseExpectation(value, at, fail) {
+    if (!isRecord(value))
+        fail(`${at} must be an object.`);
+    const raw = value;
+    allowOnly(raw, ['completionReason', 'detailIncludes', 'detailMatches', 'maxLatencyMs', 'maxCostUsd'], at, fail);
+    const expect = {};
+    if (raw.completionReason !== undefined) {
+        if (typeof raw.completionReason !== 'string' || raw.completionReason === '')
+            fail(`${at}.completionReason must be a nonempty string.`);
+        expect.completionReason = raw.completionReason;
+    }
+    if (raw.detailIncludes !== undefined) {
+        if (!Array.isArray(raw.detailIncludes) || raw.detailIncludes.some(item => typeof item !== 'string'))
+            fail(`${at}.detailIncludes must be an array of strings.`);
+        expect.detailIncludes = [...raw.detailIncludes];
+    }
+    if (raw.detailMatches !== undefined) {
+        if (typeof raw.detailMatches !== 'string')
+            fail(`${at}.detailMatches must be a string.`);
+        try {
+            new RegExp(raw.detailMatches, 'u');
+        }
+        catch {
+            fail(`${at}.detailMatches is not a valid regular expression.`);
+        }
+        expect.detailMatches = raw.detailMatches;
+    }
+    if (raw.maxLatencyMs !== undefined)
+        expect.maxLatencyMs = nonNegative(raw.maxLatencyMs, `${at}.maxLatencyMs`, fail);
+    if (raw.maxCostUsd !== undefined)
+        expect.maxCostUsd = nonNegative(raw.maxCostUsd, `${at}.maxCostUsd`, fail);
+    return expect;
+}
+function allowOnly(raw, keys, at, fail) {
+    for (const key of Object.keys(raw))
+        if (!keys.includes(key))
+            fail(`${at} has unknown field "${key}".`);
+}
+function nonNegative(value, at, fail) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
+        fail(`\`${at}\` must be a non-negative number.`);
+    return value;
+}
+export function isRecord(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+//# sourceMappingURL=flow-eval-suite.js.map

package/dist/flow-eval-suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-suite.js","sourceRoot":"","sources":["../src/flow-eval-suite.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAC1E,+CAA+C;AAE/C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAA2C,MAAM,iBAAiB,CAAC;AAC7F,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAEzC,oEAAoE;AACpE,MAAM,YAAY,GAAuB,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,sBAAsB,EAAE,CAAC,CAAC;AA0C9H,uEAAuE;AACvE,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,MAAM,IAAI,GAAG,CAAC,OAAe,EAAS,EAAE,GAAG,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC3D,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,SAAS,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC/D,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACnG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,IAAI,CAAC,uCAAuC,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAkB,CAAC;IACxC,IAAI,QAAQ,CAAC,MAAM,GAAG,mBAAmB;QAAE,IAAI,CAAC,aAAa,QAAQ,CAAC,MAAM,wBAAwB,mBAAmB,GAAG,CAAC,CAAC;IAC5H,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAgB,EAAE;QACxD,MAAM,EAAE,GAAG,SAAS,KAAK,GAAG,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,IAAI,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,KAAgC,CAAC;QAC9C,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QACrD,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,sCAAsC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACzF,IAAI,CAAC,GAAG,EAAE,wEAAwE,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAY,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAC1D,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,MAAM,QAAQ,GAAiB,EAAE,EAAE,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC;gBACH,QAAQ,CAAC,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,GAAG,EAAE,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClG,CAAC;QACH,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,QAAQ,CAAC,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QACrG,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAkB,EAAE,IAAI,EAAE,GAAG,CAAC,IAAc,EAAE,KAAK,EAAE,CAAC;IACjE,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACvE,MAAM,CAAC,GAAG,GAAG,CAAC,UAAqC,CAAC;QACpD,SAAS,CAAC,CAAC,EAAE,CAAC,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QACxF,MAAM,UAAU,GAAuB,EAAE,CAAC;QAC1C,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;gBAAE,IAAI,CAAC,wDAAwD,CAAC,CAAC;YACrJ,UAAU,CAAC,WAAW,GAAG,CAAC,CAAC,WAAqB,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS;YAAE,UAAU,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,eAAe,EAAE,4BAA4B,EAAE,IAAI,CAAC,CAAC;QACrI,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS;YAAE,UAAU,CAAC,eAAe,GAAG,WAAW,CAAC,CAAC,CAAC,eAAe,EAAE,4BAA4B,EAAE,IAAI,CAAC,CAAC;QACrI,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;IAChC,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,6CAA6C;AAC7C,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAClD,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,eAAe,IAAI,oBAAoB,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,eAAe,IAAI,sBAAsB,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,mBAAmB,CAAC,KAAoB;IACtD,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,KAA6B,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC5G,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,EAAU,EAAE,IAAgC;IACpF,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,IAAI,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,SAAS,CAAC,GAAG,EAAE,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,eAAe,EAAE,cAAc,EAAE,YAAY,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;IAChH,MAAM,MAAM,GAAwB,EAAE,CAAC;IACvC,IAAI,GAAG,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,OAAO,GAAG,CAAC,gBAAgB,KAAK,QAAQ,IAAI,GAAG,CAAC,gBAAgB,KAAK,EAAE;YAAE,IAAI,CAAC,GAAG,EAAE,8CAA8C,CAAC,CAAC;QACvI,MAAM,CAAC,gBAAgB,GAAG,GAAG,CAAC,gBAA0B,CAAC;IAC3D,CAAC;IACD,IAAI,GAAG,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;YAAE,IAAI,CAAC,GAAG,EAAE,8CAA8C,CAAC,CAAC;QAC/J,MAAM,CAAC,cAAc,GAAG,CAAC,GAAI,GAAG,CAAC,cAA2B,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,QAAQ;YAAE,IAAI,CAAC,GAAG,EAAE,kCAAkC,CAAC,CAAC;QACzF,IAAI,CAAC;YACH,IAAI,MAAM,CAAC,GAAG,CAAC,aAAuB,EAAE,GAAG,CAAC,CAAC;QAC/C,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,GAAG,EAAE,mDAAmD,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,aAAuB,CAAC;IACrD,CAAC;IACD,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;QAAE,MAAM,CAAC,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;IACpH,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;QAAE,MAAM,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;IAC5G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,GAA4B,EAAE,IAAuB,EAAE,EAAU,EAAE,IAAgC;IACpH,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI,CAAC,CAAC;AACzG,CAAC;AAED,SAAS,WAAW,CAAC,KAAc,EAAE,EAAU,EAAE,IAAgC;IAC/E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,IAAI,CAAC,KAAK,EAAE,mCAAmC,CAAC,CAAC;IACxH,OAAO,KAAe,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/flow-eval-version.d.ts

@@ -0,0 +1,59 @@
+import type { FlowSpec } from './spec.js';
+/** A flow source path (`.flow.ts`, `.flow.yaml`, spec JSON) or an in-memory spec. */
+export type FlowEvalTarget = {
+    path: string;
+} | FlowSpec;
+export interface FlowEvalVersion {
+    /** `sha256:<hex>` over the manifest (or the kernel spec hash for an in-memory spec). */
+    version: string;
+    /** Manifest paths, relative to the entry's directory; empty for an in-memory spec. */
+    files: string[];
+}
+/** Most local source files one version may span; past it the evaluation is refused, not truncated. */
+export declare const MAX_FLOW_EVAL_SOURCES = 2000;
+export declare function isPathTarget(flow: FlowEvalTarget): flow is {
+    path: string;
+};
+/** The version string alone. */
+export declare function flowEvalVersion(flow: FlowEvalTarget): Promise<string>;
+/** The version and the files it covers. */
+export declare function flowEvalSources(flow: FlowEvalTarget): Promise<FlowEvalVersion>;
+/** The sealed bytes a version names, from which every case gets its own copy. */
+export interface FlowEvalSnapshot extends FlowEvalVersion {
+    /**
+     * A fresh, read-only copy for one case: its own directory, files and
+     * directories without write permission, written from the buffers the
+     * version hashed. An in-memory spec yields a deep-frozen clone.
+     */
+    materialize(): Promise<{
+        flow: FlowEvalTarget;
+        dispose(): Promise<void>;
+    }>;
+}
+/**
+ * Seal the flow before any case runs, and execute only copies of the seal.
+ *
+ * Hashing the working tree and then running it would let the tree change
+ * between the two — an edit made after a check and restored before the next
+ * would run bytes the report never named. The sources are read once, hashed,
+ * and held in memory; each case then runs a fresh read-only copy written from
+ * those buffers, so nothing a case does to its copy reaches the next case. A
+ * local module the manifest could not see (a computed
+ * `import(\`./rules/${kind}.js\`)`, a file read at runtime) is simply absent
+ * from the copy, so such a run fails instead of executing unjudged code.
+ *
+ * Trust boundary: evaluated code runs as the caller, so it can still reach
+ * anything the caller can — including `node_modules`, which is linked rather
+ * than copied and pinned by the lockfile in the manifest. Containing hostile
+ * code needs a sandbox; this seal makes sure the code that runs is the code
+ * the version names.
+ */
+export declare function sealFlowEvalSnapshot(flow: FlowEvalTarget): Promise<FlowEvalSnapshot>;
+export declare function deepFreeze<T>(value: T): T;
+/**
+ * Relative module specifiers and relative `.flow.ts` string literals (the
+ * `use:` header) in a source file. A lexical scan, deliberately: computing a
+ * version must never execute the flow.
+ */
+export declare function localSpecifiers(source: string): string[];
+//# sourceMappingURL=flow-eval-version.d.ts.map

package/dist/flow-eval-version.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-version.d.ts","sourceRoot":"","sources":["../src/flow-eval-version.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,qFAAqF;AACrF,MAAM,MAAM,cAAc,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,QAAQ,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,wFAAwF;IACxF,OAAO,EAAE,MAAM,CAAC;IAChB,sFAAsF;IACtF,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,sGAAsG;AACtG,eAAO,MAAM,qBAAqB,OAAQ,CAAC;AAK3C,wBAAgB,YAAY,CAAC,IAAI,EAAE,cAAc,GAAG,IAAI,IAAI;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAE3E;AAED,gCAAgC;AAChC,wBAAsB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAE3E;AAED,2CAA2C;AAC3C,wBAAsB,eAAe,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC,CAIpF;AAED,iFAAiF;AACjF,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD;;;;OAIG;IACH,WAAW,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,CAAC;QAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC,CAAC;CAC5E;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAwC1F;AAkBD,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAMzC;AA4HD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAUxD"}

package/dist/flow-eval-version.js

@@ -0,0 +1,303 @@
+// What a flow evaluation judges, by content: the flow version.
+//
+// For a flow file the version is the sha256 of a manifest of every local
+// source the run reads, not of the entry file alone. Editing a helper module,
+// a `use`d flow, or the project's extension lock changes what executes, so it
+// must change the version too — otherwise `expectVersion` would accept code it
+// never judged. Packages under node_modules are not part of the manifest; they
+// are pinned by the project's own lockfile, which is. An evaluation executes a
+// sealed copy of exactly these bytes (`sealFlowEvalSnapshot`).
+import { createHash } from 'node:crypto';
+import { existsSync, statSync } from 'node:fs';
+import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rm, symlink, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { basename, dirname, join, relative, resolve, sep } from 'node:path';
+import { canonicalize, specHash } from './canonical.js';
+import { compileSpec, toKernelSpec } from './compile.js';
+import { FlowEvalError } from './flow-eval-error.js';
+import { parsePluginLock } from './plugin-lock.js';
+import { pluginStoreDirectory } from './plugin-store.js';
+/** Most local source files one version may span; past it the evaluation is refused, not truncated. */
+export const MAX_FLOW_EVAL_SOURCES = 2_000;
+const SOURCE_EXTENSIONS = ['.ts', '.mts', '.cts', '.js', '.mjs', '.cjs', '.json'];
+const PROJECT_FILES = ['package.json', 'flows.json', 'flows.lock.json', 'package-lock.json', 'bun.lock', 'pnpm-lock.yaml', 'yarn.lock'];
+export function isPathTarget(flow) {
+    return typeof flow.path === 'string' && !('steps' in flow);
+}
+/** The version string alone. */
+export async function flowEvalVersion(flow) {
+    return (await flowEvalSources(flow)).version;
+}
+/** The version and the files it covers. */
+export async function flowEvalSources(flow) {
+    if (!isPathTarget(flow))
+        return { version: `sha256:${specHash(toKernelSpec(compileSpec(flow)))}`, files: [] };
+    const { version, files } = await collectSources(flow.path);
+    return { version, files: files.map(file => file.path) };
+}
+/**
+ * Seal the flow before any case runs, and execute only copies of the seal.
+ *
+ * Hashing the working tree and then running it would let the tree change
+ * between the two — an edit made after a check and restored before the next
+ * would run bytes the report never named. The sources are read once, hashed,
+ * and held in memory; each case then runs a fresh read-only copy written from
+ * those buffers, so nothing a case does to its copy reaches the next case. A
+ * local module the manifest could not see (a computed
+ * `import(\`./rules/${kind}.js\`)`, a file read at runtime) is simply absent
+ * from the copy, so such a run fails instead of executing unjudged code.
+ *
+ * Trust boundary: evaluated code runs as the caller, so it can still reach
+ * anything the caller can — including `node_modules`, which is linked rather
+ * than copied and pinned by the lockfile in the manifest. Containing hostile
+ * code needs a sandbox; this seal makes sure the code that runs is the code
+ * the version names.
+ */
+export async function sealFlowEvalSnapshot(flow) {
+    if (!isPathTarget(flow)) {
+        // Hash and execute the same detached copy: a caller mutating its object
+        // later cannot change what the remaining cases run.
+        const frozen = deepFreeze(structuredClone(flow));
+        return { ...(await flowEvalSources(frozen)), materialize: async () => ({ flow: frozen, dispose: async () => { } }) };
+    }
+    const sources = await collectSources(flow.path);
+    // Every node_modules Node would consult from the entry, at the same relative
+    // place: a nested package resolves some dependencies from higher levels.
+    const modules = ancestorNodeModules(dirname(sources.entry));
+    const root = commonDirectory([...sources.absolute.map(file => file.path), ...modules.map(dir => join(dir, '.'))]);
+    return {
+        version: sources.version,
+        files: sources.files.map(file => file.path),
+        materialize: async () => {
+            const directory = await mkdtemp(join(tmpdir(), 'flows-eval-snapshot-'));
+            const dispose = async () => {
+                await makeWritable(directory);
+                await rm(directory, { recursive: true, force: true });
+            };
+            try {
+                for (const file of sources.absolute) {
+                    const target = join(directory, relative(root, file.path));
+                    await mkdir(dirname(target), { recursive: true });
+                    await writeFile(target, file.bytes, { mode: 0o444 });
+                }
+                for (const dir of modules) {
+                    const link = join(directory, relative(root, dir));
+                    await mkdir(dirname(link), { recursive: true });
+                    await symlink(dir, link, 'dir');
+                }
+                await makeReadOnly(directory);
+            }
+            catch (error) {
+                await dispose();
+                throw error;
+            }
+            return { flow: { path: join(directory, relative(root, sources.entry)) }, dispose };
+        },
+    };
+}
+async function makeReadOnly(directory) {
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+        if (entry.isDirectory())
+            await makeReadOnly(join(directory, entry.name));
+    }
+    await chmod(directory, 0o555);
+}
+async function makeWritable(directory) {
+    try {
+        await chmod(directory, 0o755);
+        for (const entry of await readdir(directory, { withFileTypes: true })) {
+            if (entry.isDirectory())
+                await makeWritable(join(directory, entry.name));
+        }
+    }
+    catch { /* already gone */ }
+}
+export function deepFreeze(value) {
+    if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) {
+        Object.freeze(value);
+        for (const nested of Object.values(value))
+            deepFreeze(nested);
+    }
+    return value;
+}
+async function collectSources(path) {
+    let entry;
+    try {
+        entry = await realpath(path);
+    }
+    catch {
+        throw new FlowEvalError('unreadable_flow', `Flow "${path}" is not readable.`);
+    }
+    const root = dirname(entry);
+    const contents = new Map();
+    const missing = new Set();
+    // One cap over everything sealed — sources, manifests, extension payloads —
+    // so no category can grow a version past it.
+    const add = async (file) => {
+        const known = contents.get(file);
+        if (known !== undefined)
+            return known;
+        if (contents.size >= MAX_FLOW_EVAL_SOURCES) {
+            throw new FlowEvalError('unreadable_flow', `Flow "${path}" reaches more than ${MAX_FLOW_EVAL_SOURCES} local files.`);
+        }
+        let bytes;
+        try {
+            bytes = await readFile(file);
+        }
+        catch {
+            throw new FlowEvalError('unreadable_flow', `Flow source "${file}" is not readable.`);
+        }
+        contents.set(file, bytes);
+        return bytes;
+    };
+    const pending = [entry];
+    while (pending.length > 0) {
+        const file = pending.pop();
+        if (contents.has(file))
+            continue;
+        const bytes = await add(file);
+        // The package scope that decides how this file loads (`"type"`), per file:
+        // a nested package.json can differ from the entry's.
+        const scope = nearestFile(dirname(file), 'package.json');
+        if (scope !== undefined)
+            await add(scope);
+        if (!/\.(?:[mc]?[jt]s)$/u.test(file))
+            continue;
+        for (const specifier of localSpecifiers(bytes.toString('utf8'))) {
+            const targets = resolveLocal(dirname(file), specifier);
+            if (targets.length === 0)
+                missing.add(`${relative(root, dirname(file)) || '.'}:${specifier}`);
+            for (const target of targets)
+                if (!contents.has(target))
+                    pending.push(target);
+        }
+    }
+    // The nearest of each: package.json decides module semantics, flows.json and
+    // its lock the extensions, and a package lockfile pins node_modules. Each is
+    // found by walking up on its own, so a lockfile beside package.json (not
+    // beside flows.json) still pins the version.
+    for (const name of PROJECT_FILES) {
+        const file = nearestFile(root, name);
+        if (file !== undefined)
+            await add(file);
+    }
+    // Locked flow extensions load from the project's content-addressed store,
+    // so their payloads are part of what runs: hash and seal them too.
+    const lockPath = nearestFile(root, 'flows.lock.json');
+    if (lockPath !== undefined) {
+        let lock;
+        try {
+            lock = parsePluginLock(JSON.parse(contents.get(lockPath).toString('utf8')));
+        }
+        catch (error) {
+            throw new FlowEvalError('unreadable_flow', `flows.lock.json is not a valid plugin lock: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        for (const entry of lock.plugins) {
+            const store = pluginStoreDirectory(dirname(lockPath), entry.name, entry.digest);
+            if (!existsSync(store)) {
+                throw new FlowEvalError('unreadable_flow', `Locked extension ${entry.name} is not materialized at ${store}; run \`flows plugin verify\`.`);
+            }
+            for (const file of await listFiles(store))
+                await add(file);
+        }
+    }
+    const absolute = [...contents.entries()].map(([file, bytes]) => ({ path: file, bytes }));
+    const files = absolute
+        .map(file => ({ path: relative(root, file.path) || basename(file.path), sha256: sha256(file.bytes) }))
+        .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    // An unresolved import is part of the identity too: creating that file later changes what runs.
+    const manifest = { entry: basename(entry), files, missing: [...missing].sort() };
+    return { entry, version: `sha256:${sha256(canonicalize(manifest))}`, files, absolute };
+}
+async function listFiles(directory) {
+    const files = [];
+    for (const entry of await readdir(directory, { withFileTypes: true })) {
+        const path = join(directory, entry.name);
+        if (entry.isDirectory())
+            files.push(...await listFiles(path));
+        else if (entry.isFile())
+            files.push(path);
+    }
+    return files;
+}
+function commonDirectory(paths) {
+    let common = dirname(paths[0]);
+    for (const path of paths) {
+        while (common !== dirname(common) && path !== common && !path.startsWith(common + sep))
+            common = dirname(common);
+    }
+    return common;
+}
+function nearestFile(start, name) {
+    for (let directory = start;; directory = dirname(directory)) {
+        const candidate = join(directory, name);
+        if (existsSync(candidate))
+            return candidate;
+        if (dirname(directory) === directory)
+            return undefined;
+    }
+}
+function ancestorNodeModules(start) {
+    const found = [];
+    for (let directory = start;; directory = dirname(directory)) {
+        const candidate = join(directory, 'node_modules');
+        try {
+            if (statSync(candidate).isDirectory())
+                found.push(candidate);
+        }
+        catch { /* none here */ }
+        if (dirname(directory) === directory)
+            return found;
+    }
+}
+/**
+ * Relative module specifiers and relative `.flow.ts` string literals (the
+ * `use:` header) in a source file. A lexical scan, deliberately: computing a
+ * version must never execute the flow.
+ */
+export function localSpecifiers(source) {
+    const found = new Set();
+    const patterns = [
+        /\b(?:import|export)\s+(?:type\s+)?[^'"`;]*?\bfrom\s*['"](\.{1,2}\/[^'"]+)['"]/gu,
+        /\bimport\s*['"](\.{1,2}\/[^'"]+)['"]/gu,
+        /\b(?:import|require)\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]\s*\)/gu,
+        /['"](\.{1,2}\/[^'"\s]+\.flow\.ts)['"]/gu,
+    ];
+    for (const pattern of patterns)
+        for (const match of source.matchAll(pattern))
+            found.add(match[1]);
+    return [...found];
+}
+/**
+ * The files a relative specifier can load. Where a TypeScript sibling and a
+ * JavaScript file both match (`./x.js` beside `./x.ts`), both are returned:
+ * which one a loader picks is the loader's business, and the version must not
+ * hash one while the run executes the other.
+ */
+function resolveLocal(from, specifier) {
+    const base = resolve(from, specifier.replace(/[?#].*$/u, ''));
+    const isFile = (candidate) => {
+        try {
+            return statSync(candidate).isFile();
+        }
+        catch {
+            return false;
+        }
+    };
+    // TypeScript's NodeNext convention: import './x.js' names ./x.ts.
+    const swapped = base.replace(/\.([mc]?)js$/u, '.$1ts');
+    const direct = [...new Set([swapped, base])].filter(isFile);
+    if (direct.length > 0)
+        return direct;
+    for (const candidate of [
+        ...SOURCE_EXTENSIONS.map(extension => base + extension),
+        ...SOURCE_EXTENSIONS.map(extension => join(base, `index${extension}`)),
+    ])
+        if (isFile(candidate))
+            return [candidate];
+    return [];
+}
+function sha256(data) {
+    return createHash('sha256').update(data).digest('hex');
+}
+//# sourceMappingURL=flow-eval-version.js.map

package/dist/flow-eval-version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval-version.js","sourceRoot":"","sources":["../src/flow-eval-version.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,EAAE;AACF,yEAAyE;AACzE,8EAA8E;AAC9E,8EAA8E;AAC9E,+EAA+E;AAC/E,+EAA+E;AAC/E,+EAA+E;AAC/E,+DAA+D;AAE/D,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9G,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAmB,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAazD,sGAAsG;AACtG,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AAE3C,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAClF,MAAM,aAAa,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,UAAU,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC;AAExI,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,OAAQ,IAA2B,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC;AACrF,CAAC;AAED,gCAAgC;AAChC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAoB;IACxD,OAAO,CAAC,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC;AAC/C,CAAC;AAED,2CAA2C;AAC3C,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAoB;IACxD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,UAAU,QAAQ,CAAC,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAC9G,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC1D,CAAC;AAYD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAoB;IAC7D,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,wEAAwE;QACxE,oDAAoD;QACpD,MAAM,MAAM,GAAG,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QACjD,OAAO,EAAE,GAAG,CAAC,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,GAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IACtH,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChD,6EAA6E;IAC7E,yEAAyE;IACzE,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,eAAe,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClH,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3C,WAAW,EAAE,KAAK,IAAI,EAAE;YACtB,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;YACxE,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;gBACxC,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;gBAC9B,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACxD,CAAC,CAAC;YACF,IAAI,CAAC;gBACH,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;oBACpC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC1D,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAClD,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBACvD,CAAC;gBACD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;oBAClD,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAChD,MAAM,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;gBACD,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC;YACd,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC;QACrF,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC3C,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACtE,IAAI,KAAK,CAAC,WAAW,EAAE;gBAAE,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAI,KAAQ;IACpC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3E,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrB,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,KAAgC,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AASD,KAAK,UAAU,cAAc,CAAC,IAAY;IACxC,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,SAAS,IAAI,oBAAoB,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,4EAA4E;IAC5E,6CAA6C;IAC7C,MAAM,GAAG,GAAG,KAAK,EAAE,IAAY,EAAmB,EAAE;QAClD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,IAAI,QAAQ,CAAC,IAAI,IAAI,qBAAqB,EAAE,CAAC;YAC3C,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,SAAS,IAAI,uBAAuB,qBAAqB,eAAe,CAAC,CAAC;QACvH,CAAC;QACD,IAAI,KAAa,CAAC;QAClB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,gBAAgB,IAAI,oBAAoB,CAAC,CAAC;QACvF,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC;IACxB,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAG,CAAC;QAC5B,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QACjC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,2EAA2E;QAC3E,qDAAqD;QACrD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc,CAAC,CAAC;QACzD,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC/C,KAAK,MAAM,SAAS,IAAI,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC,CAAC;YACvD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC,CAAC;YAC9F,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;oBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IACD,6EAA6E;IAC7E,6EAA6E;IAC7E,yEAAyE;IACzE,6CAA6C;IAC7C,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,KAAK,SAAS;YAAE,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IACD,0EAA0E;IAC1E,mEAAmE;IACnE,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IACtD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,IAAI,IAAgB,CAAC;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,+CAA+C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtJ,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAChF,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,aAAa,CAAC,iBAAiB,EAAE,oBAAoB,KAAK,CAAC,IAAI,2BAA2B,KAAK,gCAAgC,CAAC,CAAC;YAC7I,CAAC;YACD,KAAK,MAAM,IAAI,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;gBAAE,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IACzF,MAAM,KAAK,GAAG,QAAQ;SACnB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACrG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpE,gGAAgG;IAChG,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;IACjF,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACzF,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;aACzD,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAAwB;IAC/C,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACnH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,IAAY;IAC9C,KAAK,IAAI,SAAS,GAAG,KAAK,GAAI,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;QAC5C,IAAI,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;IACzD,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,KAAK,GAAI,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,QAAQ,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;gBAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC;QAC3B,IAAI,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;IACrD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,QAAQ,GAAG;QACf,iFAAiF;QACjF,wCAAwC;QACxC,8DAA8D;QAC9D,yCAAyC;KAC1C,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,QAAQ;QAAE,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IACnG,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACpB,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,IAAY,EAAE,SAAiB;IACnD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,CAAC,SAAiB,EAAW,EAAE;QAC5C,IAAI,CAAC;YACH,OAAO,QAAQ,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,CAAC;IACF,kEAAkE;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC;IACrC,KAAK,MAAM,SAAS,IAAI;QACtB,GAAG,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,GAAG,SAAS,CAAC;QACvD,GAAG,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;KACvE;QAAE,IAAI,MAAM,CAAC,SAAS,CAAC;YAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7C,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,MAAM,CAAC,IAAqB;IACnC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC"}

package/dist/flow-eval.d.ts

@@ -0,0 +1,41 @@
+import { type FlowEvalExecutor, type FlowEvalReport, type FlowEvalScorer } from './flow-eval-report.js';
+import { type FlowEvalSuite } from './flow-eval-suite.js';
+import { type FlowEvalTarget } from './flow-eval-version.js';
+export { FlowEvalError, type FlowEvalErrorCode } from './flow-eval-error.js';
+export { parseFlowEvalBaseline, type FlowEvalBaseline } from './flow-eval-gate.js';
+export * from './flow-eval-report.js';
+export { MAX_FLOW_EVAL_CASES, flowEvalSuiteSha256, loadFlowEvalSuite, parseFlowEvalSuite, type FlowEvalCase, type FlowEvalExpectation, type FlowEvalSuite, type FlowEvalThresholds, } from './flow-eval-suite.js';
+export { MAX_FLOW_EVAL_SOURCES, flowEvalSources, flowEvalVersion, sealFlowEvalSnapshot, type FlowEvalSnapshot, type FlowEvalTarget, type FlowEvalVersion, } from './flow-eval-version.js';
+export interface EvaluateFlowOptions {
+    flow: FlowEvalTarget;
+    suite: FlowEvalSuite;
+    executor: FlowEvalExecutor;
+    /** Recorded in the report; defaults to `custom`. */
+    executorName?: string;
+    scorers?: FlowEvalScorer[];
+    /** Cases executed at once. Defaults to 1, which keeps runs from contending. */
+    concurrency?: number;
+    /** Refuse unless the flow version is exactly this (`sha256:<hex>`). */
+    expectVersion?: string;
+    /** Refuse unless the suite hashes to exactly this (`sha256:<hex>`). */
+    expectSuiteSha256?: string;
+    /**
+     * A prior report on the same suite (parsed JSON is fine; it is validated
+     * before any case runs). Any case that passed there and does not pass here
+     * is a regression, and a regression fails the gate.
+     */
+    baseline?: unknown;
+    signal?: AbortSignal;
+    /** Clock injection for tests. */
+    now?: () => number;
+}
+/**
+ * Re-execute `flow` once per suite case and score the results.
+ *
+ * Never throws for a case that fails or errors: those are verdicts, recorded
+ * in the report. Throws `FlowEvalError` only when the evaluation itself is
+ * invalid (bad suite, baseline or options, or a pinned version/suite that does
+ * not match), and rethrows an abort.
+ */
+export declare function evaluateFlow(options: EvaluateFlowOptions): Promise<FlowEvalReport>;
+//# sourceMappingURL=flow-eval.d.ts.map

package/dist/flow-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flow-eval.d.ts","sourceRoot":"","sources":["../src/flow-eval.ts"],"names":[],"mappings":"AAsBA,OAAO,EAEoB,KAAK,gBAAgB,EAAE,KAAK,cAAc,EAAoB,KAAK,cAAc,EAC3G,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAA8D,KAAK,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACtH,OAAO,EAA6D,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExH,OAAO,EAAE,aAAa,EAAE,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,KAAK,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACnF,cAAc,uBAAuB,CAAC;AACtC,OAAO,EACL,mBAAmB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,kBAAkB,EAC/E,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAE,KAAK,aAAa,EAAE,KAAK,kBAAkB,GACzF,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,qBAAqB,EAAE,eAAe,EAAE,eAAe,EAAE,oBAAoB,EAC7E,KAAK,gBAAgB,EAAE,KAAK,cAAc,EAAE,KAAK,eAAe,GACjE,MAAM,wBAAwB,CAAC;AAEhC,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,cAAc,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;IACrB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,oDAAoD;IACpD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;IAC3B,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uEAAuE;IACvE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,iCAAiC;IACjC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,cAAc,CAAC,CAkExF"}