@plumloom/cli 0.1.0

package/dist/commands/types.d.ts

@@ -0,0 +1,139 @@
+import type { GateThresholds } from '../actions/gate.js';
+import type { QuickstartSampleKind } from '../quickstart/sample.js';
+export type DeterministicCommand = {
+    kind: 'login';
+    key?: string;
+} | {
+    kind: 'logout';
+} | {
+    kind: 'whoami';
+} | {
+    kind: 'workspace-list';
+} | {
+    kind: 'workspace-create';
+    name: string;
+    description?: string;
+} | {
+    kind: 'evaluation-list';
+    workspaceId: string;
+} | {
+    kind: 'evaluation-create';
+    workspaceId: string;
+    name?: string;
+} | {
+    kind: 'evaluation-update-title';
+    workspaceId: string;
+    evaluationId: string;
+    name: string;
+    userSystemId: string;
+    description?: string;
+    page?: number;
+    size?: number;
+} | {
+    kind: 'quality-standard-create';
+    inputFile: string;
+    workspaceId?: string;
+} | {
+    kind: 'quality-standard-update';
+    qualityStandardId: string;
+    inputFile: string;
+} | {
+    kind: 'quality-standard-assign';
+    workspaceId: string;
+    qualityStandardId: string;
+} | {
+    kind: 'quality-standard-workspace';
+    workspaceId: string;
+} | {
+    kind: 'quality-standard-show';
+    qualityStandardId: string;
+} | {
+    kind: 'evaluation-validate';
+    inputFile: string;
+} | {
+    kind: 'evaluation-create-from';
+    workspaceId: string;
+    inputFiles: readonly string[];
+    run: boolean;
+    judgeModelId?: string;
+    primaryModelId?: string;
+    runConcurrency?: number;
+    runStartStaggerMs?: number;
+} | {
+    kind: 'trace-import';
+    source: 'deepseek-harness';
+    sessionFile: string;
+    templateFile: string;
+    outputFile: string;
+    evaluationName?: string;
+    serviceName?: string;
+    workspaceId?: string;
+    run: boolean;
+} | {
+    kind: 'suite-run';
+    manifestFile: string;
+    workspaceId?: string;
+    concurrency?: number;
+    staggerMs?: number;
+    judgeModelId?: string;
+    primaryModelId?: string;
+} | {
+    kind: 'suite-gate';
+    manifestFile: string;
+    workspaceId?: string;
+    concurrency?: number;
+    staggerMs?: number;
+    judgeModelId?: string;
+    primaryModelId?: string;
+} | {
+    kind: 'evaluation-run-configured';
+    evaluationId: string;
+    inputFile: string;
+} | {
+    kind: 'evaluation-versions';
+    evaluationId: string;
+} | {
+    kind: 'evaluation-show';
+    evaluationId: string;
+    version?: number;
+} | {
+    kind: 'models';
+} | {
+    kind: 'quickstart';
+    workspaceId?: string;
+    judgeModelId?: string;
+    primaryModelId?: string;
+    inputFile?: string;
+    sample: QuickstartSampleKind;
+    assumeYes: boolean;
+} | {
+    kind: 'doctor';
+    workspaceId?: string;
+    manifestFile?: string;
+    inputFiles: readonly string[];
+} | {
+    kind: 'run';
+    evaluationId: string;
+} | {
+    kind: 'status';
+    evaluationId: string;
+    runId: string;
+} | {
+    kind: 'results';
+    evaluationId: string;
+    runId: string;
+    showOutputs?: boolean;
+} | {
+    kind: 'gate';
+    evaluationId: string;
+    thresholdsFile?: string;
+    thresholds: GateThresholds;
+};
+export type ExecutionOptions = {
+    json: boolean;
+    debug: boolean;
+};
+export interface CommandExecutor {
+    execute(command: DeterministicCommand, options: ExecutionOptions): Promise<void>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/commands/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/commands/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAEpE,MAAM,MAAM,oBAAoB,GAC5B;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/B;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IAAE,IAAI,EAAE,gBAAgB,CAAA;CAAE,GAC1B;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IACE,IAAI,EAAE,yBAAyB,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GACD;IAAE,IAAI,EAAE,yBAAyB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5E;IAAE,IAAI,EAAE,yBAAyB,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACjF;IACE,IAAI,EAAE,yBAAyB,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;CAC3B,GACD;IAAE,IAAI,EAAE,4BAA4B,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,iBAAiB,EAAE,MAAM,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAClD;IACE,IAAI,EAAE,wBAAwB,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,GAAG,EAAE,OAAO,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B,GACD;IACE,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;CACd,GACD;IACE,IAAI,EAAE,WAAW,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GACD;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,GACD;IAAE,IAAI,EAAE,2BAA2B,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GACnE;IAAE,IAAI,EAAE,QAAQ,CAAA;CAAE,GAClB;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,SAAS,EAAE,OAAO,CAAC;CACpB,GACD;IACE,IAAI,EAAE,QAAQ,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;CAC/B,GACD;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,GAC/E;IACE,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,cAAc,CAAC;CAC5B,CAAC;AAEN,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAClF"}

package/dist/commands/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/commands/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/commands/types.ts"],"names":[],"mappings":""}

package/dist/config.d.ts

@@ -0,0 +1,12 @@
+export type AutoevalConfiguration = {
+    apiBaseUrl: URL;
+    requestTimeoutMs: number;
+    maxResponseBytes: number;
+    pollIntervalMs: number;
+    pollTimeoutMs: number;
+    resultReadyIntervalMs: number;
+    resultReadyTimeoutMs: number;
+};
+export declare function parseApiBaseUrl(input: string): URL;
+export declare function loadConfiguration(environment?: Readonly<Record<string, string | undefined>>): AutoevalConfiguration;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,qBAAqB,GAAG;IAClC,UAAU,EAAE,GAAG,CAAC;IAChB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAMF,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,CA4ClD;AAED,wBAAgB,iBAAiB,CAC/B,WAAW,GAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAe,GACtE,qBAAqB,CAqBvB"}

package/dist/config.js

@@ -0,0 +1,57 @@
+import { AutoevalError } from './errors/autoeval-error.js';
+function isLocalhost(hostname) {
+    return hostname === 'localhost' || hostname === '127.0.0.1' || hostname === '[::1]';
+}
+export function parseApiBaseUrl(input) {
+    let parsed;
+    try {
+        parsed = new URL(input);
+    }
+    catch (error) {
+        throw new AutoevalError('AUTOEVAL_API_BASE_URL must be a valid URL.', {
+            kind: 'validation',
+            code: 'INVALID_BASE_URL',
+            cause: error,
+        });
+    }
+    if (parsed.protocol !== 'https:' &&
+        !(parsed.protocol === 'http:' && isLocalhost(parsed.hostname))) {
+        throw new AutoevalError('The Autoeval API must use HTTPS; HTTP is allowed only for localhost development.', {
+            kind: 'validation',
+            code: 'INSECURE_BASE_URL',
+        });
+    }
+    if (parsed.username || parsed.password || parsed.search || parsed.hash) {
+        throw new AutoevalError('AUTOEVAL_API_BASE_URL must not contain credentials, query parameters, or fragments.', {
+            kind: 'validation',
+            code: 'INVALID_BASE_URL',
+        });
+    }
+    if (parsed.pathname !== '/' && parsed.pathname !== '') {
+        throw new AutoevalError('AUTOEVAL_API_BASE_URL must be an origin without a path.', {
+            kind: 'validation',
+            code: 'INVALID_BASE_URL',
+        });
+    }
+    parsed.pathname = '/';
+    return parsed;
+}
+export function loadConfiguration(environment = process.env) {
+    const apiBaseUrl = environment.AUTOEVAL_API_BASE_URL?.trim();
+    if (!apiBaseUrl) {
+        throw new AutoevalError('AUTOEVAL_API_BASE_URL is not defined. Set it to the Autoeval API origin to continue.', {
+            kind: 'validation',
+            code: 'MISSING_API_BASE_URL',
+        });
+    }
+    return {
+        apiBaseUrl: parseApiBaseUrl(apiBaseUrl),
+        requestTimeoutMs: 30_000,
+        maxResponseBytes: 2 * 1024 * 1024,
+        pollIntervalMs: 2_000,
+        pollTimeoutMs: 10 * 60_000,
+        resultReadyIntervalMs: 2_000,
+        resultReadyTimeoutMs: 60_000,
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAY3D,SAAS,WAAW,CAAC,QAAgB;IACnC,OAAO,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,OAAO,CAAC;AACtF,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAC,4CAA4C,EAAE;YACpE,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,kBAAkB;YACxB,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IAED,IACE,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAC5B,CAAC,CAAC,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAC9D,CAAC;QACD,MAAM,IAAI,aAAa,CACrB,kFAAkF,EAClF;YACE,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,mBAAmB;SAC1B,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACvE,MAAM,IAAI,aAAa,CACrB,qFAAqF,EACrF;YACE,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,kBAAkB;SACzB,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;QACtD,MAAM,IAAI,aAAa,CAAC,yDAAyD,EAAE;YACjF,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC;IACtB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,cAA4D,OAAO,CAAC,GAAG;IAEvE,MAAM,UAAU,GAAG,WAAW,CAAC,qBAAqB,EAAE,IAAI,EAAE,CAAC;IAC7D,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,aAAa,CACrB,sFAAsF,EACtF;YACE,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,sBAAsB;SAC7B,CACF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU,EAAE,eAAe,CAAC,UAAU,CAAC;QACvC,gBAAgB,EAAE,MAAM;QACxB,gBAAgB,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI;QACjC,cAAc,EAAE,KAAK;QACrB,aAAa,EAAE,EAAE,GAAG,MAAM;QAC1B,qBAAqB,EAAE,KAAK;QAC5B,oBAAoB,EAAE,MAAM;KAC7B,CAAC;AACJ,CAAC"}

package/dist/configured-run/file-inputs.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Expands `artifactFile` and `referenceDocumentFiles` in an already-parsed eval
+ * file. `baseDirectory` is the directory holding the eval file.
+ */
+export declare function resolveEvalFileInputs(parsedJson: unknown, baseDirectory: string): Promise<unknown>;
+/** Reads an eval file from disk and expands any file references it declares. */
+export declare function readEvalFile(path: string): Promise<unknown>;
+//# sourceMappingURL=file-inputs.d.ts.map

package/dist/configured-run/file-inputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-inputs.d.ts","sourceRoot":"","sources":["../../src/configured-run/file-inputs.ts"],"names":[],"mappings":"AAsGA;;;GAGG;AACH,wBAAsB,qBAAqB,CACzC,UAAU,EAAE,OAAO,EACnB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,OAAO,CAAC,CAuDlB;AAED,gFAAgF;AAChF,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkBjE"}

package/dist/configured-run/file-inputs.js

@@ -0,0 +1,160 @@
+import { readFile } from 'node:fs/promises';
+import { basename, dirname, isAbsolute, resolve } from 'node:path';
+import { AutoevalError } from '../errors/autoeval-error.js';
+/**
+ * Reading an eval file that points at its own inputs.
+ *
+ * A configured-run file is the contract between the developer and Autoeval, and
+ * it is committed alongside the code. Inlining a whole transcript, trace, or
+ * policy document into it makes that file unreadable in review, so the file may
+ * instead reference them by path:
+ *
+ *   "artifactFile": "./data/conversation.json"
+ *   "referenceDocumentFiles": ["./policy/escalation.md"]
+ *
+ * Paths resolve relative to the eval file, so a checkout can live anywhere. The
+ * references are expanded here, before validation, into exactly the inline
+ * shape the existing schema and run path already accept — nothing downstream
+ * knows the difference.
+ */
+const MAX_ARTIFACT_CHARS = 2_000_000;
+const MAX_REFERENCE_DOCUMENT_CHARS = 200_000;
+function readFailure(path, error) {
+    return new AutoevalError(`File was not found or could not be read: ${path}`, {
+        kind: 'usage',
+        code: 'EVAL_FILE_READ_FAILED',
+        cause: error,
+    });
+}
+async function readBounded(path, limit) {
+    let content;
+    try {
+        content = await readFile(path, 'utf8');
+    }
+    catch (error) {
+        throw readFailure(path, error);
+    }
+    if (content.length > limit) {
+        throw new AutoevalError(`${path} is larger than ${limit} characters.`, {
+            kind: 'validation',
+            code: 'EVAL_FILE_TOO_LARGE',
+        });
+    }
+    return content;
+}
+function resolveAgainst(baseDirectory, path) {
+    return isAbsolute(path) ? path : resolve(baseDirectory, path);
+}
+function asRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+/**
+ * An agent trace file is usually the raw OTLP export, whose top level is
+ * `resourceSpans`. The configuration schema expects it under `trace`, so the
+ * raw export is wrapped rather than rejected.
+ */
+function normalizeArtifact(contextType, artifact, path) {
+    if (contextType !== 'agent_trace')
+        return artifact;
+    const record = asRecord(artifact);
+    if (!record) {
+        throw new AutoevalError(`${path} must contain a JSON object.`, {
+            kind: 'validation',
+            code: 'ARTIFACT_FILE_INVALID_SHAPE',
+        });
+    }
+    return 'trace' in record ? record : { trace: record };
+}
+async function readArtifactFile(path) {
+    const content = await readBounded(path, MAX_ARTIFACT_CHARS);
+    try {
+        return JSON.parse(content);
+    }
+    catch (error) {
+        throw new AutoevalError(`${path} is not valid JSON.`, {
+            kind: 'validation',
+            code: 'ARTIFACT_FILE_INVALID_JSON',
+            cause: error,
+        });
+    }
+}
+async function readReferenceDocument(path) {
+    const content = await readBounded(path, MAX_REFERENCE_DOCUMENT_CHARS);
+    if (content.trim() === '') {
+        throw new AutoevalError(`${path} is empty.`, {
+            kind: 'validation',
+            code: 'REFERENCE_DOCUMENT_EMPTY',
+        });
+    }
+    return { filename: basename(path), content };
+}
+/**
+ * Expands `artifactFile` and `referenceDocumentFiles` in an already-parsed eval
+ * file. `baseDirectory` is the directory holding the eval file.
+ */
+export async function resolveEvalFileInputs(parsedJson, baseDirectory) {
+    const root = asRecord(parsedJson);
+    const configuration = root === undefined ? undefined : asRecord(root.configuration);
+    if (root === undefined || configuration === undefined)
+        return parsedJson;
+    const artifactFile = configuration.artifactFile;
+    const referenceDocumentFiles = configuration.referenceDocumentFiles;
+    if (artifactFile === undefined && referenceDocumentFiles === undefined)
+        return parsedJson;
+    const resolved = { ...configuration };
+    delete resolved.artifactFile;
+    delete resolved.referenceDocumentFiles;
+    if (artifactFile !== undefined) {
+        if (typeof artifactFile !== 'string' || artifactFile.trim() === '') {
+            throw new AutoevalError('configuration.artifactFile must be a file path.', {
+                kind: 'validation',
+                code: 'ARTIFACT_FILE_INVALID_PATH',
+            });
+        }
+        if (configuration.artifact !== undefined) {
+            throw new AutoevalError('configuration has both artifact and artifactFile. Keep one of them.', { kind: 'validation', code: 'ARTIFACT_FILE_CONFLICT' });
+        }
+        const path = resolveAgainst(baseDirectory, artifactFile);
+        resolved.artifact = normalizeArtifact(configuration.contextType, await readArtifactFile(path), path);
+    }
+    if (referenceDocumentFiles !== undefined) {
+        if (!Array.isArray(referenceDocumentFiles) ||
+            referenceDocumentFiles.some((entry) => typeof entry !== 'string' || entry.trim() === '')) {
+            throw new AutoevalError('configuration.referenceDocumentFiles must be an array of file paths.', { kind: 'validation', code: 'REFERENCE_DOCUMENT_INVALID_PATH' });
+        }
+        const existing = Array.isArray(configuration.referenceDocuments)
+            ? configuration.referenceDocuments
+            : [];
+        const documents = [...existing];
+        for (const entry of referenceDocumentFiles) {
+            documents.push(await readReferenceDocument(resolveAgainst(baseDirectory, entry)));
+        }
+        resolved.referenceDocuments = documents;
+    }
+    return { ...root, configuration: resolved };
+}
+/** Reads an eval file from disk and expands any file references it declares. */
+export async function readEvalFile(path) {
+    let content;
+    try {
+        content = await readFile(path, 'utf8');
+    }
+    catch (error) {
+        throw readFailure(path, error);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch (error) {
+        throw new AutoevalError(`${path} must contain valid JSON.`, {
+            kind: 'usage',
+            code: 'INVALID_CONFIGURED_RUN_JSON',
+            cause: error,
+        });
+    }
+    return resolveEvalFileInputs(parsed, dirname(resolve(path)));
+}
+//# sourceMappingURL=file-inputs.js.map

package/dist/configured-run/file-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-inputs.js","sourceRoot":"","sources":["../../src/configured-run/file-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEnE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,kBAAkB,GAAG,SAAS,CAAC;AACrC,MAAM,4BAA4B,GAAG,OAAO,CAAC;AAI7C,SAAS,WAAW,CAAC,IAAY,EAAE,KAAc;IAC/C,OAAO,IAAI,aAAa,CAAC,4CAA4C,IAAI,EAAE,EAAE;QAC3E,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,uBAAuB;QAC7B,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,KAAa;IACpD,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC;QAC3B,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,mBAAmB,KAAK,cAAc,EAAE;YACrE,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,aAAqB,EAAE,IAAY;IACzD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACzE,CAAC,CAAE,KAA6B;QAChC,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,WAAoB,EAAE,QAAiB,EAAE,IAAY;IAC9E,IAAI,WAAW,KAAK,aAAa;QAAE,OAAO,QAAQ,CAAC;IACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,8BAA8B,EAAE;YAC7D,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,6BAA6B;SACpC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY;IAC1C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;IAC5D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAY,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,qBAAqB,EAAE;YACpD,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,4BAA4B;YAClC,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,IAAY;IAC/C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;IACtE,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,YAAY,EAAE;YAC3C,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,0BAA0B;SACjC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,UAAmB,EACnB,aAAqB;IAErB,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,aAAa,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACpF,IAAI,IAAI,KAAK,SAAS,IAAI,aAAa,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAEzE,MAAM,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;IAChD,MAAM,sBAAsB,GAAG,aAAa,CAAC,sBAAsB,CAAC;IACpE,IAAI,YAAY,KAAK,SAAS,IAAI,sBAAsB,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAE1F,MAAM,QAAQ,GAAwB,EAAE,GAAG,aAAa,EAAE,CAAC;IAC3D,OAAO,QAAQ,CAAC,YAAY,CAAC;IAC7B,OAAO,QAAQ,CAAC,sBAAsB,CAAC;IAEvC,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACnE,MAAM,IAAI,aAAa,CAAC,iDAAiD,EAAE;gBACzE,IAAI,EAAE,YAAY;gBAClB,IAAI,EAAE,4BAA4B;aACnC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,aAAa,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzC,MAAM,IAAI,aAAa,CACrB,qEAAqE,EACrE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,wBAAwB,EAAE,CACvD,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,cAAc,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QACzD,QAAQ,CAAC,QAAQ,GAAG,iBAAiB,CACnC,aAAa,CAAC,WAAW,EACzB,MAAM,gBAAgB,CAAC,IAAI,CAAC,EAC5B,IAAI,CACL,CAAC;IACJ,CAAC;IAED,IAAI,sBAAsB,KAAK,SAAS,EAAE,CAAC;QACzC,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,sBAAsB,CAAC;YACtC,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EACxF,CAAC;YACD,MAAM,IAAI,aAAa,CACrB,sEAAsE,EACtE,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,iCAAiC,EAAE,CAChE,CAAC;QACJ,CAAC;QACD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC;YAC9D,CAAC,CAAE,aAAa,CAAC,kBAAgC;YACjD,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,SAAS,GAAc,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,sBAA2C,EAAE,CAAC;YAChE,SAAS,CAAC,IAAI,CAAC,MAAM,qBAAqB,CAAC,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACpF,CAAC;QACD,QAAQ,CAAC,kBAAkB,GAAG,SAAS,CAAC;IAC1C,CAAC;IAED,OAAO,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;AAC9C,CAAC;AAED,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAY,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CAAC,GAAG,IAAI,2BAA2B,EAAE;YAC1D,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,6BAA6B;YACnC,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IACD,OAAO,qBAAqB,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC"}

package/dist/configured-run/validation.d.ts

@@ -0,0 +1,103 @@
+import { z } from 'zod';
+import type { EvaluationConfigInput } from '../actions/evaluations.js';
+import type { SupportedModel } from '../domain/types.js';
+export declare const configuredRunFileSchema: z.ZodObject<{
+    methodology: z.ZodObject<{
+        userSystemId: z.ZodOptional<z.ZodString>;
+        judgeModel: z.ZodString;
+        judgeModelId: z.ZodUUID;
+        runsPerScenario: z.ZodOptional<z.ZodNumber>;
+        evaluatorInstructions: z.ZodString;
+        changeLog: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    configuration: z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
+        evaluationName: z.ZodOptional<z.ZodString>;
+        contextName: z.ZodString;
+        contextType: z.ZodLiteral<"scenario">;
+        artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+        expected: z.ZodString;
+        referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            filename: z.ZodString;
+            content: z.ZodString;
+        }, z.core.$strict>>>;
+        selectedMetrics: z.ZodArray<z.ZodString>;
+        temperatureContext: z.ZodOptional<z.ZodNumber>;
+        autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        scenarioGeneration: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
+    }, z.core.$strict>, z.ZodObject<{
+        evaluationName: z.ZodOptional<z.ZodString>;
+        contextName: z.ZodString;
+        contextType: z.ZodLiteral<"conversation">;
+        artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+        expected: z.ZodString;
+        referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            filename: z.ZodString;
+            content: z.ZodString;
+        }, z.core.$strict>>>;
+        selectedMetrics: z.ZodArray<z.ZodString>;
+        temperatureContext: z.ZodOptional<z.ZodNumber>;
+        autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>, z.ZodObject<{
+        evaluationName: z.ZodOptional<z.ZodString>;
+        contextName: z.ZodString;
+        contextType: z.ZodLiteral<"agent_trace">;
+        artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+        expected: z.ZodString;
+        referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            filename: z.ZodString;
+            content: z.ZodString;
+        }, z.core.$strict>>>;
+        selectedMetrics: z.ZodArray<z.ZodString>;
+        temperatureContext: z.ZodOptional<z.ZodNumber>;
+        autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>], "contextType">, z.ZodObject<{
+        evaluationName: z.ZodOptional<z.ZodString>;
+        contextName: z.ZodOptional<z.ZodString>;
+        primaryModelId: z.ZodUUID;
+        comparisonModelIds: z.ZodDefault<z.ZodArray<z.ZodUUID>>;
+        promptText: z.ZodString;
+        scenarios: z.ZodUnion<readonly [z.ZodArray<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>]>;
+        referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            filename: z.ZodString;
+            content: z.ZodString;
+        }, z.core.$strict>>>;
+        selectedMetrics: z.ZodArray<z.ZodString>;
+        expected: z.ZodOptional<z.ZodString>;
+        temperatureContext: z.ZodOptional<z.ZodNumber>;
+        autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+    }, z.core.$strict>]>;
+}, z.core.$strict>;
+export type ConfiguredRunFileInput = z.input<typeof configuredRunFileSchema>;
+export type ParsedConfiguredRunInput = {
+    methodology: {
+        userSystemId?: string;
+        judgeModel: string;
+        judgeModelId: string;
+        runsPerScenario?: number;
+        evaluatorInstructions: string;
+        changeLog?: string;
+    };
+    evaluationName?: string;
+    configuration: EvaluationConfigInput;
+};
+export type ConfiguredRunValidationResult = {
+    valid: true;
+    contextType: EvaluationConfigInput['contextType'];
+    artifactCount: number;
+    models: {
+        judge: SupportedModel;
+        primary?: SupportedModel;
+        comparisons: SupportedModel[];
+    };
+};
+export type ConfiguredRunModelOverrides = {
+    judgeModelId?: string;
+    primaryModelId?: string;
+};
+export declare function parseConfiguredRunInput(parsedJson: unknown): ParsedConfiguredRunInput;
+export declare function validateConfiguredRunInput(input: ParsedConfiguredRunInput, enabledModels: readonly SupportedModel[]): ConfiguredRunValidationResult;
+/** Applies explicit CLI model UUIDs without mutating the parsed file input. */
+export declare function applyModelOverrides(input: ParsedConfiguredRunInput, overrides: ConfiguredRunModelOverrides): ParsedConfiguredRunInput;
+/** Fails fast when a sanitized public fixture was not given account model UUIDs. */
+export declare function requireModelOverridesForPublicPlaceholders(input: ParsedConfiguredRunInput): void;
+//# sourceMappingURL=validation.d.ts.map

package/dist/configured-run/validation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../../src/configured-run/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AA4EzD,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiBzB,CAAC;AAEZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAuC7E,MAAM,MAAM,wBAAwB,GAAG;IACrC,WAAW,EAAE;QACX,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,qBAAqB,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,6BAA6B,GAAG;IAC1C,KAAK,EAAE,IAAI,CAAC;IACZ,WAAW,EAAE,qBAAqB,CAAC,aAAa,CAAC,CAAC;IAClD,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE;QACN,KAAK,EAAE,cAAc,CAAC;QACtB,OAAO,CAAC,EAAE,cAAc,CAAC;QACzB,WAAW,EAAE,cAAc,EAAE,CAAC;KAC/B,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACxC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAwBF,wBAAgB,uBAAuB,CAAC,UAAU,EAAE,OAAO,GAAG,wBAAwB,CA4ErF;AAuCD,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,wBAAwB,EAC/B,aAAa,EAAE,SAAS,cAAc,EAAE,GACvC,6BAA6B,CAgE/B;AAED,+EAA+E;AAC/E,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,wBAAwB,EAC/B,SAAS,EAAE,2BAA2B,GACrC,wBAAwB,CAqB1B;AAED,oFAAoF;AACpF,wBAAgB,0CAA0C,CAAC,KAAK,EAAE,wBAAwB,GAAG,IAAI,CAiBhG"}