@plumloom/cli 0.1.4 → 0.1.6

package/dist/actions/init.d.ts

@@ -0,0 +1,47 @@
+import type { SupportedModel, Workspace } from '../domain/types.js';
+import { AutoevalError } from '../errors/autoeval-error.js';
+import type { BundledPack } from '../packs/registry.js';
+import { type ModelRole } from '../quickstart/selection.js';
+import type { AutoevalSettings } from '../settings/settings.js';
+import type { ActionContext } from './context.js';
+import type { SuiteSummary } from './suite.js';
+export type ResolvedInitDefaults = {
+    workspaceId: string;
+    judgeModelId: string;
+    /** Present only for scenario packs, which grade a model under test. */
+    primaryModelId?: string;
+};
+export declare function settingsToPersist(current: AutoevalSettings, resolved: ResolvedInitDefaults): AutoevalSettings;
+export type ActivationSelection = {
+    workspace: Workspace;
+    judge: SupportedModel;
+    primary?: SupportedModel;
+};
+export type ActivationResult = ActivationSelection & {
+    summary: SuiteSummary;
+    persistedDefaults: boolean;
+};
+export type RunActivationInput = {
+    pack: BundledPack;
+    settings: AutoevalSettings;
+    workspaceId?: string;
+    judgeModelId?: string;
+    primaryModelId?: string;
+    /** Used by `init --yes`; selects the first ordered candidate instead of prompting. */
+    assumeFirstChoice?: boolean;
+    chooseWorkspace?: (candidates: readonly Workspace[]) => Promise<Workspace>;
+    chooseModel?: (role: ModelRole, candidates: readonly SupportedModel[]) => Promise<SupportedModel>;
+    runPack: (selection: ActivationSelection) => Promise<SuiteSummary>;
+    /** Present for init only. Failure is fatal because persisted defaults are required. */
+    persistSettings?: (settings: AutoevalSettings) => Promise<void>;
+    signal?: AbortSignal;
+};
+/**
+ * Shared activation workflow used by `init` and `pack run`. The CLI supplies
+ * prompt and filesystem ports; workspace/model resolution, run sequencing,
+ * success determination, and required persistence remain reusable product logic.
+ */
+export declare function runActivation(context: ActionContext, input: RunActivationInput): Promise<ActivationResult>;
+/** Produces the non-zero error used after a failed activation summary is rendered. */
+export declare function activationRunFailure(summary: SuiteSummary, code: string): AutoevalError;
+//# sourceMappingURL=init.d.ts.map

package/dist/actions/init.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/actions/init.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAIL,KAAK,SAAS,EACf,MAAM,4BAA4B,CAAC;AAEpC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGlD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG/C,MAAM,MAAM,oBAAoB,GAAG;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE,oBAAoB,GAC7B,gBAAgB,CAUlB;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,cAAc,CAAC;IACtB,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,mBAAmB,GAAG;IACnD,OAAO,EAAE,YAAY,CAAC;IACtB,iBAAiB,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,WAAW,CAAC;IAClB,QAAQ,EAAE,gBAAgB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,sFAAsF;IACtF,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,eAAe,CAAC,EAAE,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,KAAK,OAAO,CAAC,SAAS,CAAC,CAAC;IAC3E,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,cAAc,EAAE,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAClG,OAAO,EAAE,CAAC,SAAS,EAAE,mBAAmB,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;IACnE,uFAAuF;IACvF,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAChE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAgDF;;;;GAIG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,kBAAkB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAkC3B;AAED,sFAAsF;AACtF,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,aAAa,CASvF"}

package/dist/actions/init.js

@@ -0,0 +1,89 @@
+import { AutoevalError } from '../errors/autoeval-error.js';
+import { enabledModels, selectQuickstartModel, selectQuickstartWorkspace, } from '../quickstart/selection.js';
+import { resolveModelIds, resolveWorkspaceId } from '../settings/resolve.js';
+import { getModels } from './models.js';
+import { packRequiresPrimaryModel } from './pack.js';
+import { listWorkspaces } from './workspaces.js';
+export function settingsToPersist(current, resolved) {
+    return {
+        ...current,
+        version: 1,
+        defaultWorkspaceId: resolved.workspaceId,
+        defaultJudgeModelId: resolved.judgeModelId,
+        ...(resolved.primaryModelId !== undefined
+            ? { defaultPrimaryModelId: resolved.primaryModelId }
+            : {}),
+    };
+}
+function workspaceRequired() {
+    return new AutoevalError('More than one workspace is available. Save one with "autoeval config set-workspace <workspace-id>", pass --workspace to init, or run in a terminal to choose.', { kind: 'usage', code: 'INIT_WORKSPACE_REQUIRED' });
+}
+function modelRequired() {
+    return new AutoevalError('More than one model is enabled and none matches the preference list. Save model defaults with "autoeval config", pass model IDs to init, or run in a terminal to choose; "autoeval models" lists IDs.', { kind: 'usage', code: 'INIT_MODEL_REQUIRED' });
+}
+async function resolveWorkspace(candidates, overrideId, input) {
+    const resolution = selectQuickstartWorkspace(candidates, overrideId);
+    if (resolution.kind === 'resolved')
+        return resolution.workspace;
+    if (input.assumeFirstChoice === true) {
+        const first = resolution.candidates[0];
+        if (first !== undefined)
+            return first;
+    }
+    if (input.chooseWorkspace === undefined)
+        throw workspaceRequired();
+    return input.chooseWorkspace(resolution.candidates);
+}
+async function resolveModel(candidates, role, overrideId, input, excludeModelId) {
+    const resolution = selectQuickstartModel(candidates, role, overrideId, excludeModelId);
+    if (resolution.kind === 'resolved')
+        return resolution.model;
+    if (input.assumeFirstChoice === true) {
+        const first = resolution.candidates[0];
+        if (first !== undefined)
+            return first;
+    }
+    if (input.chooseModel === undefined)
+        throw modelRequired();
+    return input.chooseModel(role, resolution.candidates);
+}
+/**
+ * Shared activation workflow used by `init` and `pack run`. The CLI supplies
+ * prompt and filesystem ports; workspace/model resolution, run sequencing,
+ * success determination, and required persistence remain reusable product logic.
+ */
+export async function runActivation(context, input) {
+    const workspacePage = await listWorkspaces(context, {}, input.signal);
+    const workspace = await resolveWorkspace(workspacePage.items, resolveWorkspaceId(input.workspaceId, input.settings), input);
+    const models = enabledModels(await getModels(context, input.signal));
+    const resolvedModelIds = resolveModelIds(input, input.settings);
+    const judge = await resolveModel(models, 'judge', resolvedModelIds.judgeModelId, input);
+    const primary = packRequiresPrimaryModel(input.pack.context)
+        ? await resolveModel(models, 'primary', resolvedModelIds.primaryModelId, input, judge.id)
+        : undefined;
+    const selection = {
+        workspace,
+        judge,
+        ...(primary === undefined ? {} : { primary }),
+    };
+    const summary = await input.runPack(selection);
+    let persistedDefaults = false;
+    if (summary.failed === 0 && input.persistSettings !== undefined) {
+        await input.persistSettings(settingsToPersist(input.settings, {
+            workspaceId: workspace.id,
+            judgeModelId: judge.id,
+            ...(primary === undefined ? {} : { primaryModelId: primary.id }),
+        }));
+        persistedDefaults = true;
+    }
+    return { ...selection, summary, persistedDefaults };
+}
+/** Produces the non-zero error used after a failed activation summary is rendered. */
+export function activationRunFailure(summary, code) {
+    const failed = summary.evals
+        .filter((entry) => entry.status !== 'completed')
+        .map((entry) => `${entry.inputFile} (${entry.status})`)
+        .join(', ');
+    return new AutoevalError(`${summary.failed} of ${summary.total} evaluation(s) did not finish: ${failed}.`, { kind: 'run_failed', code });
+}
+//# sourceMappingURL=init.js.map

package/dist/actions/init.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/actions/init.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EACL,aAAa,EACb,qBAAqB,EACrB,yBAAyB,GAE1B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAG7E,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,wBAAwB,EAAE,MAAM,WAAW,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AASjD,MAAM,UAAU,iBAAiB,CAC/B,OAAyB,EACzB,QAA8B;IAE9B,OAAO;QACL,GAAG,OAAO;QACV,OAAO,EAAE,CAAC;QACV,kBAAkB,EAAE,QAAQ,CAAC,WAAW;QACxC,mBAAmB,EAAE,QAAQ,CAAC,YAAY;QAC1C,GAAG,CAAC,QAAQ,CAAC,cAAc,KAAK,SAAS;YACvC,CAAC,CAAC,EAAE,qBAAqB,EAAE,QAAQ,CAAC,cAAc,EAAE;YACpD,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AA6BD,SAAS,iBAAiB;IACxB,OAAO,IAAI,aAAa,CACtB,+JAA+J,EAC/J,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,yBAAyB,EAAE,CACnD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa;IACpB,OAAO,IAAI,aAAa,CACtB,uMAAuM,EACvM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAC/C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,UAAgC,EAChC,UAA8B,EAC9B,KAAyB;IAEzB,MAAM,UAAU,GAAG,yBAAyB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,UAAU,CAAC,SAAS,CAAC;IAChE,IAAI,KAAK,CAAC,iBAAiB,KAAK,IAAI,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS;QAAE,MAAM,iBAAiB,EAAE,CAAC;IACnE,OAAO,KAAK,CAAC,eAAe,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AACtD,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,UAAqC,EACrC,IAAe,EACf,UAA8B,EAC9B,KAAyB,EACzB,cAAuB;IAEvB,MAAM,UAAU,GAAG,qBAAqB,CAAC,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;IACvF,IAAI,UAAU,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,UAAU,CAAC,KAAK,CAAC;IAC5D,IAAI,KAAK,CAAC,iBAAiB,KAAK,IAAI,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS;QAAE,MAAM,aAAa,EAAE,CAAC;IAC3D,OAAO,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,UAAU,CAAC,UAAU,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAsB,EACtB,KAAyB;IAEzB,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,EAAE,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CACtC,aAAa,CAAC,KAAK,EACnB,kBAAkB,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,EACrD,KAAK,CACN,CAAC;IAEF,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,gBAAgB,GAAG,eAAe,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;QAC1D,CAAC,CAAC,MAAM,YAAY,CAAC,MAAM,EAAE,SAAS,EAAE,gBAAgB,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;QACzF,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,SAAS,GAAwB;QACrC,SAAS;QACT,KAAK;QACL,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;KAC9C,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAChE,MAAM,KAAK,CAAC,eAAe,CACzB,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE;YAChC,WAAW,EAAE,SAAS,CAAC,EAAE;YACzB,YAAY,EAAE,KAAK,CAAC,EAAE;YACtB,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC;SACjE,CAAC,CACH,CAAC;QACF,iBAAiB,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,OAAO,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,iBAAiB,EAAE,CAAC;AACtD,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,oBAAoB,CAAC,OAAqB,EAAE,IAAY;IACtE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK;SACzB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC;SAC/C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC;SACtD,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,OAAO,IAAI,aAAa,CACtB,GAAG,OAAO,CAAC,MAAM,OAAO,OAAO,CAAC,KAAK,kCAAkC,MAAM,GAAG,EAChF,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAC7B,CAAC;AACJ,CAAC"}

package/dist/actions/pack.d.ts

@@ -0,0 +1,36 @@
+import type { EvaluationContextType } from '../domain/types.js';
+import type { BundledPack } from '../packs/registry.js';
+import { type SuiteManifest } from '../suite/manifest.js';
+/** Pack installation planning and integrity-checked filesystem operations. */
+export type PackFileToWrite = {
+    absolutePath: string;
+    contents: string;
+};
+export type PackInstallPlan = {
+    packName: string;
+    context: EvaluationContextType;
+    /** Absolute root selected by the caller, containing the per-pack directory. */
+    targetDirectory: string;
+    /** Absolute `<targetDirectory>/<packName>` directory the pack installs into. */
+    installDirectory: string;
+    /** Absolute path of the generated suite manifest. */
+    manifestPath: string;
+    /** Pack files plus the generated manifest, in write order. */
+    files: readonly PackFileToWrite[];
+    /** Absolute eval file paths referenced by the manifest, in manifest order. */
+    evalFileAbsolutePaths: readonly string[];
+};
+/** Scenario packs grade a model under test; other contexts need only a judge. */
+export declare function packRequiresPrimaryModel(context: EvaluationContextType): boolean;
+export declare function planPackInstall(pack: BundledPack, targetDirectory: string, options?: {
+    workspaceId?: string;
+}): PackInstallPlan;
+export type PackInstallResult = {
+    status: 'installed' | 'already-installed';
+    manifest: SuiteManifest;
+};
+/** Installs a fresh pack and refuses every pre-existing pack file. */
+export declare function installPack(plan: PackInstallPlan): Promise<void>;
+/** Installs a missing pack, or reuses it only when every existing file is trusted. */
+export declare function ensurePackInstalled(plan: PackInstallPlan): Promise<PackInstallResult>;
+//# sourceMappingURL=pack.d.ts.map

package/dist/actions/pack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.d.ts","sourceRoot":"","sources":["../../src/actions/pack.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAsB,KAAK,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE9E,8EAA8E;AAC9E,MAAM,MAAM,eAAe,GAAG;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,qBAAqB,CAAC;IAC/B,+EAA+E;IAC/E,eAAe,EAAE,MAAM,CAAC;IACxB,gFAAgF;IAChF,gBAAgB,EAAE,MAAM,CAAC;IACzB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;IAClC,8EAA8E;IAC9E,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAC;CAC1C,CAAC;AAWF,iFAAiF;AACjF,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAEhF;AAoCD,wBAAgB,eAAe,CAC7B,IAAI,EAAE,WAAW,EACjB,eAAe,EAAE,MAAM,EACvB,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACrC,eAAe,CA4BjB;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,WAAW,GAAG,mBAAmB,CAAC;IAC1C,QAAQ,EAAE,aAAa,CAAC;CACzB,CAAC;AA6MF,sEAAsE;AACtE,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAWtE;AAED,sFAAsF;AACtF,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAe3F"}

package/dist/actions/pack.js

@@ -0,0 +1,271 @@
+import { lstat, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { isDeepStrictEqual } from 'node:util';
+import { dirname, isAbsolute, relative, resolve, sep } from 'node:path';
+import { stringify as stringifyYaml } from 'yaml';
+import { AutoevalError } from '../errors/autoeval-error.js';
+import { parseSuiteManifest } from '../suite/manifest.js';
+const MANIFEST_FILENAME = 'autoeval.suite.yaml';
+/**
+ * Synthetic placeholder workspace used only when no default is known yet. It is
+ * overridden at run time by `--workspace` or the resolved workspace, so a pack
+ * installed before a default is saved still runs once a workspace is resolved.
+ */
+const SYNTHETIC_WORKSPACE_PLACEHOLDER = 'ffffffff-ffff-4fff-8fff-ffffffffffff';
+/** Scenario packs grade a model under test; other contexts need only a judge. */
+export function packRequiresPrimaryModel(context) {
+    return context === 'scenario';
+}
+/** A pack name must be a single, safe path segment (it becomes a directory). */
+function assertSafePackName(name) {
+    if (name === '' || name.includes('/') || name.includes('\\') || name === '.' || name === '..') {
+        throw new AutoevalError(`Pack "${name}" has an unsafe name.`, {
+            kind: 'validation',
+            code: 'PACK_INVALID_PATH',
+        });
+    }
+}
+/**
+ * Resolves a pack-declared relative path under the install directory and
+ * confirms it does not escape it. Pack contents are treated as untrusted (a
+ * `..` or absolute entry must never write outside the install directory), even
+ * though the bundled packs are first-party today.
+ */
+function resolveContainedPath(installDirectory, relativePath) {
+    if (relativePath === '' || isAbsolute(relativePath)) {
+        throw new AutoevalError(`Pack path "${relativePath}" must be a relative path.`, {
+            kind: 'validation',
+            code: 'PACK_INVALID_PATH',
+        });
+    }
+    const absolutePath = resolve(installDirectory, relativePath);
+    const rel = relative(installDirectory, absolutePath);
+    if (rel === '' || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
+        throw new AutoevalError(`Pack path "${relativePath}" escapes the install directory.`, {
+            kind: 'validation',
+            code: 'PACK_INVALID_PATH',
+        });
+    }
+    return absolutePath;
+}
+export function planPackInstall(pack, targetDirectory, options = {}) {
+    assertSafePackName(pack.name);
+    const installDirectory = resolve(targetDirectory, pack.name);
+    const files = pack.files.map((file) => ({
+        absolutePath: resolveContainedPath(installDirectory, file.relativePath),
+        contents: file.contents,
+    }));
+    const manifestObject = {
+        workspace: options.workspaceId ?? SYNTHETIC_WORKSPACE_PLACEHOLDER,
+        ...(pack.gate === undefined ? {} : { gate: pack.gate }),
+        evals: [...pack.evals],
+    };
+    const manifestPath = resolveContainedPath(installDirectory, MANIFEST_FILENAME);
+    files.push({ absolutePath: manifestPath, contents: stringifyYaml(manifestObject) });
+    return {
+        packName: pack.name,
+        context: pack.context,
+        targetDirectory: resolve(targetDirectory),
+        installDirectory,
+        manifestPath,
+        files,
+        evalFileAbsolutePaths: pack.evals.map((evalPath) => resolveContainedPath(installDirectory, evalPath)),
+    };
+}
+function fileDisplayPath(plan, absolutePath) {
+    return relative(plan.targetDirectory, absolutePath) || '.';
+}
+function errorCode(error) {
+    return typeof error === 'object' && error !== null && 'code' in error
+        ? String(error.code)
+        : undefined;
+}
+async function assertNoSymlinkComponents(plan) {
+    let existingAncestor = plan.targetDirectory;
+    while (true) {
+        try {
+            const metadata = await lstat(existingAncestor);
+            if (metadata.isSymbolicLink()) {
+                throw new AutoevalError('A pack directory cannot be a symbolic link.', {
+                    kind: 'validation',
+                    code: 'PACK_INVALID_PATH',
+                });
+            }
+            if (!metadata.isDirectory()) {
+                throw new AutoevalError('A pack path component must be a directory.', {
+                    kind: 'validation',
+                    code: 'PACK_INVALID_PATH',
+                });
+            }
+            break;
+        }
+        catch (error) {
+            if (error instanceof AutoevalError)
+                throw error;
+            if (errorCode(error) !== 'ENOENT') {
+                throw new AutoevalError('A pack directory could not be inspected safely.', {
+                    kind: 'usage',
+                    code: 'PACK_WRITE_FAILED',
+                    cause: error,
+                });
+            }
+            const parent = dirname(existingAncestor);
+            if (parent === existingAncestor)
+                break;
+            existingAncestor = parent;
+        }
+    }
+    const directories = new Set([plan.targetDirectory, plan.installDirectory]);
+    for (const file of plan.files)
+        directories.add(dirname(file.absolutePath));
+    for (const target of directories) {
+        const rel = relative(plan.targetDirectory, target);
+        const segments = rel === '' ? [] : rel.split(sep);
+        let current = plan.targetDirectory;
+        for (let index = -1; index < segments.length; index += 1) {
+            if (index >= 0)
+                current = resolve(current, segments[index] ?? '');
+            try {
+                const metadata = await lstat(current);
+                if (metadata.isSymbolicLink()) {
+                    throw new AutoevalError('A pack directory cannot be a symbolic link.', {
+                        kind: 'validation',
+                        code: 'PACK_INVALID_PATH',
+                    });
+                }
+                if (!metadata.isDirectory()) {
+                    throw new AutoevalError('A pack path component must be a directory.', {
+                        kind: 'validation',
+                        code: 'PACK_INVALID_PATH',
+                    });
+                }
+            }
+            catch (error) {
+                if (error instanceof AutoevalError)
+                    throw error;
+                if (errorCode(error) === 'ENOENT')
+                    break;
+                throw new AutoevalError('A pack directory could not be inspected safely.', {
+                    kind: 'usage',
+                    code: 'PACK_WRITE_FAILED',
+                    cause: error,
+                });
+            }
+        }
+    }
+}
+async function readExistingFile(plan, file) {
+    try {
+        const metadata = await lstat(file.absolutePath);
+        if (metadata.isSymbolicLink()) {
+            throw new AutoevalError(`Existing pack path ${fileDisplayPath(plan, file.absolutePath)} cannot be a symbolic link.`, { kind: 'validation', code: 'PACK_FILES_MODIFIED' });
+        }
+        if (!metadata.isFile()) {
+            throw new AutoevalError(`Existing pack path ${fileDisplayPath(plan, file.absolutePath)} is not a regular file.`, { kind: 'validation', code: 'PACK_FILES_MODIFIED' });
+        }
+        const expectedBytes = Buffer.byteLength(file.contents, 'utf8');
+        if (metadata.size !== expectedBytes && file.absolutePath !== plan.manifestPath) {
+            return { file, contents: '' };
+        }
+        const contents = await readFile(file.absolutePath, 'utf8');
+        return { file, contents };
+    }
+    catch (error) {
+        if (error instanceof AutoevalError)
+            throw error;
+        if (errorCode(error) === 'ENOENT')
+            return undefined;
+        throw new AutoevalError(`Existing pack path ${fileDisplayPath(plan, file.absolutePath)} could not be inspected.`, { kind: 'usage', code: 'PACK_WRITE_FAILED', cause: error });
+    }
+}
+async function writePlanTransactionally(plan) {
+    const createdFiles = [];
+    const createdDirectories = [];
+    const rollback = async () => {
+        for (const filePath of createdFiles.reverse()) {
+            await rm(filePath, { force: true }).catch(() => undefined);
+        }
+        for (const directoryPath of createdDirectories.reverse()) {
+            await rm(directoryPath, { recursive: true, force: true }).catch(() => undefined);
+        }
+    };
+    try {
+        for (const file of plan.files) {
+            const firstCreatedDirectory = await mkdir(dirname(file.absolutePath), { recursive: true });
+            if (firstCreatedDirectory !== undefined)
+                createdDirectories.push(firstCreatedDirectory);
+            try {
+                await writeFile(file.absolutePath, file.contents, { encoding: 'utf8', flag: 'wx' });
+                createdFiles.push(file.absolutePath);
+            }
+            catch (error) {
+                if (errorCode(error) === 'EEXIST') {
+                    throw new AutoevalError(`A pack file already exists at ${fileDisplayPath(plan, file.absolutePath)}. Remove it or choose a different --dir.`, { kind: 'usage', code: 'PACK_FILE_EXISTS', cause: error });
+                }
+                throw new AutoevalError(`A pack file could not be written at ${fileDisplayPath(plan, file.absolutePath)}.`, { kind: 'usage', code: 'PACK_WRITE_FAILED', cause: error });
+            }
+        }
+    }
+    catch (error) {
+        await rollback();
+        throw error;
+    }
+}
+function expectedManifest(plan) {
+    const manifestFile = plan.files.find((file) => file.absolutePath === plan.manifestPath);
+    if (manifestFile === undefined) {
+        throw new AutoevalError('The bundled pack does not contain its suite manifest.', {
+            kind: 'validation',
+            code: 'PACK_INVALID_PATH',
+        });
+    }
+    return parseSuiteManifest(manifestFile.contents, plan.manifestPath);
+}
+function assertTrustedExistingPack(plan, existingFiles) {
+    for (const existing of existingFiles) {
+        if (existing.file.absolutePath === plan.manifestPath)
+            continue;
+        if (existing.contents !== existing.file.contents) {
+            throw new AutoevalError(`Pack file ${fileDisplayPath(plan, existing.file.absolutePath)} differs from the bundled pack. Run it explicitly with "autoeval suite run --manifest ${fileDisplayPath(plan, plan.manifestPath)}" or remove the pack directory and reinstall it.`, { kind: 'validation', code: 'PACK_FILES_MODIFIED' });
+        }
+    }
+    const actualSource = existingFiles.find((existing) => existing.file.absolutePath === plan.manifestPath)?.contents;
+    if (actualSource === undefined) {
+        throw new AutoevalError('The installed pack is missing its suite manifest.', {
+            kind: 'validation',
+            code: 'PACK_FILES_MODIFIED',
+        });
+    }
+    const actual = parseSuiteManifest(actualSource, plan.manifestPath);
+    const expected = expectedManifest(plan);
+    const sameFiles = isDeepStrictEqual(actual.evalFiles, expected.evalFiles);
+    const samePolicies = isDeepStrictEqual(actual.entries.map((entry) => entry.gate), expected.entries.map((entry) => entry.gate));
+    if (!sameFiles || !samePolicies) {
+        throw new AutoevalError(`Pack manifest ${fileDisplayPath(plan, plan.manifestPath)} differs from the bundled pack. Run the adapted manifest with "autoeval suite run --manifest ${fileDisplayPath(plan, plan.manifestPath)}" or remove the pack directory and reinstall it.`, { kind: 'validation', code: 'PACK_FILES_MODIFIED' });
+    }
+    return actual;
+}
+/** Installs a fresh pack and refuses every pre-existing pack file. */
+export async function installPack(plan) {
+    await assertNoSymlinkComponents(plan);
+    const existing = await Promise.all(plan.files.map((file) => readExistingFile(plan, file)));
+    const collision = existing.find((entry) => entry !== undefined);
+    if (collision !== undefined) {
+        throw new AutoevalError(`A pack file already exists at ${fileDisplayPath(plan, collision.file.absolutePath)}. Remove it or choose a different --dir.`, { kind: 'usage', code: 'PACK_FILE_EXISTS' });
+    }
+    await writePlanTransactionally(plan);
+}
+/** Installs a missing pack, or reuses it only when every existing file is trusted. */
+export async function ensurePackInstalled(plan) {
+    await assertNoSymlinkComponents(plan);
+    const inspected = await Promise.all(plan.files.map((file) => readExistingFile(plan, file)));
+    const existing = inspected.filter((entry) => entry !== undefined);
+    if (existing.length === 0) {
+        await writePlanTransactionally(plan);
+        return { status: 'installed', manifest: expectedManifest(plan) };
+    }
+    if (existing.length !== plan.files.length) {
+        throw new AutoevalError('The pack directory contains only part of the bundled pack. Remove the directory and reinstall it.', { kind: 'validation', code: 'PACK_FILES_MODIFIED' });
+    }
+    return { status: 'already-installed', manifest: assertTrustedExistingPack(plan, existing) };
+}
+//# sourceMappingURL=pack.js.map

package/dist/actions/pack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pack.js","sourceRoot":"","sources":["../../src/actions/pack.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAExE,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AAGlD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,kBAAkB,EAAsB,MAAM,sBAAsB,CAAC;AAuB9E,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;AAEhD;;;;GAIG;AACH,MAAM,+BAA+B,GAAG,sCAAsC,CAAC;AAE/E,iFAAiF;AACjF,MAAM,UAAU,wBAAwB,CAAC,OAA8B;IACrE,OAAO,OAAO,KAAK,UAAU,CAAC;AAChC,CAAC;AAED,gFAAgF;AAChF,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,IAAI,KAAK,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAC9F,MAAM,IAAI,aAAa,CAAC,SAAS,IAAI,uBAAuB,EAAE;YAC5D,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,mBAAmB;SAC1B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAAC,gBAAwB,EAAE,YAAoB;IAC1E,IAAI,YAAY,KAAK,EAAE,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,aAAa,CAAC,cAAc,YAAY,4BAA4B,EAAE;YAC9E,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,mBAAmB;SAC1B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,QAAQ,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IACrD,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,aAAa,CAAC,cAAc,YAAY,kCAAkC,EAAE;YACpF,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,mBAAmB;SAC1B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,IAAiB,EACjB,eAAuB,EACvB,UAAoC,EAAE;IAEtC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,gBAAgB,GAAG,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7D,MAAM,KAAK,GAAsB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzD,YAAY,EAAE,oBAAoB,CAAC,gBAAgB,EAAE,IAAI,CAAC,YAAY,CAAC;QACvE,QAAQ,EAAE,IAAI,CAAC,QAAQ;KACxB,CAAC,CAAC,CAAC;IAEJ,MAAM,cAAc,GAAG;QACrB,SAAS,EAAE,OAAO,CAAC,WAAW,IAAI,+BAA+B;QACjE,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACvD,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;KACvB,CAAC;IACF,MAAM,YAAY,GAAG,oBAAoB,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAEpF,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,eAAe,EAAE,OAAO,CAAC,eAAe,CAAC;QACzC,gBAAgB;QAChB,YAAY;QACZ,KAAK;QACL,qBAAqB,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjD,oBAAoB,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CACjD;KACF,CAAC;AACJ,CAAC;AASD,SAAS,eAAe,CAAC,IAAqB,EAAE,YAAoB;IAClE,OAAO,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,IAAI,GAAG,CAAC;AAC7D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,IAAI,KAAK;QACnE,CAAC,CAAC,MAAM,CAAE,KAA4B,CAAC,IAAI,CAAC;QAC5C,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,IAAqB;IAC5D,IAAI,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC;IAC5C,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAC/C,IAAI,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC9B,MAAM,IAAI,aAAa,CAAC,6CAA6C,EAAE;oBACrE,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,mBAAmB;iBAC1B,CAAC,CAAC;YACL,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC5B,MAAM,IAAI,aAAa,CAAC,4CAA4C,EAAE;oBACpE,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,mBAAmB;iBAC1B,CAAC,CAAC;YACL,CAAC;YACD,MAAM;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,aAAa;gBAAE,MAAM,KAAK,CAAC;YAChD,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClC,MAAM,IAAI,aAAa,CAAC,iDAAiD,EAAE;oBACzE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,KAAK;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACzC,IAAI,MAAM,KAAK,gBAAgB;gBAAE,MAAM;YACvC,gBAAgB,GAAG,MAAM,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAS,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACnF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK;QAAE,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAE3E,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;QACnD,MAAM,QAAQ,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClD,IAAI,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QACnC,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,IAAI,KAAK,IAAI,CAAC;gBAAE,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;gBACtC,IAAI,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC;oBAC9B,MAAM,IAAI,aAAa,CAAC,6CAA6C,EAAE;wBACrE,IAAI,EAAE,YAAY;wBAClB,IAAI,EAAE,mBAAmB;qBAC1B,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC5B,MAAM,IAAI,aAAa,CAAC,4CAA4C,EAAE;wBACpE,IAAI,EAAE,YAAY;wBAClB,IAAI,EAAE,mBAAmB;qBAC1B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,aAAa;oBAAE,MAAM,KAAK,CAAC;gBAChD,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ;oBAAE,MAAM;gBACzC,MAAM,IAAI,aAAa,CAAC,iDAAiD,EAAE;oBACzE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,mBAAmB;oBACzB,KAAK,EAAE,KAAK;iBACb,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,IAAqB,EACrB,IAAqB;IAErB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,cAAc,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,aAAa,CACrB,sBAAsB,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,6BAA6B,EAC3F,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAE,CACpD,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACvB,MAAM,IAAI,aAAa,CACrB,sBAAsB,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,yBAAyB,EACvF,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAE,CACpD,CAAC;QACJ,CAAC;QACD,MAAM,aAAa,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAChC,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,aAAa;YAAE,MAAM,KAAK,CAAC;QAChD,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QACpD,MAAM,IAAI,aAAa,CACrB,sBAAsB,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,0BAA0B,EACxF,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,EAAE,CAC3D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,IAAqB;IAC3D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,kBAAkB,GAAa,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,KAAK,IAAmB,EAAE;QACzC,KAAK,MAAM,QAAQ,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7D,CAAC;QACD,KAAK,MAAM,aAAa,IAAI,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,MAAM,EAAE,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACnF,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,qBAAqB,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3F,IAAI,qBAAqB,KAAK,SAAS;gBAAE,kBAAkB,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACxF,IAAI,CAAC;gBACH,MAAM,SAAS,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBACpF,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACvC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;oBAClC,MAAM,IAAI,aAAa,CACrB,iCAAiC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,0CAA0C,EACnH,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,CAC1D,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,aAAa,CACrB,uCAAuC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,GAAG,EAClF,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,KAAK,EAAE,CAC3D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,EAAE,CAAC;QACjB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAqB;IAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,YAAY,CAAC,CAAC;IACxF,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,aAAa,CAAC,uDAAuD,EAAE;YAC/E,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,mBAAmB;SAC1B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,kBAAkB,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,yBAAyB,CAChC,IAAqB,EACrB,aAAsC;IAEtC,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;QACrC,IAAI,QAAQ,CAAC,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,YAAY;YAAE,SAAS;QAC/D,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjD,MAAM,IAAI,aAAa,CACrB,aAAa,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,yFAAyF,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,kDAAkD,EACjP,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAE,CACpD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CACrC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,YAAY,CAC/D,EAAE,QAAQ,CAAC;IACZ,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,aAAa,CAAC,mDAAmD,EAAE;YAC3E,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,qBAAqB;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,MAAM,MAAM,GAAG,kBAAkB,CAAC,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,iBAAiB,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC1E,MAAM,YAAY,GAAG,iBAAiB,CACpC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EACzC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAC5C,CAAC;IACF,IAAI,CAAC,SAAS,IAAI,CAAC,YAAY,EAAE,CAAC;QAChC,MAAM,IAAI,aAAa,CACrB,iBAAiB,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,gGAAgG,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,kDAAkD,EACnP,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAE,CACpD,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAqB;IACrD,MAAM,yBAAyB,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAChE,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,MAAM,IAAI,aAAa,CACrB,iCAAiC,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,0CAA0C,EAC7H,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAC5C,CAAC;IACJ,CAAC;IACD,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AAED,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAqB;IAC7D,MAAM,yBAAyB,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAyB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACzF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,wBAAwB,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAC1C,MAAM,IAAI,aAAa,CACrB,mGAAmG,EACnG,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,qBAAqB,EAAE,CACpD,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,yBAAyB,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;AAC9F,CAAC"}

package/dist/actions/suite.d.ts

@@ -2,6 +2,7 @@ import type { JsonObject } from '../domain/common.js';
 import type { EvaluationContextType, EvaluationResults, RunStatus } from '../domain/types.js';
 import { type ConfiguredRunModelOverrides, type ParsedConfiguredRunInput } from '../configured-run/validation.js';
 import type { PollClock } from '../polling/clock.js';
+import { type RunStandardEvidence } from '../reliability/evidence.js';
 import { type SuiteEvalRecord, type SuiteItem } from '../suite/execution.js';
 import type { ActionContext } from './context.js';
 /** Per-eval identity and terminal execution state, preserved for the summary. */
@@ -14,6 +15,8 @@ export type SuiteEvalExecution = {
     elapsedMs: number;
     /** Terminal run-status payload, kept for the gate plane. */
     statusRaw: JsonObject;
+    /** Public-safe metadata from the exact methodology/configuration used by this run. */
+    standard?: RunStandardEvidence;
 };
 export type SuiteEvalEntry = SuiteEvalRecord<SuiteEvalExecution, EvaluationResults>;
 export type SuiteSummary = {
@@ -39,6 +42,7 @@ export type RunSuiteInput = {
     signal?: AbortSignal;
     onStatus?: (item: SuiteItem, status: RunStatus) => void;
     modelOverrides?: ConfiguredRunModelOverrides;
+    includeReliabilityEvidence?: boolean;
 };
 /**
  * Runs a suite in two planes: execution (bounded concurrency) and result

package/dist/actions/suite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../../src/actions/suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC9F,OAAO,EAGL,KAAK,2BAA2B,EAChC,KAAK,wBAAwB,EAC9B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAoB,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE/F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAIlD,iFAAiF;AACjF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,qBAAqB,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,SAAS,EAAE,UAAU,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,eAAe,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAC;AAEpF,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,sFAAsF;IACtF,YAAY,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,wBAAwB,CAAC,CAAC;IACvE,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACxD,cAAc,CAAC,EAAE,2BAA2B,CAAC;CAC9C,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,YAAY,CAAC,CA+DvB"}
+{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../../src/actions/suite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAGL,KAAK,2BAA2B,EAChC,KAAK,wBAAwB,EAC9B,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAA8B,KAAK,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAoB,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAE/F,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAIlD,iFAAiF;AACjF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,qBAAqB,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,SAAS,EAAE,UAAU,CAAC;IACtB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,mBAAmB,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,eAAe,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,CAAC;AAEpF,MAAM,MAAM,YAAY,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9B,sFAAsF;IACtF,YAAY,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,wBAAwB,CAAC,CAAC;IACvE,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACxD,cAAc,CAAC,EAAE,2BAA2B,CAAC;IAC7C,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACtC,CAAC;AAEF;;;;GAIG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,YAAY,CAAC,CAsFvB"}

package/dist/actions/suite.js

@@ -1,4 +1,6 @@
+import { jsonObjectSchema } from '../api/schemas.js';
 import { applyModelOverrides, requireModelOverridesForPublicPlaceholders, } from '../configured-run/validation.js';
+import { extractRunStandardEvidence } from '../reliability/evidence.js';
 import { runTwoPlaneSuite } from '../suite/execution.js';
 import { runConfiguredEvaluation, validateConfiguredEvaluation } from './configured-runs.js';
 import { createEvaluation } from './evaluations.js';
@@ -37,6 +39,22 @@ export async function runSuite(context, input) {
                     ? { onStatus: (status) => input.onStatus?.(item, status) }
                     : {}),
             });
+            let standard;
+            if (input.includeReliabilityEvidence === true) {
+                let exactConfiguration;
+                try {
+                    const response = await context.api.getEvaluationConfiguration(evaluation.evaluationId, execution.run.methodologyVersionId, execution.run.configVersionId, input.signal);
+                    if (response.evaluation_id === evaluation.evaluationId &&
+                        response.methodology_version_id === execution.run.methodologyVersionId &&
+                        response.config_version_id === execution.run.configVersionId) {
+                        exactConfiguration = jsonObjectSchema.parse(response);
+                    }
+                }
+                catch {
+                    // Reporting metadata is optional and must never alter execution or gate behavior.
+                }
+                standard = extractRunStandardEvidence(configuredRun, exactConfiguration);
+            }
             return {
                 evaluationId: evaluation.evaluationId,
                 evaluationName,
@@ -45,6 +63,7 @@ export async function runSuite(context, input) {
                 state: execution.outcome.status.state,
                 elapsedMs: execution.outcome.elapsedMs,
                 statusRaw: execution.outcome.status.raw,
+                ...(standard === undefined ? {} : { standard }),
             };
         },
         fetchResults: (execution) => getResults(context, {

package/dist/actions/suite.js.map

@@ -1 +1 @@
-{"version":3,"file":"suite.js","sourceRoot":"","sources":["../../src/actions/suite.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,mBAAmB,EACnB,0CAA0C,GAG3C,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,gBAAgB,EAAwC,MAAM,uBAAuB,CAAC;AAC/F,OAAO,EAAE,uBAAuB,EAAE,4BAA4B,EAAE,MAAM,sBAAsB,CAAC;AAE7F,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AA0C1C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,OAAsB,EACtB,KAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAwC;QAC1E,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,oBAAoB,EAAE,KAAK,CAAC,WAAW;QACvC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,iBAAiB,KAAK,SAAS;YACvC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC;QACnD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACtB,MAAM,mBAAmB,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,mBAAmB,CAAC,mBAAmB,EAAE,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;YAC3F,0CAA0C,CAAC,aAAa,CAAC,CAAC;YAC1D,MAAM,4BAA4B,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACzE,MAAM,cAAc,GAClB,aAAa,CAAC,cAAc,IAAI,aAAa,CAAC,aAAa,CAAC,WAAW,CAAC;YAC1E,MAAM,UAAU,GAAG,MAAM,gBAAgB,CACvC,OAAO,EACP,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,cAAc,EAAE,EAClD,KAAK,CAAC,MAAM,CACb,CAAC;YACF,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,OAAO,EAAE;gBACvD,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,aAAa;gBACb,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,YAAY,EAAE,IAAI;gBAClB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjD,GAAG,CAAC,KAAK,CAAC,QAAQ;oBAChB,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,MAAiB,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;oBACrE,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CAAC;YACH,OAAO;gBACL,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,cAAc;gBACd,WAAW,EAAE,aAAa,CAAC,aAAa,CAAC,WAAW;gBACpD,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,KAAK;gBAC1B,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK;gBACrC,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,SAAS;gBACtC,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;aACxC,CAAC;QACJ,CAAC;QACD,YAAY,EAAE,CAAC,SAAS,EAAE,EAAE,CAC1B,UAAU,CAAC,OAAO,EAAE;YAClB,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;YAClD,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;YAChD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;KACL,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IAC/E,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,SAAS;QACT,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,SAAS;QAChC,KAAK;KACN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"suite.js","sourceRoot":"","sources":["../../src/actions/suite.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EACL,mBAAmB,EACnB,0CAA0C,GAG3C,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,0BAA0B,EAA4B,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAE,gBAAgB,EAAwC,MAAM,uBAAuB,CAAC;AAC/F,OAAO,EAAE,uBAAuB,EAAE,4BAA4B,EAAE,MAAM,sBAAsB,CAAC;AAE7F,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AA6C1C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,OAAsB,EACtB,KAAoB;IAEpB,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAwC;QAC1E,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,oBAAoB,EAAE,KAAK,CAAC,WAAW;QACvC,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,iBAAiB,KAAK,SAAS;YACvC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC;QACnD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACtB,MAAM,mBAAmB,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,aAAa,GAAG,mBAAmB,CAAC,mBAAmB,EAAE,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;YAC3F,0CAA0C,CAAC,aAAa,CAAC,CAAC;YAC1D,MAAM,4BAA4B,CAAC,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACzE,MAAM,cAAc,GAClB,aAAa,CAAC,cAAc,IAAI,aAAa,CAAC,aAAa,CAAC,WAAW,CAAC;YAC1E,MAAM,UAAU,GAAG,MAAM,gBAAgB,CACvC,OAAO,EACP,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,cAAc,EAAE,EAClD,KAAK,CAAC,MAAM,CACb,CAAC;YACF,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,OAAO,EAAE;gBACvD,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,aAAa;gBACb,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,aAAa,EAAE,KAAK,CAAC,aAAa;gBAClC,YAAY,EAAE,IAAI;gBAClB,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjD,GAAG,CAAC,KAAK,CAAC,QAAQ;oBAChB,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,MAAiB,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE;oBACrE,CAAC,CAAC,EAAE,CAAC;aACR,CAAC,CAAC;YACH,IAAI,QAAyC,CAAC;YAC9C,IAAI,KAAK,CAAC,0BAA0B,KAAK,IAAI,EAAE,CAAC;gBAC9C,IAAI,kBAA0C,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAC3D,UAAU,CAAC,YAAY,EACvB,SAAS,CAAC,GAAG,CAAC,oBAAoB,EAClC,SAAS,CAAC,GAAG,CAAC,eAAe,EAC7B,KAAK,CAAC,MAAM,CACb,CAAC;oBACF,IACE,QAAQ,CAAC,aAAa,KAAK,UAAU,CAAC,YAAY;wBAClD,QAAQ,CAAC,sBAAsB,KAAK,SAAS,CAAC,GAAG,CAAC,oBAAoB;wBACtE,QAAQ,CAAC,iBAAiB,KAAK,SAAS,CAAC,GAAG,CAAC,eAAe,EAC5D,CAAC;wBACD,kBAAkB,GAAG,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,kFAAkF;gBACpF,CAAC;gBACD,QAAQ,GAAG,0BAA0B,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,OAAO;gBACL,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,cAAc;gBACd,WAAW,EAAE,aAAa,CAAC,aAAa,CAAC,WAAW;gBACpD,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,KAAK;gBAC1B,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK;gBACrC,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,SAAS;gBACtC,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG;gBACvC,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;aAChD,CAAC;QACJ,CAAC;QACD,YAAY,EAAE,CAAC,SAAS,EAAE,EAAE,CAC1B,UAAU,CAAC,OAAO,EAAE;YAClB,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,qBAAqB,EAAE,KAAK,CAAC,qBAAqB;YAClD,oBAAoB,EAAE,KAAK,CAAC,oBAAoB;YAChD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;KACL,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;IAC/E,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,SAAS;QACT,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,SAAS;QAChC,KAAK;KACN,CAAC;AACJ,CAAC"}

package/dist/api/api.d.ts

@@ -1,4 +1,5 @@
 import type { JsonObject } from '../domain/common.js';
+import { type ExternalImportRequest } from './external-import.js';
 import type { EvaluationPage, EvaluationSummary, RunStatus, SupportedModel, UserIdentity, Workspace, WorkspacePage } from '../domain/types.js';
 import type { ApiClient } from './client.js';
 import { type ConfigVersionCreateResponse, type EvaluationCreateResponse, type EvaluationConfigurationResponse, type EvaluationDraftResponse, type EvaluationVersionsResponse, type MethodologyVersionCreateResponse, type RunCreateResponse } from './schemas.js';
@@ -13,6 +14,7 @@ export type CreateRunInput = {
     signal?: AbortSignal;
 };
 export interface AutoevalApi {
+    externalImport(request: ExternalImportRequest): Promise<unknown>;
     getCurrentUser(signal?: AbortSignal): Promise<UserIdentity>;
     listWorkspaces(input?: PageInput, signal?: AbortSignal): Promise<WorkspacePage>;
     createWorkspace(input: {
@@ -125,6 +127,7 @@ export interface AutoevalApi {
 export declare class AutoevalApiClient implements AutoevalApi {
     #private;
     constructor(client: ApiClient);
+    externalImport(request: ExternalImportRequest): Promise<unknown>;
     getCurrentUser(signal?: AbortSignal): Promise<UserIdentity>;
     listWorkspaces(input?: PageInput, signal?: AbortSignal): Promise<WorkspacePage>;
     createWorkspace(input: {