@alisio/core 0.5.2 → 0.7.0

package/dist/eval/engine.d.ts

@@ -0,0 +1,144 @@
+/**
+ * Live engine of the reliability lab (specs/010 Phase 2): runs ONE task repetition through the
+ * real Application (runner, tools, store) on a throwaway copy of its fixture and returns an
+ * `EvalRunRecord` with the conditions it was measured under.
+ *
+ * What the isolation guarantees, and what it does not:
+ * - The fixture is copied; the original is never touched. Config, state, home and tmp live in the
+ *   run directory and the process environment points at them while the application exists.
+ * - File tools are mediated: a write to `../x` or to an absolute path outside the copy is refused
+ *   (`safePath`). That is path confinement, NOT an OS sandbox.
+ * - A granted `process` effect (or a `command` verifier) runs with the user's privileges and can
+ *   write anywhere the user can. When `process` is granted the "nothing outside changes" claim does
+ *   not hold. A subprocess is not a sandbox.
+ *
+ * Runs never overlap: `ALISIO_CONFIG_HOME`/`ALISIO_STATE_HOME` are process-global, so `runTask`
+ * serializes concurrent callers.
+ */
+import type { EvalBudget, EvalRunRecord, EvalTask, ModelProvider, RunEvent } from "@alisio/sdk";
+import { type BuiltinPlugin, createApplication } from "../application.ts";
+import { type Config } from "../config.ts";
+import type { RunOptions } from "../core/runner.ts";
+import { type Cassette, CassetteRecorder } from "./cassette.ts";
+import type { LoadedSuite } from "./suite.ts";
+export type EvalApplication = Awaited<ReturnType<typeof createApplication>>;
+type BudgetKey = NonNullable<EvalRunRecord["budgetExceeded"]>;
+/** The run was refused before it started (policy, `full` without opt-in, unsafe fixture). */
+export declare class EvalRunRefusedError extends Error {
+    constructor(message: string);
+}
+/** Abort reason of the engine's own budgets (`maxWallMs`, `maxToolCalls`). */
+export declare class EvalBudgetExceeded extends Error {
+    readonly budget: BudgetKey;
+    constructor(budget: BudgetKey);
+}
+/** Effects the invoker (the person or CI job running the lab) holds. A task can never exceed them. */
+export interface InvokerPermissions {
+    write: boolean;
+    process: boolean;
+    external: boolean;
+    analysis: boolean;
+    readOnly?: boolean;
+}
+/**
+ * `live` talks to `provider`; `record` is a live run whose provider calls are also written into the
+ * `CassetteRecorder` passed as `cassette`; `replay` serves the recorded `Cassette` passed as
+ * `cassette` and needs no provider. Record and replay install a seeded id source for the run (so
+ * session, plan and execution ids do not depend on fresh randomness) and wait for the model
+ * catalog before the first request (so the output limit does not depend on a race).
+ */
+export type EvalRunMode = "live" | "record" | "replay";
+export interface RunTaskOptions {
+    suite: LoadedSuite;
+    /**
+     * Already resolved by the caller; injected into the application (no network lookups here).
+     * Required except in `replay` mode, where the cassette supplies the provider.
+     */
+    provider?: ModelProvider;
+    mode?: EvalRunMode;
+    cassette?: CassetteRecorder | Cassette;
+    /** Receives every `RunEvent` of the run (tests and tooling; the engine keeps its own counts). */
+    onEvent?: (event: RunEvent) => void;
+    /**
+     * Written (stripped of plugins, skills, MCP, extra roots, scratch root and memory `dbPath`) as the global
+     * config layer of the run. Phase 3 variant overrides (D15) MUST be applied to this object before
+     * calling: the `configHash` is computed from what is written.
+     */
+    config: Config;
+    /** Orders runs only. The provider never receives it, so it is not determinism. */
+    seed: number;
+    repetition: number;
+    invoker: InvokerPermissions;
+    /** Required for a task asking for `full`, regardless of what the invoker holds. */
+    allowSuiteFull?: boolean;
+    keep?: boolean;
+    builtins?: BuiltinPlugin[];
+    /** Turns `task.agent` into run options; without it a task naming an agent ends `error`. */
+    resolveAgent?: (app: EvalApplication, agent: string) => RunOptions | Promise<RunOptions>;
+    /** Parent of the run directory (default `os.tmpdir()`). */
+    tmpRoot?: string;
+}
+/**
+ * Which top-level config keys feed `configHash`. The `satisfies` clause fails the typecheck, and
+ * `tests/eval-engine.test.ts` fails at runtime, when a key is added to the schema without being
+ * classified here: a new setting must be consciously hashed or excluded.
+ */
+export declare const CONFIG_HASH_COVERAGE: {
+    readonly limits: "hashed";
+    readonly compaction: "hashed";
+    readonly context: "hashed";
+    readonly forge: "hashed";
+    readonly analysis: "hashed";
+    readonly websearch: "hashed";
+    readonly decisions: "hashed";
+    readonly skillOverrides: "hashed";
+    readonly schemaVersion: "excluded";
+    readonly provider: "excluded";
+    readonly plugins: "excluded";
+    readonly skills: "excluded";
+    readonly additionalDirectories: "excluded";
+    readonly mcp: "excluded";
+    readonly builtinPlugins: "excluded";
+    readonly pluginOverrides: "excluded";
+    readonly pluginHooks: "excluded";
+    readonly tasks: "excluded";
+    readonly goal: "excluded";
+    readonly plan: "excluded";
+    readonly tui: "excluded";
+    readonly agents: "excluded";
+    readonly web: "excluded";
+};
+export interface ConfigHashExtras {
+    /** The effective policy of the run after the invoker check and `readOnly`. */
+    policy: {
+        write: boolean;
+        process: boolean;
+        external: boolean;
+        analysis: boolean;
+        readOnly?: boolean;
+    };
+    approvalMode: string;
+    budget: EvalBudget;
+    /** Ids of the built-in plugins handed to the application (kept only when not disabled). */
+    builtinIds: string[];
+}
+/**
+ * `sha256:` of the allowlisted effective configuration plus policy, approval mode, budget and
+ * enabled built-ins. Excludes secrets, env names, plugin options, absolute paths, endpoints, the
+ * provider and the model (those are separate columns of the run record).
+ */
+export declare function computeConfigHash(config: Config, extras: ConfigHashExtras): string;
+/**
+ * Throws the same `EvalRunRefusedError` `runTask` would, without running anything. Lets a caller
+ * refuse a whole batch up front instead of discovering a refusal after earlier runs finished.
+ */
+export declare function checkTaskPolicy(task: EvalTask, opts: Pick<RunTaskOptions, "invoker" | "allowSuiteFull">): void;
+/**
+ * Runs one repetition of `task`. Refusals (`EvalRunRefusedError`) reject before anything is
+ * created; infrastructure failures resolve with `status: "error"` and `error` set.
+ */
+export declare function runTask(task: EvalTask, opts: RunTaskOptions): Promise<{
+    record: EvalRunRecord;
+    keptDir?: string;
+}>;
+export {};