@alisio/alisio-code 0.6.0 → 0.8.0

package/dist/eval.d.ts

@@ -0,0 +1,101 @@
+/**
+ * `alisio eval list|run|record|compare|variants|export` (specs/010 5.4, Phases 3-4). Every function
+ * returns the exit code and talks only through the injected `EvalEnv`, so tests drive them without
+ * a process:
+ *
+ *   0  completed (independent of which tasks passed)   1  usage or infrastructure error
+ *   2  a replay diverged from its cassette             3  `compare` of incompatible suites
+ *
+ * `record` runs each task once live and writes `<suite>/cassettes/<taskId>.cassette.jsonl`;
+ * `run --mode replay` serves those cassettes (no provider, no credentials, no network). `export`
+ * writes one local, scrubbed JSON bundle of a stored run (nothing is uploaded). Reports go to `<stateHome>/eval/<runId>/` (or `--out DIR`): report.json, report.md, runs.jsonl.
+ * A suite given with `--suite-dir` is third-party code: it needs `--yes` (or an interactive
+ * confirmation) because a subprocess is not a sandbox.
+ */
+import type { BuiltinPlugin, Config, EvalApplication } from "@alisio/core";
+import type { ModelProvider } from "@alisio/sdk";
+/** The part of the command line `eval` reads, global flags included. */
+export interface EvalFlags {
+    json?: boolean;
+    allowWrite?: boolean;
+    allowProcess?: boolean;
+    allowExternal?: boolean;
+    allowAnalysis?: boolean;
+    readOnly?: boolean;
+    disablePlugin?: string[];
+    suiteDir?: string;
+    tasks?: string;
+    k?: string;
+    seed?: string;
+    mode?: string;
+    keep?: boolean;
+    out?: string;
+    allowSuiteFull?: boolean;
+    yes?: boolean;
+    inputRate?: string;
+    outputRate?: string;
+    variants?: string;
+    md?: boolean;
+    cassetteDir?: string;
+    force?: boolean;
+}
+/** What a live run needs, resolved BEFORE any task starts (the engine takes a provider). */
+export interface EvalRuntime {
+    provider: ModelProvider;
+    /** The effective configuration (variant overrides are applied on top of it). */
+    config: Config;
+    builtins: BuiltinPlugin[];
+    /** Exact credential values of the run's provider profile (a cassette holding one is refused). */
+    secrets?: string[];
+    resolveAgent?: (app: EvalApplication, agent: string) => ReturnType<NonNullable<ResolveAgent>>;
+    close(): Promise<void>;
+}
+type ResolveAgent = NonNullable<import("@alisio/core").RunEvaluationOptions["resolveAgent"]>;
+export interface EvalEnv {
+    out(text: string): void;
+    err(text: string): void;
+    /** `<stateHome>`: reports default to `<stateDir>/eval/<runId>/`. */
+    stateDir: string;
+    /** Directory of the bundled suites: `tests/fixtures/eval` of a source checkout, absent elsewhere. */
+    bundledDir?: string;
+    interactive: boolean;
+    confirm(question: string): Promise<boolean>;
+    /**
+     * `requireProvider: false` (replay) must not fail when no provider is configured: the cassette
+     * is the provider, and the returned `provider` is then ignored.
+     */
+    openRuntime(flags: EvalFlags, options?: {
+        requireProvider?: boolean;
+    }): Promise<EvalRuntime>;
+    newRunId(): string;
+    /** Parent of the per-run temporary directories (tests); default `os.tmpdir()`. */
+    tmpRoot?: string;
+}
+export declare function evalList(env: EvalEnv, flags: EvalFlags): Promise<number>;
+export declare function evalRun(env: EvalEnv, flags: EvalFlags, suiteName: string): Promise<number>;
+/**
+ * Runs each selected task ONCE live and writes its cassette. A task whose run errored, whose
+ * recording is unstable, or whose cassette holds a secret or is over the size limits writes
+ * nothing and makes the command exit 1; the other tasks are still recorded.
+ */
+export declare function evalRecord(env: EvalEnv, flags: EvalFlags, suiteName: string): Promise<number>;
+export declare function evalCompare(env: EvalEnv, flags: EvalFlags, pathA: string, pathB: string): Promise<number>;
+export declare function evalVariants(env: EvalEnv, flags: EvalFlags, suiteName: string): Promise<number>;
+/**
+ * Writes ONE shareable JSON bundle `{schemaVersion, report, runs, markdown}` of the stored run
+ * `<stateDir>/eval/<runId>/`, scrubbed of absolute paths and secrets. A secret hit refuses it
+ * (exit 1). The file is local: nothing is uploaded.
+ */
+export declare function evalExport(env: EvalEnv, flags: EvalFlags, runId: string): Promise<number>;
+type CliAppOptions = import("@alisio/core").AppOptions;
+/**
+ * The real environment of `alisio eval`: the provider is resolved from the same sources as
+ * `alisio run` (flags, saved profile, trusted config) through a throwaway Application that only
+ * lends its provider and configuration. It does not carry the invoker's `--read-only`, which
+ * would stop the provider plugins from loading; the permissions the runs get come from
+ * `InvokerPermissions` instead. Its in-memory database never touches the user's sessions.
+ */
+export declare function realEvalEnv(appOptions: (flags: EvalFlags) => Promise<CliAppOptions>, defaults: () => Promise<{
+    builtins: BuiltinPlugin[];
+}>): Promise<EvalEnv>;
+export {};