@foldrun/core 0.2.0

package/dist/src/evals.d.ts

@@ -0,0 +1,150 @@
+import { type Effort } from "./store.ts";
+export type Assertion = {
+    type: "contains";
+    value: string;
+} | {
+    type: "not-contains";
+    value: string;
+} | {
+    type: "matches";
+    value: string;
+} | {
+    type: "file";
+    value: string;
+} | {
+    type: "run";
+    value: string;
+} | {
+    type: "judge";
+    value: string;
+};
+export interface EvalCase {
+    name: string;
+    task: string;
+    expect: Assertion[];
+}
+export interface EvalInfo {
+    name: string;
+    file: string;
+    /** The agent under test, or null when a flow is. */
+    agent: string | null;
+    flow: string | null;
+    /** Model for the judge. Cases run the agent on its own model. */
+    model: string;
+    /** Effort for the judge. PASS/FAIL against one sentence is not deep work,
+     *  so this is one of the few places a low default is the right one. */
+    effort: Effort | null;
+    /** When it runs. `deploy` (default): after every push, with the rest.
+     *  `manual`: only when a person presses Run or the CLI asks — for a flow
+     *  eval that costs a whole run and touches real systems every time. */
+    trigger: "deploy" | "manual";
+    cases: EvalCase[];
+}
+export declare function parseEval(file: string, raw: string): EvalInfo;
+export declare function listEvals(tenant: string, workspace: string): EvalInfo[];
+/**
+ * Turn a finished run into a regression case.
+ *
+ * A run that worked is the best test of the agent that did it: the task is
+ * on record, and so is the answer. This writes both into `evals/` — the task
+ * verbatim, the answer as a rubric the judge holds the next run to — so a
+ * later edit to the markdown that changes the conclusion shows up as a red
+ * case rather than as a customer noticing. Appends to the target's
+ * regressions file when there is one, so a flow collects its cases in one
+ * place instead of one file per press.
+ */
+export declare function promoteRunToEval(tenant: string, workspace: string, runId: string, opts?: {
+    evalName?: string;
+    caseName?: string;
+    expect?: string[];
+    by?: string;
+}): {
+    file: string;
+    created: boolean;
+    caseName: string;
+};
+export interface AssertionResult {
+    assertion: Assertion;
+    passed: boolean;
+    detail: string;
+}
+export interface CaseResult {
+    name: string;
+    task: string;
+    runId: string | null;
+    passed: boolean;
+    /** Set when the agent itself failed, before any assertion could run. */
+    error: string | null;
+    assertions: AssertionResult[];
+    costUsd: number;
+}
+export interface EvalResult {
+    eval: string;
+    startedAt: string;
+    finishedAt: string;
+    passed: number;
+    failed: number;
+    costUsd: number;
+    cases: CaseResult[];
+    /** Why the eval could not run at all — a missing secret, an agent that
+     *  failed to load. Set only when `cases` is empty for that reason, so a
+     *  0-of-n result says what went wrong instead of looking like n wrong
+     *  answers. */
+    error?: string;
+}
+export declare function runEval(tenant: string, workspace: string, info: EvalInfo): Promise<EvalResult>;
+/**
+ * The latest result, and one line of history.
+ *
+ * `<eval>.json` is the latest run, overwritten each time — what the evals
+ * page shows. `history.jsonl` is every run, one line each, stamped with the
+ * commit the workspace was running. That stamp is the whole feature: a score
+ * with no commit says quality changed; a score with one says WHICH change.
+ * Manual runs are stamped too — the commit on record is whatever deploy put
+ * there — so a button press and a post-deploy run land in the same series.
+ */
+export declare function writeEvalResult(tenant: string, workspace: string, result: EvalResult, commit?: string | null): void;
+export interface EvalHistoryEntry {
+    eval: string;
+    at: string;
+    /** The commit the workspace was running. Null on an install that has
+     *  never deployed from git — the series still exists, just unattributed. */
+    commit: string | null;
+    passed: number;
+    failed: number;
+    costUsd: number;
+}
+export declare function readEvalHistory(tenant: string, workspace: string): EvalHistoryEntry[];
+export interface EvalTrend {
+    eval: string;
+    /** Newest first. */
+    latest: EvalHistoryEntry;
+    /** The most recent run on a DIFFERENT commit — the comparison that names a
+     *  change. Null when every run so far was on the same commit. */
+    previous: EvalHistoryEntry | null;
+    /** Pass count now minus pass count on the previous commit. */
+    delta: number | null;
+}
+/**
+ * Per eval: where it stands, and against which commit that is a change.
+ *
+ * Compares to the last run on a different commit rather than the last run,
+ * because two runs on one commit differ only by the model's mood — and a
+ * "regression" that is really variance would teach people to ignore the
+ * real ones.
+ */
+export declare function evalTrends(tenant: string, workspace: string): EvalTrend[];
+/**
+ * Run every eval against what was just deployed, stamped with its commit.
+ *
+ * Fire-and-forget from the deploy: a deploy answers in milliseconds and an
+ * eval takes minutes and money, so the two are never in one request. Runs
+ * one at a time — evals are model calls and a deploy should not fan out
+ * into all of them at once. A second deploy while one is still evaluating
+ * waits its turn behind the lock rather than doubling the spend.
+ */
+export declare function evaluateDeployed(tenant: string, workspace: string, commit?: string | null): Promise<{
+    ran: number;
+    skipped: string | null;
+}>;
+export declare function readEvalResult(tenant: string, workspace: string, name: string): EvalResult | null;