@stringsync/splates 0.0.61 → 1.13.3

package/vendor/@stringsync/splates-server/evals/import.d.ts

@@ -0,0 +1,10 @@
+import type { ContractBatch, ContractDetail, ContractResult } from "./contract";
+/** A framework's results turned into a splates-evals/1 batch, before it is written. */
+export interface Imported {
+    batch: ContractBatch;
+    results: ContractResult[];
+    /** Each cell's detail file, by cell id, written as `<cell>.json`. */
+    details: Map<string, ContractDetail>;
+}
+/** Writes `imported` into directory `dir`, replacing a batch already there. */
+export declare function writeImported(dir: string, imported: Imported): Promise<void>;

package/vendor/@stringsync/splates-server/evals/index.d.ts

@@ -0,0 +1,19 @@
+export type { EvalAgents } from "./agents";
+export { EvalAgentsStore } from "./agents";
+export type { EvalAgentsView } from "./agents-routes";
+export { EvalAgentsRoutes } from "./agents-routes";
+export type { ContractBatch, ContractCell, ContractDetail, ContractResult, ContractTarget, } from "./contract";
+export { CONTRACT_FORMAT, contractRunId, RESULTS_FILE } from "./contract";
+export { evalsDonePrompt } from "./done";
+export type { Imported } from "./import";
+export { writeImported } from "./import";
+export type { InspectLog } from "./inspect";
+export { inspectBatch } from "./inspect";
+export type { EvalCommands, EvalFinished, EvalLaunch, EvalPlan, EvalScenario, } from "./launch";
+export { EvalLaunches } from "./launch";
+export type { ConversationDirs, EvalProjects } from "./launch-routes";
+export { EvalLaunchRoutes } from "./launch-routes";
+export { EvalRoutes, projectEvalCommands } from "./routes";
+export { EvalStore } from "./store";
+export { targetName } from "./target";
+export type { EvalDetail, EvalGrid } from "./types";

package/vendor/@stringsync/splates-server/evals/index.js

@@ -0,0 +1,40 @@
+import {
+  EvalAgentsStore2,
+  EvalAgentsRoutes2,
+  CONTRACT_FORMAT2,
+  RESULTS_FILE2,
+  contractRunId2,
+  targetName2,
+  evalsDonePrompt2,
+  writeImported2,
+  inspectBatch2,
+  EvalLaunches2,
+  EvalLaunchRoutes2,
+  EvalStore2,
+  EvalRoutes2,
+  projectEvalCommands2
+} from "../index-raqrwzdq.js";
+import"../index-acbcm66f.js";
+import"../index-caep3sde.js";
+import"../index-01z7ycdk.js";
+import"../index-e9z7s5hh.js";
+import"../index-kmzdvvt0.js";
+import"../index-t9w5a8ed.js";
+import"../index-0jttnv86.js";
+import"../index-g18nxyn7.js";
+export {
+  CONTRACT_FORMAT2 as CONTRACT_FORMAT,
+  EvalAgentsRoutes2 as EvalAgentsRoutes,
+  EvalAgentsStore2 as EvalAgentsStore,
+  EvalLaunchRoutes2 as EvalLaunchRoutes,
+  EvalLaunches2 as EvalLaunches,
+  EvalRoutes2 as EvalRoutes,
+  EvalStore2 as EvalStore,
+  RESULTS_FILE2 as RESULTS_FILE,
+  contractRunId2 as contractRunId,
+  evalsDonePrompt2 as evalsDonePrompt,
+  inspectBatch2 as inspectBatch,
+  projectEvalCommands2 as projectEvalCommands,
+  targetName2 as targetName,
+  writeImported2 as writeImported
+};

package/vendor/@stringsync/splates-server/evals/inspect.d.ts

@@ -0,0 +1,66 @@
+import type { Imported } from "./import";
+/**
+ * An Inspect AI eval log, as `inspect eval` writes it with `--log-format json`
+ * or `inspect log dump` prints it: only the fields a batch is made from.
+ */
+export interface InspectLog {
+    status?: string;
+    eval: {
+        task: string;
+        model: string;
+        created?: string;
+    };
+    stats?: {
+        started_at?: string;
+    };
+    error?: {
+        message?: string;
+    } | null;
+    samples?: InspectSample[];
+}
+interface InspectSample {
+    id: string | number;
+    epoch?: number;
+    input: string | InspectMessage[];
+    target?: string | string[];
+    messages?: InspectMessage[];
+    output?: {
+        completion?: string;
+    };
+    scores?: Record<string, InspectScore> | null;
+    error?: {
+        message?: string;
+    } | null;
+    model_usage?: Record<string, {
+        total_tokens?: number;
+        total_cost?: number;
+    }>;
+    total_time?: number | null;
+}
+interface InspectScore {
+    value: unknown;
+    answer?: string | null;
+    explanation?: string | null;
+}
+interface InspectMessage {
+    role: string;
+    content: string | InspectContent[];
+    tool_calls?: {
+        id: string;
+        function: string;
+        arguments: unknown;
+    }[] | null;
+    tool_call_id?: string | null;
+}
+type InspectContent = {
+    type: string;
+    text?: string;
+    reasoning?: string;
+};
+/**
+ * `log` as a splates-evals/1 batch: a cell per sample and epoch, which passes
+ * when every scorer's value, as Inspect reads it as a number (`C` 1, `P` 0.5,
+ * `I` 0), reaches `passAt`.
+ */
+export declare function inspectBatch(log: InspectLog, passAt: number): Imported;
+export {};

package/vendor/@stringsync/splates-server/evals/launch-routes.d.ts

@@ -0,0 +1,39 @@
+import { type Handler } from "../server/shared";
+import type { EvalCommands, EvalLaunches } from "./launch";
+/** Where a conversation works, for a run its card starts to use the code there. */
+export interface ConversationDirs {
+    /** The directory conversation `id` works in; null when it is not known. */
+    cwdOf(id: string): Promise<string | null>;
+}
+/** Which eval commands a request's project names. */
+export interface EvalProjects {
+    /** The eval commands of the project `request` names; null when it names no `evals`. */
+    commandsOf(request: Request): Promise<EvalCommands | null>;
+}
+/**
+ * How the Evals card lists, plans, starts, follows and stops a paid run,
+ * through the commands the project of the request names.
+ */
+export declare class EvalLaunchRoutes {
+    private readonly launches;
+    private readonly projects;
+    private readonly dirs;
+    constructor(launches: EvalLaunches, projects: EvalProjects, dirs?: ConversationDirs);
+    /** Every case the project lists, for the card to group; `?conversation=` lists its worktree's. */
+    readonly scenarios: Handler;
+    /** The free plan for the body's cases and agents: what the run would reserve. */
+    readonly plan: Handler;
+    /** Starts the run a card's Run button asks for, the person's alone to press, and answers with its launch. */
+    readonly start: Handler;
+    /** `?key=` names the card; null when it has no run here. */
+    readonly read: Handler;
+    /** Stops the run `?key=` names. */
+    readonly stop: Handler;
+    private commands;
+    private runnable;
+    /** Where conversation `id` works, when it still exists; null for none. */
+    private cwdOf;
+    private conversation;
+    private key;
+    private spec;
+}

package/vendor/@stringsync/splates-server/evals/launch.d.ts

@@ -0,0 +1,123 @@
+import type { AsyncResource } from "../../../core/disposable/index.js";
+import { type Eventful } from "../../../core/events/index.js";
+import type { Ps } from "../../../core/system/index.js";
+import type { AgentPick } from "../models";
+import type { Storage } from "../storage";
+/** `claude:MODEL@EFFORT` for a pick, as `--agent` takes it. */
+export declare function targetOf(pick: AgentPick): string;
+/** A project's eval commands, from the `evals` block of its `splates.config.ts`. */
+export interface EvalCommands {
+    /** The project's root, where the commands run when a run names no worktree. */
+    root: string;
+    /** Where batches are written, absolute, as `SPLATES_EVALS_RESULTS` tells `run`. */
+    results: string;
+    /** Prints the cases, one JSON object a line; null when the project names none. */
+    list: string | null;
+    /** Runs the cases and targets in the env into a new batch; null when the project names none. */
+    run: string | null;
+    /** Prints what a run would come to and cost at most; null when the project names none. */
+    plan: string | null;
+}
+/** What the person's Run button asks for: the evals of one card, on the agents it ended with. */
+export interface EvalLaunchSpec {
+    /** Names the card, so a reload finds its run again. */
+    key: string;
+    /** The card's label, the skill or suite it names. */
+    skill: string;
+    scenarios: string[];
+    agents: AgentPick[];
+    /** The revision the change is judged against; the runner's own default when null. */
+    baseline: string | null;
+    /** Runs of each cell; the runner's own default when absent. */
+    reps?: number;
+    /** The conversation whose card it is, told once it is done; null for none. */
+    conversation?: string | null;
+    /** Where the commands run, the conversation's task worktree; the project's root when null. */
+    cwd?: string | null;
+}
+/** How a launch stands. `refused` is the runner declining before it wrote a batch. */
+export type EvalLaunchStatus = "running" | "done" | "refused" | "stopped";
+/** A launch as the card reads it. */
+export interface EvalLaunch {
+    id: string;
+    key: string;
+    status: EvalLaunchStatus;
+    /** The batch directory it writes, once the runner has made it. */
+    batch: string | null;
+    /** The runner's last words: a refusal's reason. */
+    output: string;
+}
+/** What a project's `plan` says a run comes to; without one, nothing is known of its cost. */
+export interface EvalPlan {
+    /** Case-by-target runs the plan comes to; null when the project prints no plan. */
+    cells: number | null;
+    /** The most they can cost, in dollars; null when there is no ceiling to show. */
+    usd: number | null;
+    /** Whether the plan turned the run down, its reason in `output`. */
+    refused: boolean;
+    /** The planner's output, for a refusal or anything the numbers leave out. */
+    output: string;
+}
+/** A case a project's `list` prints. */
+export interface EvalScenario {
+    id: string;
+    /** What the card heads the case with, as `group` in `batch.json`. */
+    group: string | null;
+    /** What the case checks, as `about` in `batch.json`. */
+    about: string | null;
+}
+/**
+ * Starts, follows and stops the paid runs the Evals card asks for, through
+ * the `list`, `plan` and `run` commands a project names. Listen to `running`
+ * to show a conversation's card run as going, and to `finished` to tell the
+ * conversation its results. A restart runs again each launch left in progress.
+ */
+export declare class EvalLaunches implements Eventful<EvalLaunchesEvents>, AsyncResource {
+    private readonly ps;
+    private readonly dispatcher;
+    readonly events: import("../../../core/events/index.js").Events<EvalLaunchesEvents>;
+    /** By project root and card key. */
+    private readonly launches;
+    private readonly watching;
+    private readonly record;
+    private writing;
+    private disposing;
+    constructor(ps: Ps, storage: Storage);
+    /** Every case the project's `list` prints. */
+    scenarios(commands: EvalCommands, cwd?: string | null): Promise<EvalScenario[]>;
+    /** Runs the project's free `plan` for `spec`; without one, nothing is refused and no ceiling shown. */
+    plan(commands: EvalCommands, spec: Omit<EvalLaunchSpec, "key">): Promise<EvalPlan>;
+    /** Starts `spec`, unless its card already has a run going. */
+    start(commands: EvalCommands, spec: EvalLaunchSpec): Promise<EvalLaunch>;
+    /** Runs again each launch a stopped server left in progress. */
+    resume(commands: EvalCommands[]): Promise<void>;
+    /** The run of the card `key` names, or null when it has none. */
+    get(commands: EvalCommands, key: string): Promise<EvalLaunch | null>;
+    /** Stops the run of the card `key` names; its landed cells stay on the Evals page. */
+    stop(commands: EvalCommands, key: string): Promise<EvalLaunch | null>;
+    /** Stops runners and waits for their writes, leaving interrupted runs in progress to resume. */
+    disposeAsync(): Promise<void>;
+    /** Collects the child's words and settles the launch when it exits. */
+    private watch;
+    /** Says whether any run from `launch`'s conversation is still going. */
+    private running;
+    /** The launch, its batch found if the runner has made one since it started. */
+    private read;
+    private save;
+}
+/** A run that finished with its batch written, for the conversation whose card started it. */
+export interface EvalFinished {
+    launch: EvalLaunch;
+    /** The directory the batch is in. */
+    results: string;
+    conversation: string;
+}
+/** Whether a conversation has a run its card started going, as one starts or ends. */
+export interface EvalRunning {
+    conversation: string;
+    running: boolean;
+}
+export type EvalLaunchesEvents = {
+    finished: EvalFinished;
+    running: EvalRunning;
+};

package/vendor/@stringsync/splates-server/evals/routes.d.ts

@@ -0,0 +1,39 @@
+import type { Ps } from "../../../core/system/index.js";
+import type { EvalsConfig } from "../config";
+import type { Handler } from "../server/shared";
+import type { EvalCommands } from "./launch";
+/**
+ * The saved eval batches the Evals page and the report card read, from the
+ * directory a project's `evals.results` names, which any framework writes in
+ * the public format. For a project that names none, the list answers null and
+ * the other routes 404.
+ */
+export declare class EvalRoutes {
+    private readonly opts;
+    private readonly stores;
+    constructor(opts: EvalOptions);
+    /** The batches, or null when the project names no results. */
+    readonly list: Handler;
+    /** `?id=` names the batch, as `EvalBatch.id` does. */
+    readonly batch: Handler;
+    /** `?id=` names the run, as `EvalRun.id` does. */
+    readonly run: Handler;
+    /**
+     * What tells the agent whose card started batch `id`, in `directory`, that
+     * it is done, with its report condensed; null when the batch is not there.
+     */
+    donePrompt(directory: string, id: string): Promise<string | null>;
+    private opened;
+    private storeOf;
+}
+/**
+ * A project's eval commands from its `evals` block, `results` resolved
+ * against `root` unless absolute or under `~`; null for a project that
+ * names none.
+ */
+export declare function projectEvalCommands(ps: Ps, root: string, evals: EvalsConfig | null): EvalCommands | null;
+interface EvalOptions {
+    /** The directory the project `request` names reads its batches from, or null for none. */
+    directory: (request: Request) => Promise<string | null>;
+}
+export {};

package/vendor/@stringsync/splates-server/evals/store.d.ts

@@ -0,0 +1,17 @@
+import type { EvalBatch, EvalDetail, EvalGrid } from "./types";
+/**
+ * The eval batches under `directory`: each directory's `batch.json`, in the
+ * public `splates-evals/1` format any framework writes (`contract.ts`),
+ * `splates dev eval run` among them.
+ */
+export declare class EvalStore {
+    private readonly directory;
+    constructor(directory: string);
+    /** Every batch, newest first. */
+    batches(): Promise<EvalBatch[]>;
+    /** The batch `id` names with its every cell; null when there is none. */
+    grid(id: string): Promise<EvalGrid | null>;
+    /** The run `id` names, `<batch>#<cell>`, opened whole. */
+    run(id: string): Promise<EvalDetail>;
+    private ids;
+}

package/vendor/@stringsync/splates-server/evals/target.d.ts

@@ -0,0 +1,3 @@
+import type { EvalTarget } from "./types";
+/** `claude sonnet high`, then `@<revision>` for a baseline skill, as the page names a target. */
+export declare function targetName(target: EvalTarget): string;

package/vendor/@stringsync/splates-server/evals/types.d.ts

@@ -0,0 +1,105 @@
+/** A run's outcome, as the harness graded it, or `pending` for a cell yet to land and `unreadable` for a result that cannot be read. */
+export type EvalStatus = "pass" | "fail" | "error" | "pending" | "unreadable";
+/** An agent a batch ran on, with the skill it ran: its model and thinking level, and the skill's revision when it was not the checkout's. */
+export interface EvalTarget {
+    /** `<agent>/<model>@<effort>`, then ` @<revision>` for a baseline skill, what a cell names it by. */
+    key: string;
+    agent: string;
+    model: string | null;
+    /** The thinking level it ran at; null for the agent's default. */
+    effort: string | null;
+    /** The skill revision it ran, for a baseline, or a framework's `variant`; null for the checkout's skill or no variant. */
+    skill: string | null;
+}
+/** One run of a project's evals: a `splates dev eval run`, or any framework's batch. */
+export interface EvalBatch {
+    /** Its directory under the results directory. */
+    id: string;
+    /** The skill it ran, or a framework batch's title, when it says. */
+    skill: string | null;
+    /** Who wrote it: `splates` for `splates dev eval run`, else its framework when it says. */
+    producer: string | null;
+    /** When it started, as ISO 8601. */
+    startedAt: string;
+    targets: EvalTarget[];
+    /** How many cells ended each way, `pending` among them. */
+    tally: Partial<Record<EvalStatus, number>>;
+    cells: number;
+    /** Summed over its runs; null when none could say. */
+    tokens: number | null;
+    costUsd: number | null;
+}
+/** One saved run, as a cell shows it. */
+export interface EvalRun {
+    /** `<batch>#<cell>`: its batch's directory under the results directory, and its cell. */
+    id: string;
+    status: EvalStatus;
+    tokens: number | null;
+    costUsd: number | null;
+    milliseconds: number;
+    /** What broke, for `error`, or why it is `unreadable`. */
+    error?: string;
+    /** From 0 to 1, when its framework grades on a scale. */
+    score?: number;
+    /** Why it ended as it did, in its framework's words. */
+    reason?: string;
+    /** Its framework's own view of it. */
+    url?: string;
+}
+/** One cell of a batch's grid: a scenario (`scenario`, as run cards name it) on a target at a repetition. */
+export interface EvalCell {
+    scenario: string;
+    aspect: string;
+    requirement: string;
+    target: string;
+    repetition: number;
+    /** Null while it has yet to land. */
+    run: EvalRun | null;
+}
+/** A batch with every cell, planned or landed: what the report card draws. */
+export interface EvalGrid extends EvalBatch {
+    grid: EvalCell[];
+}
+/** One code check of a run and how it went. */
+export interface EvalCheck {
+    id: string;
+    /** What it checks, in words. */
+    about: string;
+    pass: boolean;
+    /** Why it could not be told, when the check itself broke. */
+    error?: string;
+}
+/** What the judge decided of a run, and why. */
+export interface EvalVerdict {
+    pass: boolean;
+    reason: string;
+}
+/** One step of what the agent did: something it said, or a tool it called and what came back. */
+export type EvalStep = {
+    kind: "text";
+    text: string;
+} | {
+    kind: "tool";
+    name: string;
+    input: string;
+    output: string | null;
+};
+/** A run opened: what it asked, each check, the judge's verdict and what the agent did. */
+export interface EvalDetail extends EvalRun {
+    /** Who wrote its batch, as `EvalBatch.producer`. */
+    producer: string | null;
+    scenario: string;
+    aspect: string;
+    requirement: string;
+    target: EvalTarget;
+    repetition: number;
+    /** The person's message to the agent. */
+    input: string;
+    /** What the judge was asked to decide; null when the scenario has no judge. */
+    rubric: string | null;
+    checks: EvalCheck[];
+    judge: EvalVerdict | null;
+    /** The agent's reply, as written; empty when it gave none. */
+    reply: string;
+    transcript: EvalStep[];
+}

package/vendor/@stringsync/splates-server/exit.d.ts

@@ -0,0 +1,55 @@
+import type { Deps, Middleware } from "../../core/cmd/index.js";
+/** Exit codes are the API: an agent branches on these, not on prose. */
+export declare const EXIT: {
+    readonly usage: 1;
+    readonly conflict: 2;
+    readonly tests_failed: 3;
+    readonly lease_lost: 4;
+    readonly budget_exhausted: 5;
+    readonly lease_held: 6;
+    readonly dirty: 7;
+    readonly not_found: 8;
+    readonly hook_failed: 9;
+    readonly exists: 10;
+    readonly orphaned: 11;
+    readonly merge_failed: 12;
+    readonly already_removed: 13;
+    readonly timeout: 14;
+    readonly blocked: 16;
+    readonly cycle: 18;
+    readonly outdated: 19;
+    readonly uninitialized: 20;
+    readonly too_large: 21;
+    readonly dismissed: 22;
+    readonly no_secret_store: 23;
+    readonly cloudflare: 24;
+    readonly requested: 25;
+    readonly slots_full: 26;
+    readonly slot_running: 27;
+    readonly not_granted: 28;
+    readonly migration_failed: 30;
+    readonly no_runner: 31;
+};
+export type Reason = keyof typeof EXIT;
+/**
+ * A refusal, carrying the reason to branch on, a message to show, and any
+ * data worth reporting with it. `exits` turns one into output and a status
+ * code.
+ */
+export declare class Exit extends Error {
+    readonly reason: Reason;
+    readonly data: Record<string, unknown>;
+    constructor(reason: Reason, message: string, data?: Record<string, unknown>);
+}
+/**
+ * How a command refuses. Every call site means "and go no further", which is
+ * what the `never` return buys: the code below a `fail` is unreachable, and
+ * TypeScript narrows accordingly.
+ */
+export declare function fail(reason: Reason, message: string, data?: Record<string, unknown>): never;
+/**
+ * Decides what a refusal looks like (a machine-readable reason on stdout, a
+ * human explanation on stderr) and is the only place that ends the process.
+ * An agent branches on the first and reads the second.
+ */
+export declare function exits<C extends Deps>(): Middleware<C>;

package/vendor/@stringsync/splates-server/exit.js

@@ -0,0 +1,12 @@
+import {
+  EXIT2,
+  Exit2,
+  fail2,
+  exits2
+} from "./index-t9w5a8ed.js";
+export {
+  EXIT2 as EXIT,
+  Exit2 as Exit,
+  exits2 as exits,
+  fail2 as fail
+};

package/vendor/@stringsync/splates-server/git-identity.d.ts

@@ -0,0 +1,17 @@
+import type { GitRunner } from "./git";
+/** A git author's name and email. */
+export interface GitIdentity {
+    name: string;
+    email: string;
+}
+/** What names the author commits are signed as where the machine has none. */
+export interface Author {
+    author(): Promise<GitIdentity | null>;
+}
+/** The author `runner`'s git is configured with in `cwd`; null unless both name and email are set. */
+export declare function authorOf(runner: GitRunner, cwd: string): Promise<GitIdentity | null>;
+/**
+ * `-c` options naming `given`, else `splates`, for whichever of the name and
+ * email the machine leaves unset: a configured one is never overridden.
+ */
+export declare function fallbackIdentity(runner: GitRunner, cwd: string, given: Author | undefined): Promise<string[]>;

package/vendor/@stringsync/splates-server/git-process.d.ts

@@ -0,0 +1,8 @@
+import type { Ps } from "../../core/system/index.js";
+import type { GitRunner } from "./git";
+/** The git command to run, resolved once and cached. */
+export declare function gitBinary(): string;
+/** `GitRunner` as a process through `ps`. */
+export declare function processGit(ps: Pick<Ps, "start">): GitRunner;
+/** Whether git reaches `url` over SSH, as a deploy key signs in. */
+export declare function sshUrl(url: string | null): boolean;