@spendgraph/harness 0.1.0 → 0.2.0

package/dist/parallel/types.d.ts

@@ -0,0 +1,61 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+/** One branch of the fan-out. */
+export interface Task {
+    /** Unique in the run, and the name its step is recorded under. */
+    name: string;
+    run(values: Record<string, unknown>, emit: Emit): Promise<unknown> | unknown;
+}
+/** What one task produced, or why it did not. */
+export interface TaskResult {
+    name: string;
+    status: "completed" | "failed";
+    value?: unknown;
+    error?: string;
+    inputTokens: number;
+    outputTokens: number;
+}
+export interface ParallelOptions {
+    tasks: Task[];
+    /**
+     * Combines what came back into the answer.
+     *
+     * Only the tasks that succeeded are passed. Sectioning usually joins them;
+     * voting usually counts them — `majority` is exported for that.
+     */
+    merge(results: TaskResult[], values: Record<string, unknown>): unknown;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /** Most tasks in flight at once. Default 4. */
+    concurrency?: number;
+    /**
+     * How many must succeed before merging is worth doing. Defaults to all.
+     *
+     * Sectioning wants all of them: a missing section is a hole in the answer.
+     * Voting wants a majority, and is right to ignore the one call that timed out.
+     */
+    minSuccess?: number;
+    /** Called as each task lands, so a slow fan-out is not silent. */
+    onTask?: (result: TaskResult) => void;
+}
+export interface ParallelResult {
+    status: "completed" | "failed";
+    output: string;
+    error?: string;
+    steps: RolloutStep[];
+    outputs: Record<string, unknown>;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    /** Every task, in the order they were declared — never the order they landed. */
+    results: TaskResult[];
+    succeeded: string[];
+    failed: string[];
+    /** What `merge` returned. Undefined when too few tasks succeeded. */
+    merged?: unknown;
+}

package/dist/parallel/types.js

@@ -0,0 +1 @@
+export {};

package/dist/refine/attempt.d.ts

@@ -0,0 +1,22 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Attempt } from "./types.js";
+/**
+ * What one attempt consumed.
+ *
+ * Both halves, because a refine loop's cost is dominated by the input it resends
+ * every round — the draft, the critique and the instruction to try again.
+ */
+export declare function tokensOf(attempt: Attempt): number;
+/**
+ * One attempt as a step.
+ *
+ * Recorded whether it was accepted or not: the rejected drafts are the evidence
+ * for whether refining paid at all, and a loop that keeps only the winner cannot
+ * say what the answer cost.
+ */
+export declare function attemptStep(index: number, round: number, attempt: Attempt): RolloutStep;
+/** Both halves of every attempt so far, for the result's totals. */
+export declare function totalTokens(steps: RolloutStep[]): {
+    inputTokens: number;
+    outputTokens: number;
+};

package/dist/refine/attempt.js

@@ -0,0 +1,38 @@
+/**
+ * What one attempt consumed.
+ *
+ * Both halves, because a refine loop's cost is dominated by the input it resends
+ * every round — the draft, the critique and the instruction to try again.
+ */
+export function tokensOf(attempt) {
+    return (attempt.inputTokens ?? 0) + (attempt.outputTokens ?? 0);
+}
+/**
+ * One attempt as a step.
+ *
+ * Recorded whether it was accepted or not: the rejected drafts are the evidence
+ * for whether refining paid at all, and a loop that keeps only the winner cannot
+ * say what the answer cost.
+ */
+export function attemptStep(index, round, attempt) {
+    return {
+        index,
+        source: `attempt_${round}`,
+        model: attempt.model,
+        output: attempt.output,
+        status: attempt.status ?? "completed",
+        error: attempt.error,
+        inputTokens: attempt.inputTokens,
+        outputTokens: attempt.outputTokens,
+    };
+}
+/** Both halves of every attempt so far, for the result's totals. */
+export function totalTokens(steps) {
+    let inputTokens = 0;
+    let outputTokens = 0;
+    for (const step of steps) {
+        inputTokens += step.inputTokens ?? 0;
+        outputTokens += step.outputTokens ?? 0;
+    }
+    return { inputTokens, outputTokens };
+}

package/dist/refine/index.d.ts

@@ -0,0 +1,4 @@
+export { attemptStep, tokensOf } from "./attempt.js";
+export { best, judgeStep } from "./judge.js";
+export { refine } from "./refine.js";
+export type { Attempt, RefineOptions, RefineResult, RoundRecord, Verdict } from "./types.js";

package/dist/refine/index.js

@@ -0,0 +1,3 @@
+export { attemptStep, tokensOf } from "./attempt.js";
+export { best, judgeStep } from "./judge.js";
+export { refine } from "./refine.js";

package/dist/refine/judge.d.ts

@@ -0,0 +1,15 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { RoundRecord, Verdict } from "./types.js";
+/** One verdict as a step, so a run says why each draft was turned down. */
+export declare function judgeStep(index: number, round: number, verdict: Verdict): RolloutStep;
+/**
+ * The attempt to hand back when nothing was accepted.
+ *
+ * Highest score wins, and the last attempt wins when nothing was scored — a loop
+ * that improves each round has its best work at the end, and returning the first
+ * draft after three revisions would throw the work away.
+ *
+ * Here rather than in the loop because the choice is the judge's: it is made on
+ * the verdicts, and a judge that never scores gets the fallback.
+ */
+export declare function best(history: RoundRecord[]): RoundRecord | undefined;

package/dist/refine/judge.js

@@ -0,0 +1,25 @@
+/** One verdict as a step, so a run says why each draft was turned down. */
+export function judgeStep(index, round, verdict) {
+    return {
+        index,
+        source: `judge_${round}`,
+        output: JSON.stringify(verdict),
+        status: "completed",
+    };
+}
+/**
+ * The attempt to hand back when nothing was accepted.
+ *
+ * Highest score wins, and the last attempt wins when nothing was scored — a loop
+ * that improves each round has its best work at the end, and returning the first
+ * draft after three revisions would throw the work away.
+ *
+ * Here rather than in the loop because the choice is the judge's: it is made on
+ * the verdicts, and a judge that never scores gets the fallback.
+ */
+export function best(history) {
+    const scored = history.filter((r) => typeof r.verdict.score === "number");
+    if (scored.length === 0)
+        return history.at(-1);
+    return scored.reduce((a, b) => ((b.verdict.score ?? 0) > (a.verdict.score ?? 0) ? b : a));
+}

package/dist/refine/refine.d.ts

@@ -0,0 +1,10 @@
+import type { RefineOptions, RefineResult } from "./types.js";
+/**
+ * Draft, judge, revise, until it is good enough or the budget is out.
+ *
+ * The pattern worth naming when there is a clear criterion and a first draft
+ * rarely meets it. Every attempt and every judgement is recorded, the rejected
+ * ones included: a loop that keeps only the winner cannot say what the answer
+ * cost, and the rejected drafts are the evidence for whether refining paid.
+ */
+export declare function refine(opts: RefineOptions): Promise<RefineResult>;

package/dist/refine/refine.js

@@ -0,0 +1,67 @@
+import { attemptStep, tokensOf, totalTokens } from "./attempt.js";
+import { best, judgeStep } from "./judge.js";
+const DEFAULT_ROUNDS = 3;
+/**
+ * Draft, judge, revise, until it is good enough or the budget is out.
+ *
+ * The pattern worth naming when there is a clear criterion and a first draft
+ * rarely meets it. Every attempt and every judgement is recorded, the rejected
+ * ones included: a loop that keeps only the winner cannot say what the answer
+ * cost, and the rejected drafts are the evidence for whether refining paid.
+ */
+export async function refine(opts) {
+    const startedAt = Date.now();
+    const rounds = Math.max(1, opts.rounds ?? DEFAULT_ROUNDS);
+    const steps = [];
+    const history = [];
+    let feedback = null;
+    let spent = 0;
+    let stoppedBy = "rounds";
+    for (let round = 0; round < rounds; round++) {
+        if (opts.maxTokens !== undefined && spent >= opts.maxTokens) {
+            stoppedBy = "tokens";
+            break;
+        }
+        const attempt = await opts.attempt(feedback, round, (text, stage) => opts.emit?.(text, stage ?? `round ${round}`));
+        steps.push(attemptStep(steps.length, round, attempt));
+        spent += tokensOf(attempt);
+        if (attempt.status === "failed") {
+            return {
+                status: "failed",
+                output: "",
+                error: attempt.error ?? "The attempt failed.",
+                steps,
+                outputs: { history },
+                latencyMs: Date.now() - startedAt,
+                ...totalTokens(steps),
+                accepted: false,
+                rounds: round + 1,
+                stoppedBy: "failed",
+                history,
+            };
+        }
+        const verdict = await opts.judge(attempt, round);
+        steps.push(judgeStep(steps.length, round, verdict));
+        const record = { round, attempt, verdict, tokens: tokensOf(attempt) };
+        history.push(record);
+        opts.onRound?.(record);
+        if (verdict.accepted) {
+            stoppedBy = "accepted";
+            break;
+        }
+        feedback = verdict.feedback ?? null;
+    }
+    const winner = best(history);
+    return {
+        status: "completed",
+        output: winner?.attempt.output ?? "",
+        steps,
+        outputs: { history },
+        latencyMs: Date.now() - startedAt,
+        ...totalTokens(steps),
+        accepted: stoppedBy === "accepted",
+        rounds: history.length,
+        stoppedBy,
+        history,
+    };
+}

package/dist/refine/types.d.ts

@@ -0,0 +1,90 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+/** What one attempt produced. Structurally the `TraceOutcome` a prompt accepts. */
+export interface Attempt {
+    output: string;
+    model: string;
+    status?: "completed" | "failed";
+    error?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}
+/** What the judge decided about an attempt. */
+export interface Verdict {
+    /** True stops the loop. */
+    accepted: boolean;
+    /**
+     * What to fix, handed to the next attempt.
+     *
+     * A rejection with no feedback asks the model to try again with nothing new,
+     * which is how a refine loop spends three rounds producing the same draft.
+     */
+    feedback?: string;
+    /** Higher is better. Used to pick a winner when nothing was accepted. */
+    score?: number;
+}
+export interface RefineOptions {
+    /**
+     * A draft. `feedback` is null on the first round and the judge's note after.
+     *
+     * One callback rather than separate generate and revise, because in practice
+     * they are the same prompt with one extra paragraph.
+     */
+    attempt(feedback: string | null, round: number, emit: Emit): Promise<Attempt> | Attempt;
+    /** Whether that will do, and what is wrong with it if not. */
+    judge(attempt: Attempt, round: number): Promise<Verdict> | Verdict;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /** Most attempts to make. Default 3. */
+    rounds?: number;
+    /**
+     * Stop once the attempts have consumed this many tokens.
+     *
+     * Tokens rather than money because this package cannot price a call — that
+     * happens on the server, after the fact. Checked before each round, so the
+     * ceiling is never crossed rather than merely noticed.
+     */
+    maxTokens?: number;
+    /** Called after each verdict, so a long loop is not silent. */
+    onRound?: (round: RoundRecord) => void;
+}
+export interface RoundRecord {
+    round: number;
+    attempt: Attempt;
+    verdict: Verdict;
+    tokens: number;
+}
+/**
+ * What the loop produced, in the shape of a rollout.
+ *
+ * `steps` holds every attempt and every judgement, including the rejected ones.
+ * A loop that recorded only the winner could not say what the answer cost, which
+ * is the whole point of recording it here.
+ */
+export interface RefineResult {
+    status: "completed" | "failed";
+    output: string;
+    error?: string;
+    steps: RolloutStep[];
+    outputs: Record<string, unknown>;
+    latencyMs: number;
+    /**
+     * Summed across **every** attempt, the rejected ones included.
+     *
+     * The only question a refine loop has to answer is whether refining paid, and
+     * a total that counted the winner alone would always say yes.
+     */
+    inputTokens: number;
+    outputTokens: number;
+    /** False when the rounds or the token ceiling ran out first. */
+    accepted: boolean;
+    rounds: number;
+    /** Why it ended: a verdict, the round limit, or the token ceiling. */
+    stoppedBy: "accepted" | "rounds" | "tokens" | "failed";
+    history: RoundRecord[];
+}

package/dist/refine/types.js

@@ -0,0 +1 @@
+export {};

package/dist/route/classify.d.ts

@@ -0,0 +1,52 @@
+import type { Classification, RouteOptions } from "./types.js";
+/** The classifier's node name, and the key its decision is stored under. */
+export declare const CLASSIFY = "classify";
+/**
+ * The branch that runs when no other one does.
+ *
+ * Declared here rather than beside the dispatch it names, because `decide` is
+ * what produces it — and the other way round leaves the two files importing
+ * each other, which is the thing a reader has to stop and untangle.
+ */
+export declare const FALLBACK = "fallback";
+/** What the classifier said, once a bare string has been expanded. */
+export type Said = Exclude<Classification, string>;
+/**
+ * What the classifier said, and what will run because of it.
+ *
+ * The two are kept apart on purpose: a router that recorded only the branch it
+ * ran could not tell "the classifier said `legal` and we have no such branch"
+ * from "it was unsure about `billing`". The first is a missing branch, the
+ * second a miscalibrated one, and they want opposite fixes.
+ */
+export interface Decision {
+    said: string;
+    taken: string;
+    confidence?: number;
+    reason?: string;
+}
+/** A bare string is shorthand for `{ route }`. */
+export declare function normalise(value: Classification): Said;
+/**
+ * Which branch runs.
+ *
+ * A class no branch handles falls back, and so does one the classifier is not
+ * sure enough about — a hedged guess sent to the wrong specialist is worse than
+ * the general path.
+ */
+export declare function decide(said: Said, names: string[], minConfidence?: number): Decision;
+/** A step's worth of outcome, with the decision the edges will read. */
+export interface ClassifyOutput extends Decision {
+    output: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}
+/**
+ * The entry node.
+ *
+ * Its return is what the edges read, so the decision travels through the graph's
+ * own context rather than a variable closed over from outside — which means a
+ * finished run can be read apart from `outputs.classify` alone.
+ */
+export declare function classifier(opts: RouteOptions, names: string[]): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>;

package/dist/route/classify.js

@@ -0,0 +1,57 @@
+import { node } from "@spendgraph/graph";
+/** The classifier's node name, and the key its decision is stored under. */
+export const CLASSIFY = "classify";
+/**
+ * The branch that runs when no other one does.
+ *
+ * Declared here rather than beside the dispatch it names, because `decide` is
+ * what produces it — and the other way round leaves the two files importing
+ * each other, which is the thing a reader has to stop and untangle.
+ */
+export const FALLBACK = "fallback";
+/** A bare string is shorthand for `{ route }`. */
+export function normalise(value) {
+    return typeof value === "string" ? { route: value } : value;
+}
+/**
+ * Which branch runs.
+ *
+ * A class no branch handles falls back, and so does one the classifier is not
+ * sure enough about — a hedged guess sent to the wrong specialist is worse than
+ * the general path.
+ */
+export function decide(said, names, minConfidence) {
+    const known = names.includes(said.route);
+    const confident = minConfidence === undefined ||
+        said.confidence === undefined ||
+        said.confidence >= minConfidence;
+    return {
+        said: said.route,
+        taken: known && confident ? said.route : FALLBACK,
+        confidence: said.confidence,
+        reason: said.reason,
+    };
+}
+/**
+ * The entry node.
+ *
+ * Its return is what the edges read, so the decision travels through the graph's
+ * own context rather than a variable closed over from outside — which means a
+ * finished run can be read apart from `outputs.classify` alone.
+ */
+export function classifier(opts, names) {
+    return node({
+        name: CLASSIFY,
+        run: async (input, ctx) => {
+            const said = normalise(await opts.classify(input, ctx));
+            return {
+                ...decide(said, names, opts.minConfidence),
+                // The step records what the classifier said, not what we did about it.
+                output: said.route,
+                model: said.model,
+                inputTokens: said.inputTokens,
+                outputTokens: said.outputTokens,
+            };
+        },
+    });
+}

package/dist/route/dispatch.d.ts

@@ -0,0 +1,12 @@
+import { type Edge } from "@spendgraph/graph";
+import type { RouteOptions } from "./types.js";
+/** One node per branch, plus the fallback. */
+export declare function branches(opts: RouteOptions, names: string[]): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>[];
+/**
+ * Classify, then exactly one branch, then stop.
+ *
+ * Conditional edges first and the fallback unconditional last, because `graph`
+ * refuses the other order — an unconditional edge ahead of a conditional one is
+ * a router whose default hides a branch, and it would never be reached.
+ */
+export declare function edgesTo(names: string[]): Edge[];

package/dist/route/dispatch.js

@@ -0,0 +1,33 @@
+import { edge, end, node } from "@spendgraph/graph";
+import { FALLBACK } from "./classify.js";
+function handler(name, run) {
+    return node({
+        name,
+        run: (input, ctx) => run(input, ctx),
+    });
+}
+/** One node per branch, plus the fallback. */
+export function branches(opts, names) {
+    return [
+        ...names.map((name) => handler(name, opts.routes[name])),
+        handler(FALLBACK, opts.fallback),
+    ];
+}
+function taken(ctx) {
+    return ctx.outputs.classify?.taken ?? FALLBACK;
+}
+/**
+ * Classify, then exactly one branch, then stop.
+ *
+ * Conditional edges first and the fallback unconditional last, because `graph`
+ * refuses the other order — an unconditional edge ahead of a conditional one is
+ * a router whose default hides a branch, and it would never be reached.
+ */
+export function edgesTo(names) {
+    return [
+        ...names.map((name) => edge("classify", name, (ctx) => taken(ctx) === name)),
+        edge("classify", FALLBACK),
+        ...names.map((name) => end(name)),
+        end(FALLBACK),
+    ];
+}

package/dist/route/index.d.ts

@@ -0,0 +1,3 @@
+export { CLASSIFY, type Decision, decide, FALLBACK, normalise, type Said, } from "./classify.js";
+export { route } from "./route.js";
+export type { Classification, Handler, RouteOptions, RouteResult } from "./types.js";

package/dist/route/index.js

@@ -0,0 +1,2 @@
+export { CLASSIFY, decide, FALLBACK, normalise, } from "./classify.js";
+export { route } from "./route.js";

package/dist/route/route.d.ts

@@ -0,0 +1,11 @@
+import type { RouteOptions, RouteResult } from "./types.js";
+/**
+ * Classify once, then hand the work to the branch built for it.
+ *
+ * Compiled to a graph rather than looped here: `@spendgraph/graph` already
+ * validates that every branch is reachable and that no default sits in front of
+ * one, already records a step per node with its tokens, and already returns the
+ * rollout shape. A second execution engine in this package would be a second
+ * thing to keep right.
+ */
+export declare function route(values: Record<string, unknown>, opts: RouteOptions): Promise<RouteResult>;

package/dist/route/route.js

@@ -0,0 +1,31 @@
+import { graph } from "@spendgraph/graph";
+import { runGraph } from "../stream/graph.js";
+import { CLASSIFY, classifier, FALLBACK } from "./classify.js";
+import { branches, edgesTo } from "./dispatch.js";
+/**
+ * Classify once, then hand the work to the branch built for it.
+ *
+ * Compiled to a graph rather than looped here: `@spendgraph/graph` already
+ * validates that every branch is reachable and that no default sits in front of
+ * one, already records a step per node with its tokens, and already returns the
+ * rollout shape. A second execution engine in this package would be a second
+ * thing to keep right.
+ */
+export async function route(values, opts) {
+    const names = Object.keys(opts.routes);
+    const flow = graph({
+        entry: CLASSIFY,
+        maxSteps: opts.maxSteps,
+        nodes: [classifier(opts, names), ...branches(opts, names)],
+        edges: edgesTo(names),
+    }, { now: opts.now });
+    const result = await runGraph(flow, values, opts.emit);
+    const decision = result.outputs[CLASSIFY];
+    return {
+        ...result,
+        route: !decision || decision.taken === FALLBACK ? null : decision.taken,
+        classified: decision?.said ?? "",
+        confidence: decision?.confidence,
+        reason: decision?.reason,
+    };
+}

package/dist/route/types.d.ts

@@ -0,0 +1,53 @@
+import type { GraphContext, GraphResult } from "@spendgraph/graph";
+import type { Emit } from "../stream/index.js";
+/** What the classifier decided. A bare string is shorthand for `{ route }`. */
+export type Classification = string | {
+    route: string;
+    /** Below `minConfidence` the fallback runs instead. */
+    confidence?: number;
+    reason?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+};
+/** A branch, or the fallback. Whatever it returns becomes the run's output. */
+export type Handler = (input: Record<string, unknown>, ctx: GraphContext) => Promise<unknown> | unknown;
+export interface RouteOptions {
+    /**
+     * Which branch this input belongs to.
+     *
+     * Often better as plain code than a model — a regex on a subject line is free
+     * and never invents a category.
+     */
+    classify(input: Record<string, unknown>, ctx: GraphContext): Promise<Classification> | Classification;
+    /** One handler per branch, keyed by what `classify` returns. */
+    routes: Record<string, Handler>;
+    /**
+     * Runs when the classification matches no branch, or is not confident enough.
+     *
+     * Required: a router that can fail to route fails in production at 3am, and
+     * "unknown" is a class every classifier eventually returns.
+     */
+    fallback: Handler;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /** Below this the fallback runs. Ignored when the classifier reports none. */
+    minConfidence?: number;
+    /** Ceiling on total steps, nested ones included. Default 25. */
+    maxSteps?: number;
+    /** Injectable clock, so tests do not measure real time. */
+    now?: () => number;
+}
+export interface RouteResult extends GraphResult {
+    /** The branch that ran. `null` when the fallback did. */
+    route: string | null;
+    /** What the classifier said, whether or not it was used. */
+    classified: string;
+    confidence?: number;
+    reason?: string;
+}

package/dist/route/types.js

@@ -0,0 +1 @@
+export {};

package/dist/stream/graph.d.ts

@@ -0,0 +1,9 @@
+import type { GraphResult, graph } from "@spendgraph/graph";
+import type { Emit } from "./types.js";
+/**
+ * Runs a compiled graph, forwarding what its nodes emit.
+ *
+ * Streamed only when someone is listening: an unread stream is a queue nothing
+ * drains, and `execute` is the same run without one.
+ */
+export declare function runGraph(flow: ReturnType<typeof graph>, values: Record<string, unknown>, emit?: Emit): Promise<GraphResult>;

package/dist/stream/graph.js

@@ -0,0 +1,16 @@
+/**
+ * Runs a compiled graph, forwarding what its nodes emit.
+ *
+ * Streamed only when someone is listening: an unread stream is a queue nothing
+ * drains, and `execute` is the same run without one.
+ */
+export async function runGraph(flow, values, emit) {
+    if (!emit)
+        return flow.execute(values);
+    const running = flow.stream(values);
+    for await (const event of running) {
+        if (event.type === "token")
+            emit(event.text, event.node);
+    }
+    return running.result;
+}

package/dist/stream/index.d.ts

@@ -0,0 +1,2 @@
+export { streamed } from "./stream.js";
+export type { Emit, HarnessEvent, HarnessStream } from "./types.js";

package/dist/stream/index.js

@@ -0,0 +1 @@
+export { streamed } from "./stream.js";