@spendgraph/harness 0.1.0 → 0.2.1

package/dist/refine/refine.js

@@ -0,0 +1,59 @@
+import { attemptStep, tokensOf, totalTokens } from "./attempt.js";
+import { best, judgeStep } from "./judge.js";
+const DEFAULT_ROUNDS = 3;
+export async function refine(opts) {
+    const startedAt = Date.now();
+    const rounds = Math.max(1, opts.rounds ?? DEFAULT_ROUNDS);
+    const steps = [];
+    const history = [];
+    let feedback = null;
+    let spent = 0;
+    let stoppedBy = "rounds";
+    for (let round = 0; round < rounds; round++) {
+        if (opts.maxTokens !== undefined && spent >= opts.maxTokens) {
+            stoppedBy = "tokens";
+            break;
+        }
+        const attempt = await opts.attempt(feedback, round, (text, stage) => opts.emit?.(text, stage ?? `round ${round}`));
+        steps.push(attemptStep(steps.length, round, attempt));
+        spent += tokensOf(attempt);
+        if (attempt.status === "failed") {
+            return {
+                status: "failed",
+                output: "",
+                error: attempt.error ?? "The attempt failed.",
+                steps,
+                outputs: { history },
+                latencyMs: Date.now() - startedAt,
+                ...totalTokens(steps),
+                accepted: false,
+                rounds: round + 1,
+                stoppedBy: "failed",
+                history,
+            };
+        }
+        const verdict = await opts.judge(attempt, round);
+        steps.push(judgeStep(steps.length, round, verdict));
+        const record = { round, attempt, verdict, tokens: tokensOf(attempt) };
+        history.push(record);
+        opts.onRound?.(record);
+        if (verdict.accepted) {
+            stoppedBy = "accepted";
+            break;
+        }
+        feedback = verdict.feedback ?? null;
+    }
+    const winner = best(history);
+    return {
+        status: "completed",
+        output: winner?.attempt.output ?? "",
+        steps,
+        outputs: { history },
+        latencyMs: Date.now() - startedAt,
+        ...totalTokens(steps),
+        accepted: stoppedBy === "accepted",
+        rounds: history.length,
+        stoppedBy,
+        history,
+    };
+}

package/dist/refine/types.d.ts

@@ -0,0 +1,90 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+/** What one attempt produced. Structurally the `TraceOutcome` a prompt accepts. */
+export interface Attempt {
+    output: string;
+    model: string;
+    status?: "completed" | "failed";
+    error?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}
+/** What the judge decided about an attempt. */
+export interface Verdict {
+    /** True stops the loop. */
+    accepted: boolean;
+    /**
+     * What to fix, handed to the next attempt.
+     *
+     * A rejection with no feedback asks the model to try again with nothing new,
+     * which is how a refine loop spends three rounds producing the same draft.
+     */
+    feedback?: string;
+    /** Higher is better. Used to pick a winner when nothing was accepted. */
+    score?: number;
+}
+export interface RefineOptions {
+    /**
+     * A draft. `feedback` is null on the first round and the judge's note after.
+     *
+     * One callback rather than separate generate and revise, because in practice
+     * they are the same prompt with one extra paragraph.
+     */
+    attempt(feedback: string | null, round: number, emit: Emit): Promise<Attempt> | Attempt;
+    /** Whether that will do, and what is wrong with it if not. */
+    judge(attempt: Attempt, round: number): Promise<Verdict> | Verdict;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /** Most attempts to make. Default 3. */
+    rounds?: number;
+    /**
+     * Stop once the attempts have consumed this many tokens.
+     *
+     * Tokens rather than money because this package cannot price a call — that
+     * happens on the server, after the fact. Checked before each round, so the
+     * ceiling is never crossed rather than merely noticed.
+     */
+    maxTokens?: number;
+    /** Called after each verdict, so a long loop is not silent. */
+    onRound?: (round: RoundRecord) => void;
+}
+export interface RoundRecord {
+    round: number;
+    attempt: Attempt;
+    verdict: Verdict;
+    tokens: number;
+}
+/**
+ * What the loop produced, in the shape of a rollout.
+ *
+ * `steps` holds every attempt and every judgement, including the rejected ones.
+ * A loop that recorded only the winner could not say what the answer cost, which
+ * is the whole point of recording it here.
+ */
+export interface RefineResult {
+    status: "completed" | "failed";
+    output: string;
+    error?: string;
+    steps: RolloutStep[];
+    outputs: Record<string, unknown>;
+    latencyMs: number;
+    /**
+     * Summed across **every** attempt, the rejected ones included.
+     *
+     * The only question a refine loop has to answer is whether refining paid, and
+     * a total that counted the winner alone would always say yes.
+     */
+    inputTokens: number;
+    outputTokens: number;
+    /** False when the rounds or the token ceiling ran out first. */
+    accepted: boolean;
+    rounds: number;
+    /** Why it ended: a verdict, the round limit, or the token ceiling. */
+    stoppedBy: "accepted" | "rounds" | "tokens" | "failed";
+    history: RoundRecord[];
+}

package/dist/refine/types.js

@@ -0,0 +1 @@
+export {};

package/dist/route/classify.d.ts

@@ -0,0 +1,52 @@
+import type { Classification, RouteOptions } from "./types.js";
+/** The classifier's node name, and the key its decision is stored under. */
+export declare const CLASSIFY = "classify";
+/**
+ * The branch that runs when no other one does.
+ *
+ * Declared here rather than beside the dispatch it names, because `decide` is
+ * what produces it — and the other way round leaves the two files importing
+ * each other, which is the thing a reader has to stop and untangle.
+ */
+export declare const FALLBACK = "fallback";
+/** What the classifier said, once a bare string has been expanded. */
+export type Said = Exclude<Classification, string>;
+/**
+ * What the classifier said, and what will run because of it.
+ *
+ * The two are kept apart on purpose: a router that recorded only the branch it
+ * ran could not tell "the classifier said `legal` and we have no such branch"
+ * from "it was unsure about `billing`". The first is a missing branch, the
+ * second a miscalibrated one, and they want opposite fixes.
+ */
+export interface Decision {
+    said: string;
+    taken: string;
+    confidence?: number;
+    reason?: string;
+}
+/** A bare string is shorthand for `{ route }`. */
+export declare function normalise(value: Classification): Said;
+/**
+ * Which branch runs.
+ *
+ * A class no branch handles falls back, and so does one the classifier is not
+ * sure enough about — a hedged guess sent to the wrong specialist is worse than
+ * the general path.
+ */
+export declare function decide(said: Said, names: string[], minConfidence?: number): Decision;
+/** A step's worth of outcome, with the decision the edges will read. */
+export interface ClassifyOutput extends Decision {
+    output: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}
+/**
+ * The entry node.
+ *
+ * Its return is what the edges read, so the decision travels through the graph's
+ * own context rather than a variable closed over from outside — which means a
+ * finished run can be read apart from `outputs.classify` alone.
+ */
+export declare function classifier(opts: RouteOptions, names: string[]): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>;

package/dist/route/classify.js

@@ -0,0 +1,33 @@
+import { node } from "@spendgraph/graph";
+export const CLASSIFY = "classify";
+export const FALLBACK = "fallback";
+export function normalise(value) {
+    return typeof value === "string" ? { route: value } : value;
+}
+export function decide(said, names, minConfidence) {
+    const known = names.includes(said.route);
+    const confident = minConfidence === undefined ||
+        said.confidence === undefined ||
+        said.confidence >= minConfidence;
+    return {
+        said: said.route,
+        taken: known && confident ? said.route : FALLBACK,
+        confidence: said.confidence,
+        reason: said.reason,
+    };
+}
+export function classifier(opts, names) {
+    return node({
+        name: CLASSIFY,
+        run: async (input, ctx) => {
+            const said = normalise(await opts.classify(input, ctx));
+            return {
+                ...decide(said, names, opts.minConfidence),
+                output: said.route,
+                model: said.model,
+                inputTokens: said.inputTokens,
+                outputTokens: said.outputTokens,
+            };
+        },
+    });
+}

package/dist/route/dispatch.d.ts

@@ -0,0 +1,12 @@
+import { type Edge } from "@spendgraph/graph";
+import type { RouteOptions } from "./types.js";
+/** One node per branch, plus the fallback. */
+export declare function branches(opts: RouteOptions, names: string[]): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>[];
+/**
+ * Classify, then exactly one branch, then stop.
+ *
+ * Conditional edges first and the fallback unconditional last, because `graph`
+ * refuses the other order — an unconditional edge ahead of a conditional one is
+ * a router whose default hides a branch, and it would never be reached.
+ */
+export declare function edgesTo(names: string[]): Edge[];

package/dist/route/dispatch.js

@@ -0,0 +1,25 @@
+import { edge, end, node } from "@spendgraph/graph";
+import { FALLBACK } from "./classify.js";
+function handler(name, run) {
+    return node({
+        name,
+        run: (input, ctx) => run(input, ctx),
+    });
+}
+export function branches(opts, names) {
+    return [
+        ...names.map((name) => handler(name, opts.routes[name])),
+        handler(FALLBACK, opts.fallback),
+    ];
+}
+function taken(ctx) {
+    return ctx.outputs.classify?.taken ?? FALLBACK;
+}
+export function edgesTo(names) {
+    return [
+        ...names.map((name) => edge("classify", name, (ctx) => taken(ctx) === name)),
+        edge("classify", FALLBACK),
+        ...names.map((name) => end(name)),
+        end(FALLBACK),
+    ];
+}

package/dist/route/index.d.ts

@@ -0,0 +1,3 @@
+export { CLASSIFY, type Decision, decide, FALLBACK, normalise, type Said, } from "./classify.js";
+export { route } from "./route.js";
+export type { Classification, Handler, RouteOptions, RouteResult } from "./types.js";

package/dist/route/index.js

@@ -0,0 +1,2 @@
+export { CLASSIFY, decide, FALLBACK, normalise, } from "./classify.js";
+export { route } from "./route.js";

package/dist/route/route.d.ts

@@ -0,0 +1,11 @@
+import type { RouteOptions, RouteResult } from "./types.js";
+/**
+ * Classify once, then hand the work to the branch built for it.
+ *
+ * Compiled to a graph rather than looped here: `@spendgraph/graph` already
+ * validates that every branch is reachable and that no default sits in front of
+ * one, already records a step per node with its tokens, and already returns the
+ * rollout shape. A second execution engine in this package would be a second
+ * thing to keep right.
+ */
+export declare function route(values: Record<string, unknown>, opts: RouteOptions): Promise<RouteResult>;

package/dist/route/route.js

@@ -0,0 +1,22 @@
+import { graph } from "@spendgraph/graph";
+import { runGraph } from "../stream/graph.js";
+import { CLASSIFY, classifier, FALLBACK } from "./classify.js";
+import { branches, edgesTo } from "./dispatch.js";
+export async function route(values, opts) {
+    const names = Object.keys(opts.routes);
+    const flow = graph({
+        entry: CLASSIFY,
+        maxSteps: opts.maxSteps,
+        nodes: [classifier(opts, names), ...branches(opts, names)],
+        edges: edgesTo(names),
+    }, { now: opts.now });
+    const result = await runGraph(flow, values, opts.emit);
+    const decision = result.outputs[CLASSIFY];
+    return {
+        ...result,
+        route: !decision || decision.taken === FALLBACK ? null : decision.taken,
+        classified: decision?.said ?? "",
+        confidence: decision?.confidence,
+        reason: decision?.reason,
+    };
+}

package/dist/route/types.d.ts

@@ -0,0 +1,53 @@
+import type { GraphContext, GraphResult } from "@spendgraph/graph";
+import type { Emit } from "../stream/index.js";
+/** What the classifier decided. A bare string is shorthand for `{ route }`. */
+export type Classification = string | {
+    route: string;
+    /** Below `minConfidence` the fallback runs instead. */
+    confidence?: number;
+    reason?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+};
+/** A branch, or the fallback. Whatever it returns becomes the run's output. */
+export type Handler = (input: Record<string, unknown>, ctx: GraphContext) => Promise<unknown> | unknown;
+export interface RouteOptions {
+    /**
+     * Which branch this input belongs to.
+     *
+     * Often better as plain code than a model — a regex on a subject line is free
+     * and never invents a category.
+     */
+    classify(input: Record<string, unknown>, ctx: GraphContext): Promise<Classification> | Classification;
+    /** One handler per branch, keyed by what `classify` returns. */
+    routes: Record<string, Handler>;
+    /**
+     * Runs when the classification matches no branch, or is not confident enough.
+     *
+     * Required: a router that can fail to route fails in production at 3am, and
+     * "unknown" is a class every classifier eventually returns.
+     */
+    fallback: Handler;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /** Below this the fallback runs. Ignored when the classifier reports none. */
+    minConfidence?: number;
+    /** Ceiling on total steps, nested ones included. Default 25. */
+    maxSteps?: number;
+    /** Injectable clock, so tests do not measure real time. */
+    now?: () => number;
+}
+export interface RouteResult extends GraphResult {
+    /** The branch that ran. `null` when the fallback did. */
+    route: string | null;
+    /** What the classifier said, whether or not it was used. */
+    classified: string;
+    confidence?: number;
+    reason?: string;
+}

package/dist/route/types.js

@@ -0,0 +1 @@
+export {};

package/dist/stream/graph.d.ts

@@ -0,0 +1,9 @@
+import type { GraphResult, graph } from "@spendgraph/graph";
+import type { Emit } from "./types.js";
+/**
+ * Runs a compiled graph, forwarding what its nodes emit.
+ *
+ * Streamed only when someone is listening: an unread stream is a queue nothing
+ * drains, and `execute` is the same run without one.
+ */
+export declare function runGraph(flow: ReturnType<typeof graph>, values: Record<string, unknown>, emit?: Emit): Promise<GraphResult>;

package/dist/stream/graph.js

@@ -0,0 +1,10 @@
+export async function runGraph(flow, values, emit) {
+    if (!emit)
+        return flow.execute(values);
+    const running = flow.stream(values);
+    for await (const event of running) {
+        if (event.type === "token")
+            emit(event.text, event.node);
+    }
+    return running.result;
+}

package/dist/stream/index.d.ts

@@ -0,0 +1,2 @@
+export { streamed } from "./stream.js";
+export type { Emit, HarnessEvent, HarnessStream } from "./types.js";

package/dist/stream/index.js

@@ -0,0 +1 @@
+export { streamed } from "./stream.js";

package/dist/stream/stream.d.ts

@@ -0,0 +1,19 @@
+import type { Emit, HarnessStream } from "./types.js";
+/**
+ * Any workflow, watched while it runs.
+ *
+ * One helper rather than a streaming variant of each of the seven: every
+ * workflow already takes an `emit`, so wrapping the call is all it takes, and a
+ * `stream()` per pattern would be seven places for the same queue to go wrong.
+ *
+ *     const run = streamed((emit, note) =>
+ *       cascade(values, { ...opts, emit, onTier: note })
+ *     );
+ *     for await (const event of run) { ... }
+ *     const result = await run.result;
+ *
+ * The run starts on the call, not on the first read, so a caller who wants only
+ * the result never has to iterate, and a client that connects late still sees
+ * what already happened.
+ */
+export declare function streamed<T>(run: (emit: Emit, note: (data: unknown) => void) => Promise<T>): HarnessStream<T>;

package/dist/stream/stream.js

@@ -0,0 +1,48 @@
+class Events {
+    waiting = [];
+    wake = null;
+    closed = false;
+    push(event) {
+        this.waiting.push(event);
+        this.release();
+    }
+    close() {
+        this.closed = true;
+        this.release();
+    }
+    release() {
+        const wake = this.wake;
+        this.wake = null;
+        wake?.();
+    }
+    async *drain() {
+        while (true) {
+            while (this.waiting.length > 0) {
+                yield this.waiting.shift();
+            }
+            if (this.closed)
+                return;
+            await new Promise((resolve) => {
+                this.wake = resolve;
+            });
+        }
+    }
+}
+export function streamed(run) {
+    const events = new Events();
+    const result = (async () => {
+        try {
+            const finished = await run((text, stage) => events.push({ type: "token", text, stage }), (data) => events.push({ type: "note", data }));
+            events.push({ type: "result", result: finished });
+            return finished;
+        }
+        finally {
+            events.close();
+        }
+    })();
+    void result.catch(() => { });
+    return {
+        result,
+        [Symbol.asyncIterator]: () => events.drain(),
+    };
+}

package/dist/stream/types.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Where a workflow's tokens go.
+ *
+ * `stage` is which part of the workflow produced them — a tier, a task, a node.
+ * A fan-out interleaves, and without it a stream of four workers reads as one
+ * confused voice. Assignable to a model client's `onText` either way, so a
+ * callback can pass it straight down.
+ */
+export type Emit = (text: string, stage?: string) => void;
+/** What a streamed workflow says while it runs, and what it ends with. */
+export type HarnessEvent<T> = {
+    type: "token";
+    text: string;
+    stage?: string;
+} | {
+    type: "note";
+    data: unknown;
+} | {
+    type: "result";
+    result: T;
+};
+/** A workflow as it happens, and the same run once it is over. */
+export interface HarnessStream<T> extends AsyncIterable<HarnessEvent<T>> {
+    /**
+     * The finished run.
+     *
+     * Resolves whether or not anyone iterated, and rejects where the workflow
+     * itself would have — streaming changes what you can watch, not what happens.
+     */
+    result: Promise<T>;
+}

package/dist/stream/types.js

@@ -0,0 +1 @@
+export {};

package/docs/README.md

@@ -0,0 +1,41 @@
+# Workflows
+
+A workflow is a wiring pattern that keeps being rediscovered. Each one is a
+graph you would otherwise hand-roll, with the pricing and the rollout already
+attached, so a run is recorded rather than reconstructed afterwards.
+
+| | | Built |
+| --- | --- | --- |
+| [`refine`](./refine.md) | draft, judge, revise until a bar is met | ✓ |
+| [`route`](./route.md) | classify first, then dispatch to a specialist | ✓ |
+| [`chain`](./chain.md) | steps in sequence, each on the last's output, with a gate between | ✓ |
+| [`parallel`](./parallel.md) | the same call N times, or N sections at once | ✓ |
+| [`orchestrate`](./orchestrate.md) | a lead decomposes and delegates to workers | ✓ |
+| [`loop`](./loop.md) | tools in a loop until the model stops asking | ✓ |
+| [`cascade`](./cascade.md) | try the cheap model, escalate only when it will not do | ✓ |
+
+## What every one of them owes you
+
+**A rollout shape.** The result hands to `prompt.report()` without being
+translated, because a workflow run *is* a rollout with several steps.
+
+**Every step, including the rejected ones.** A refine loop that keeps only the
+winner cannot say what the answer cost. A router that records only the branch it
+ran cannot tell a missing branch from a miscalibrated one.
+
+**A ceiling.** Each of these spends more than one call, and the interesting ones
+spend an unbounded amount. The ceiling is in **tokens**, not money: this package
+cannot price a call — that happens on the server, after the fact — so a budget in
+micro-dollars here would be a guess.
+
+## Why they are not in `graph`
+
+`graph` is the mechanism: nodes, edges, a run. These are the arrangements of it
+worth naming, each with its own failure mode — a chain needs a gate or one bad
+step poisons the rest, voting needs an odd N, a loop needs a ceiling. Putting
+them beside the mechanism would suggest the mechanism is opinionated. It is not.
+
+`route` compiles down to a graph and adds nothing to it. `refine` does not,
+because it is a **loop**: `graph` is a DAG whose `maxSteps` error reads "a
+conditional edge never turns false", so a cycle there is a runaway to be stopped
+rather than a shape to express.