@spendgraph/harness 0.1.0 → 0.2.0

package/README.md

@@ -1,162 +1,89 @@
 # @spendgraph/harness
 
-Pull a stored prompt, render it, report what it cost. Zero dependencies.
-
-Your prompt lives in spendgraph, not in your codebase. This client fetches it,
-substitutes its typed fields, and records what the call cost — without sitting
-in your request path.
-
-```sh
-npm install @spendgraph/harness
-```
-
-## The production path
-
-```ts
-import { Harness } from "@spendgraph/harness";
-
-const harness = new Harness({
-  apiKey: process.env.SPENDGRAPH_API_KEY,
-  baseUrl: "https://your-spendgraph.example.com",
-});
-
-export async function handle(request) {
-  const prompt = await harness.pull("billing-explainer-k3f9x2");
-
-  const values = { question: "why was I charged twice?" };
-  const messages = prompt.render(values);
-
-  // Call the provider yourself. spendgraph is never between you and the model.
-  const startedAt = Date.now();
-  const completion = await yourModelCall(messages);
-
-  await harness.report(prompt.id, {
-    versionId: prompt.versionId,
-    model: "anthropic/claude-sonnet-5",
-    fields: prompt.serialize(values),
-    rendered: messages,
-    output: completion.text,
-    inputTokens: completion.usage.inputTokens,
-    outputTokens: completion.usage.outputTokens,
-    cacheReadTokens: completion.usage.cacheReadTokens ?? 0,
-    cacheWriteTokens: completion.usage.cacheWriteTokens ?? 0,
-    latencyMs: Date.now() - startedAt,
-  });
-
-  return completion.text;
-}
-```
-
-Call `pull` inside the handler, not at module scope. It refreshes as a side
-effect of being called, so a service that pulls once at boot serves that wording
-for the life of the process. Called per request it is a cache read, not a round
-trip.
-
-`report` never throws and never rejects — it sits beside a request that has
-already been answered, and telemetry that can break the thing it measures is
-worse than no telemetry. Pass `onReportError` if you want to know when one is
-dropped.
-
-## Handles
-
-`pull`, `run` and the rest take a prompt's uuid or its slug.
-
-A slug is derived from the prompt's name, so **renaming the prompt mints a new
-one and the old stops resolving**. The uuid never changes. Pull by slug for
-something readable; pull by id where the handle has to outlive an edit to the
-name.
-
-When a handle stops resolving, the client keeps serving the wording it last saw
-rather than failing your request — and backs off rather than retrying on every
-call. Pass `onPullError` to be told, because otherwise that is silent:
-
-```ts
-new Harness({
-  apiKey,
-  baseUrl,
-  onPullError: (err, handle) => log.warn({ err, handle }, "prompt refresh failed"),
-});
-```
-
-## Caching
-
-`pull` is stale-while-revalidate. A fresh entry is returned outright, a stale
-one is returned *now* and refreshed behind you, and only a cold miss waits on
-the network. So an edit in the dashboard reaches production within a TTL without
-any request paying for the fetch.
-
-```ts
-new Harness({ apiKey, baseUrl, cache: { ttlSeconds: 300, maxSize: 100 } });
-```
-
-`harness.invalidate(promptId)` drops it immediately — `promote()` does this for
-you, including for entries cached under the other handle.
-
-## The batch path
-
-`run`, `sample` and `runAll` execute the prompt server-side and record the
-rollout in one call. That adds a hop and makes spendgraph a dependency of your
-uptime, which is the right trade for evaluation and the wrong one for a user
-waiting on a response.
-
-```ts
-const result = await harness.run("billing-explainer-k3f9x2", values);
-const k = await harness.sample(promptId, values, { k: 5 }); // pass^k
-const across = await harness.runAll(promptId, values);      // one per model
-```
-
-Cap what a client can spend:
-
-```ts
-new Harness({ apiKey, baseUrl, maxCostMicros: 5_000_000 }); // $5
-harness.spent();      // micro-dollars used
-harness.remaining();  // what is left
-```
-
-Omit `maxCostMicros` and it warns once that it is unbounded. Pass `null` to say
-you meant it.
-
-## Typed fields
-
-A prompt declares its placeholders — `string`, `text`, `number`, `boolean`,
-`enum`, `list`, `json` — and `render` validates before anything is sent, so a
-missing required field is an exception rather than a `{placeholder}` reaching
-the model.
-
-`harness codegen` turns those declarations into types:
-
-```sh
-npx harness codegen --url https://your-spendgraph.example.com
-# usage: harness codegen [--out <file>] [--project <id>] [--module <specifier>]
-```
-
-It reads `SPENDGRAPH_API_KEY` and `SPENDGRAPH_URL` from the environment and
-writes `src/harness.generated.d.ts`. Include that file and the values argument
-is checked per prompt, by id and by slug. Delete it and calls fall back to a
-loose `Record<string, unknown>` — the runtime validation is unchanged either
-way.
-
-## Versions
-
-```ts
-const { versions, currentVersionId } = await harness.versions(promptId);
-await harness.promote(promptId, versionId);
-```
-
-Promoting changes what every `pull` serves from that moment, and invalidates the
-cache so the change is visible immediately rather than after a TTL.
-
-## Datasets
-
-```ts
-const { cases, counts } = await harness.cases(promptId);
-await harness.setCases(promptId, cases);
-```
-
-`setCases` replaces the whole set. A dataset answers "what am I evaluating
-against" with one answer at a time, and a merge would leave no way to remove a
-case.
-
-## License
-
-MIT
+**Agent = Model + Harness.** The model reasons. The harness is everything around
+it that turns text into work — choosing what runs next, calling tools, holding
+the result together, and knowing what to do when a step fails.
+
+This is the *outer* harness: the part you assemble, on top of whatever agent SDK
+your provider ships. It is built on `@spendgraph/prompt`, `@spendgraph/tools`
+and `@spendgraph/graph`, and none of it reasons — every workflow here directs a
+model call you supply.
+
+Six shapes, all built. Each is a graph you would otherwise hand-roll, with the
+pricing and the rollout already attached, so a run is recorded rather than
+reconstructed afterwards.
+
+| Workflow | What it is | When it wins |
+| --- | --- | --- |
+| **`chain`** | steps in sequence, each on the last's output, with a gate between | a task that decomposes cleanly and reads better as stages |
+| **`route`** | classify first, then dispatch to a specialist | inputs that fall into kinds a cheaper model can tell apart |
+| **`parallel`** | the same call N times, or N sections at once | voting for confidence, sectioning for latency |
+| **`orchestrate`** | a lead decomposes and delegates to workers | the subtasks are not known until the input is read |
+| **`refine`** | generate, critique, revise until a bar is met | there is a clear criterion and a first draft rarely meets it |
+| **`loop`** | tools in a loop until the model stops asking | open-ended work with a tool surface and a step ceiling |
+| **`cascade`** | try the cheap model, escalate only when it will not do | most inputs are easy and a few are not, and you can tell which |
+
+## Inner and outer
+
+The provider's SDK is the **inner** harness: it drives one model call, and on
+Anthropic or OpenAI it will loop over tool calls for you. `@spendgraph/llms`
+wraps that.
+
+What it does not do is decide *which* call to make, what to do with the answer,
+when to stop, or what any of it cost. That is the **outer** harness, and it is
+the part that is written again in every project — usually inline, usually
+without a ceiling, and usually recording nothing.
+
+| | |
+| --- | --- |
+| Inner | one call, tool loop, streaming — `@spendgraph/llms` |
+| Outer | which call, what next, when to stop, what it cost — here |
+
+## Why here and not in `graph`
+
+`graph` is the mechanism: nodes, edges, a run. These are the seven arrangements of
+it worth naming, each with its own failure mode — a chain needs a gate or a bad
+step poisons the rest, voting needs an odd N, a loop needs a ceiling and every
+one of them needs a budget. Putting them beside the mechanism would suggest the
+mechanism is opinionated. It is not.
+
+## What every one of them owes you
+
+- A `GraphResult`, so the run reports as a rollout without translation.
+- Every step recorded, including the ones that were rejected — a refine loop that
+  keeps only the winner cannot say what it cost.
+- A budget ceiling, because each of these spends more than one call and the
+  interesting ones spend an unbounded amount.
+- A `stoppedBy`, because "it finished" and "it gave up" must never look the same.
+
+## What is missing
+
+Every workflow here is **single-run**: it starts, it finishes, and it holds
+nothing afterwards. A harness in the fuller sense also carries state across a
+pause — a run that waits for a person, resumes tomorrow, or picks up where a
+crash left it.
+
+That needs somewhere to persist a run, which is a decision about the product and
+not a wiring pattern. It is written up in [`PLAN.md`](./PLAN.md) rather than
+quietly folded in.
+
+## What is next
+
+Phase one is done: the context manager (`compact`), lifecycle hooks, tool
+annotations, budget signalling and `cascade`. What is left is **durable state** —
+a run that can wait for a person and be resumed — which [`PLAN.md`](./PLAN.md)
+plans with the decisions and the tests it needs.
+
+## Documentation
+
+Each built workflow has a page with worked examples, the cases it suits, and the
+cases it does not.
+
+- [`docs/refine.md`](./docs/refine.md) — draft, judge, revise
+- [`docs/route.md`](./docs/route.md) — classify, then dispatch
+- [`docs/chain.md`](./docs/chain.md) — steps in sequence, with a gate between
+- [`docs/parallel.md`](./docs/parallel.md) — sectioning for latency, voting for confidence
+- [`docs/loop.md`](./docs/loop.md) — tools until the model stops asking
+- [`docs/orchestrate.md`](./docs/orchestrate.md) — a lead decomposes and delegates
+- [`docs/cascade.md`](./docs/cascade.md) — cheap first, escalate on rejection
+- [`docs/README.md`](./docs) — the seven patterns, and what each owes you

package/dist/cascade/cascade.d.ts

@@ -0,0 +1,13 @@
+import type { CascadeOptions, CascadeResult } from "./types.js";
+/**
+ * Try the cheap model, and escalate only when its answer will not do.
+ *
+ * Not `route`, which chooses before it has seen an answer. Not `refine`, which
+ * revises with the same model. This is the one lever that spends less by
+ * default and more only on the inputs that turn out to need it.
+ *
+ * Whether it pays depends entirely on how often the first tier is accepted, so
+ * every tier tried is recorded and its tokens counted — a total that hid the
+ * rejected attempts would report a saving on every run.
+ */
+export declare function cascade(values: Record<string, unknown>, opts: CascadeOptions): Promise<CascadeResult>;

package/dist/cascade/cascade.js

@@ -0,0 +1,91 @@
+import { attempt, fallback, tierStep, tokensOf } from "./tier.js";
+function checkTiers(tiers) {
+    if (tiers.length === 0)
+        throw new Error("cascade needs at least one tier.");
+    const seen = new Set();
+    for (const tier of tiers) {
+        if (!tier.name.trim())
+            throw new Error("Every cascade tier needs a name.");
+        if (seen.has(tier.name))
+            throw new Error(`Two cascade tiers are called "${tier.name}".`);
+        seen.add(tier.name);
+    }
+}
+/**
+ * Try the cheap model, and escalate only when its answer will not do.
+ *
+ * Not `route`, which chooses before it has seen an answer. Not `refine`, which
+ * revises with the same model. This is the one lever that spends less by
+ * default and more only on the inputs that turn out to need it.
+ *
+ * Whether it pays depends entirely on how often the first tier is accepted, so
+ * every tier tried is recorded and its tokens counted — a total that hid the
+ * rejected attempts would report a saving on every run.
+ */
+export async function cascade(values, opts) {
+    const startedAt = Date.now();
+    checkTiers(opts.tiers);
+    const steps = [];
+    const history = [];
+    let spent = 0;
+    let accepted;
+    let stoppedBy = "exhausted";
+    for (const [index, tier] of opts.tiers.entries()) {
+        if (opts.maxTokens !== undefined && spent >= opts.maxTokens) {
+            stoppedBy = "tokens";
+            break;
+        }
+        const { answer, error } = await attempt(tier.run, values, (text, stage) => opts.emit?.(text, stage ?? tier.name));
+        const passed = answer !== null && !error && (await opts.accept(answer, index));
+        const record = {
+            index,
+            name: tier.name,
+            answer,
+            accepted: passed,
+            ...(passed ? {} : { escalatedBy: error ? "failed" : "rejected" }),
+            error,
+            tokens: tokensOf(answer),
+        };
+        steps.push(tierStep(steps.length, record));
+        spent += record.tokens;
+        history.push(record);
+        opts.onTier?.(record);
+        if (passed) {
+            accepted = record;
+            stoppedBy = "accepted";
+            break;
+        }
+    }
+    const winner = accepted ?? fallback(history);
+    const totals = history.reduce((sum, record) => ({
+        inputTokens: sum.inputTokens + (record.answer?.inputTokens ?? 0),
+        outputTokens: sum.outputTokens + (record.answer?.outputTokens ?? 0),
+    }), { inputTokens: 0, outputTokens: 0 });
+    const shared = {
+        steps,
+        outputs: { history },
+        latencyMs: Date.now() - startedAt,
+        ...totals,
+        accepted: accepted !== undefined,
+        escalated: history.filter((r) => r !== winner).map((r) => r.name),
+        history,
+    };
+    if (!winner) {
+        return {
+            ...shared,
+            status: "failed",
+            output: "",
+            error: history.at(-1)?.error ?? "No tier produced an answer.",
+            tier: null,
+            stoppedBy: stoppedBy === "tokens" ? "tokens" : "failed",
+        };
+    }
+    return {
+        ...shared,
+        status: "completed",
+        output: winner.answer?.output ?? "",
+        tier: winner.name,
+        stoppedBy,
+        ...(winner.answer?.data === undefined ? {} : { data: winner.answer.data }),
+    };
+}

package/dist/cascade/index.d.ts

@@ -0,0 +1,3 @@
+export { cascade } from "./cascade.js";
+export { attempt, fallback, tierStep, tokensOf } from "./tier.js";
+export type { Answer, CascadeOptions, CascadeResult, Escalation, Tier, TierRecord, } from "./types.js";

package/dist/cascade/index.js

@@ -0,0 +1,2 @@
+export { cascade } from "./cascade.js";
+export { attempt, fallback, tierStep, tokensOf } from "./tier.js";

package/dist/cascade/tier.d.ts

@@ -0,0 +1,31 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+import type { Answer, TierRecord } from "./types.js";
+/** What one tier consumed, both halves, whether or not its answer was used. */
+export declare function tokensOf(answer: Answer | null): number;
+/**
+ * One tier as a step, named for the tier rather than its position.
+ *
+ * A rollout that read `tier_0` would need the config beside it to mean anything,
+ * and the config is the thing most likely to have changed since.
+ */
+export declare function tierStep(index: number, record: TierRecord): RolloutStep;
+/**
+ * Runs one tier, turning a throw into an escalation.
+ *
+ * An overloaded cheap model is the case this workflow exists for, so a tier that
+ * falls over must hand on rather than end the run. The next rung is the whole
+ * point of having one.
+ */
+export declare function attempt(run: (values: Record<string, unknown>, emit: Emit) => Promise<Answer> | Answer, values: Record<string, unknown>, emit: Emit): Promise<{
+    answer: Answer | null;
+    error?: string;
+}>;
+/**
+ * The tier to fall back to when nothing was accepted.
+ *
+ * The last that produced anything, because the tiers are ordered by capability
+ * and the most capable answer is the closest thing to a best effort. A tier that
+ * threw produced nothing, so it cannot be the fallback however late it ran.
+ */
+export declare function fallback(history: TierRecord[]): TierRecord | undefined;

package/dist/cascade/tier.js

@@ -0,0 +1,53 @@
+/** What one tier consumed, both halves, whether or not its answer was used. */
+export function tokensOf(answer) {
+    if (!answer)
+        return 0;
+    return (answer.inputTokens ?? 0) + (answer.outputTokens ?? 0);
+}
+/**
+ * One tier as a step, named for the tier rather than its position.
+ *
+ * A rollout that read `tier_0` would need the config beside it to mean anything,
+ * and the config is the thing most likely to have changed since.
+ */
+export function tierStep(index, record) {
+    return {
+        index,
+        source: record.name,
+        model: record.answer?.model,
+        output: record.answer?.output ?? "",
+        status: record.escalatedBy === "failed" ? "failed" : "completed",
+        error: record.error,
+        inputTokens: record.answer?.inputTokens,
+        outputTokens: record.answer?.outputTokens,
+    };
+}
+/**
+ * Runs one tier, turning a throw into an escalation.
+ *
+ * An overloaded cheap model is the case this workflow exists for, so a tier that
+ * falls over must hand on rather than end the run. The next rung is the whole
+ * point of having one.
+ */
+export async function attempt(run, values, emit) {
+    try {
+        const answer = await run(values, emit);
+        if (answer.status === "failed") {
+            return { answer, error: answer.error ?? "The tier failed." };
+        }
+        return { answer };
+    }
+    catch (cause) {
+        return { answer: null, error: cause instanceof Error ? cause.message : String(cause) };
+    }
+}
+/**
+ * The tier to fall back to when nothing was accepted.
+ *
+ * The last that produced anything, because the tiers are ordered by capability
+ * and the most capable answer is the closest thing to a best effort. A tier that
+ * threw produced nothing, so it cannot be the fallback however late it ran.
+ */
+export function fallback(history) {
+    return [...history].reverse().find((record) => record.answer !== null);
+}

package/dist/cascade/types.d.ts

@@ -0,0 +1,98 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+/** What one tier produced. Structurally the `LlmReply` a provider read gives back. */
+export interface Answer {
+    output: string;
+    model: string;
+    status?: "completed" | "failed";
+    error?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    /** A parsed reply, where the tier asked for a shape. What `accept` usually reads. */
+    data?: unknown;
+}
+/** One rung: a name for the record, and the call it makes. */
+export interface Tier {
+    /** How it appears in `result.tier`, `escalated` and the steps. Unique. */
+    name: string;
+    run(values: Record<string, unknown>, emit: Emit): Promise<Answer> | Answer;
+}
+/** Why a tier did not answer. */
+export type Escalation = "rejected" | "failed";
+/** What one tier did, whether it answered or not. */
+export interface TierRecord {
+    index: number;
+    name: string;
+    /** What it produced, or null when the call threw. */
+    answer: Answer | null;
+    accepted: boolean;
+    /**
+     * Absent when it answered.
+     *
+     * `rejected` is `accept` saying no; `failed` is the call falling over. They
+     * want opposite fixes — a bar set too high against a model that is down — and
+     * a record that collapsed them could not tell you which you have.
+     */
+    escalatedBy?: Escalation;
+    error?: string;
+    tokens: number;
+}
+export interface CascadeOptions {
+    /**
+     * Rungs in order, cheapest first.
+     *
+     * Ordered by capability rather than price where the two disagree: the last is
+     * what a run falls back to, so it should be the one most likely to cope.
+     */
+    tiers: Tier[];
+    /**
+     * Whether that answer will do.
+     *
+     * Required. Without it there is no cascade — only a list of models and no
+     * reason to stop at any of them.
+     */
+    accept(answer: Answer, index: number): Promise<boolean> | boolean;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /**
+     * Stop once the tiers have consumed this many tokens.
+     *
+     * Checked before each tier, so the ceiling is never crossed rather than merely
+     * noticed. Tokens rather than money because this package cannot price a call.
+     */
+    maxTokens?: number;
+    /** Called after each tier, so a long escalation is not silent. */
+    onTier?: (record: TierRecord) => void;
+}
+/**
+ * What the cascade produced, in the shape of a rollout.
+ *
+ * `inputTokens` and `outputTokens` cover **every** tier tried, not the winner
+ * alone. The only question a cascade has to answer is whether it paid, and a
+ * total that hid the rejected attempts would always say yes.
+ */
+export interface CascadeResult {
+    status: "completed" | "failed";
+    output: string;
+    error?: string;
+    steps: RolloutStep[];
+    outputs: Record<string, unknown>;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    /** True when a tier passed `accept`. False when the answer is a fallback. */
+    accepted: boolean;
+    /** Whose answer this is: the one accepted, or the last that produced one. */
+    tier: string | null;
+    /** Tiers tried that did not supply the answer, in the order they were tried. */
+    escalated: string[];
+    stoppedBy: "accepted" | "exhausted" | "tokens" | "failed";
+    /** The winning tier's parsed reply, where it had one. */
+    data?: unknown;
+    history: TierRecord[];
+}

package/dist/chain/chain.d.ts

@@ -0,0 +1,9 @@
+import type { ChainOptions, ChainResult } from "./types.js";
+/**
+ * Steps in sequence, each on the last one's output, with a gate between.
+ *
+ * Compiled to a graph, which already stops at the node that failed and records
+ * every node that ran before it — which is the whole reason to gate a chain
+ * rather than let it run to the end and hand back something plausible.
+ */
+export declare function chain(values: Record<string, unknown>, opts: ChainOptions): Promise<ChainResult>;

package/dist/chain/chain.js

@@ -0,0 +1,58 @@
+import { edge, end, graph } from "@spendgraph/graph";
+import { runGraph } from "../stream/graph.js";
+import { gateName, gateNode } from "./gate.js";
+import { stepNode } from "./step.js";
+function assertNamed(steps) {
+    if (steps.length === 0)
+        throw new Error("A chain needs at least one step.");
+    const seen = new Set();
+    for (const step of steps) {
+        if (seen.has(step.name))
+            throw new Error(`Two steps are called "${step.name}".`);
+        seen.add(step.name);
+    }
+}
+/** Every node in order: a stage, then its gate when it has one. */
+function nodesFor(steps) {
+    const nodes = [];
+    const order = [];
+    let previous = null;
+    for (const step of steps) {
+        nodes.push(stepNode(step, previous));
+        order.push(step.name);
+        if (step.gate) {
+            nodes.push(gateNode(step));
+            order.push(gateName(step.name));
+        }
+        previous = step.name;
+    }
+    return { nodes, order };
+}
+/**
+ * Steps in sequence, each on the last one's output, with a gate between.
+ *
+ * Compiled to a graph, which already stops at the node that failed and records
+ * every node that ran before it — which is the whole reason to gate a chain
+ * rather than let it run to the end and hand back something plausible.
+ */
+export async function chain(values, opts) {
+    assertNamed(opts.steps);
+    const { nodes, order } = nodesFor(opts.steps);
+    const flow = graph({
+        entry: order[0],
+        maxSteps: opts.maxSteps,
+        nodes,
+        edges: [
+            ...order.slice(0, -1).map((from, at) => edge(from, order[at + 1])),
+            end(order.at(-1)),
+        ],
+    }, { now: opts.now });
+    const result = await runGraph(flow, values, opts.emit);
+    const stopped = result.status === "failed" ? result.steps.at(-1) : undefined;
+    return {
+        ...result,
+        ran: result.steps.filter((s) => !s.source.startsWith("gate_")).map((s) => s.source),
+        stoppedAt: stopped ? stopped.source.replace(/^gate_/, "") : null,
+        reason: stopped?.error,
+    };
+}

package/dist/chain/gate.d.ts

@@ -0,0 +1,16 @@
+import type { Step } from "./types.js";
+/** The gate node's name for a stage, and the key its verdict is stored under. */
+export declare function gateName(step: string): string;
+export declare class GateRefused extends Error {
+    readonly step: string;
+    readonly reason: string;
+    constructor(step: string, reason: string);
+}
+/**
+ * A gate as its own node, so a refusal is a step of its own.
+ *
+ * Folding the check into the stage would lose the output it refused — and the
+ * rejected output is the evidence for why the gate fired at all. This way the
+ * stage records what it produced and the gate records what was wrong with it.
+ */
+export declare function gateNode(step: Step): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>;