@spendgraph/harness 0.1.0 → 0.2.1

package/dist/loop/types.js

@@ -0,0 +1 @@
+export {};

package/dist/orchestrate/index.d.ts

@@ -0,0 +1,4 @@
+export { orchestrate } from "./orchestrate.js";
+export { capped, planStep } from "./plan.js";
+export type { OrchestrateEnd, OrchestrateOptions, OrchestrateResult, Outcome, Plan, Subtask, } from "./types.js";
+export { synthesisStep, work } from "./work.js";

package/dist/orchestrate/index.js

@@ -0,0 +1,3 @@
+export { orchestrate } from "./orchestrate.js";
+export { capped, planStep } from "./plan.js";
+export { synthesisStep, work } from "./work.js";

package/dist/orchestrate/orchestrate.d.ts

@@ -0,0 +1,12 @@
+import type { OrchestrateOptions, OrchestrateResult } from "./types.js";
+/**
+ * A lead decomposes, workers do the pieces, the lead puts it back together.
+ *
+ * The workflow for work whose shape is not known until the input is read — the
+ * difference from `chain`, where the stages are named in advance.
+ *
+ * It is also the one that can spend the most: the lead decides how many calls
+ * to make, so `maxWorkers` is a cap on something a model chose rather than on
+ * something you did.
+ */
+export declare function orchestrate(values: Record<string, unknown>, opts: OrchestrateOptions): Promise<OrchestrateResult>;

package/dist/orchestrate/orchestrate.js

@@ -0,0 +1,56 @@
+import { capped, planStep } from "./plan.js";
+import { synthesisStep, work } from "./work.js";
+function tokens(steps) {
+    let inputTokens = 0;
+    let outputTokens = 0;
+    for (const step of steps) {
+        inputTokens += step.inputTokens ?? 0;
+        outputTokens += step.outputTokens ?? 0;
+    }
+    return { inputTokens, outputTokens };
+}
+export async function orchestrate(values, opts) {
+    const startedAt = Date.now();
+    const steps = [];
+    let subtasks = [];
+    let dropped = [];
+    let results = [];
+    const finish = (status, stoppedBy, output, error) => ({
+        status,
+        output,
+        error,
+        steps,
+        outputs: { subtasks, results },
+        latencyMs: Date.now() - startedAt,
+        ...tokens(steps),
+        subtasks,
+        dropped,
+        ran: results.filter((r) => r.status === "completed").map((r) => r.name),
+        failed: results.filter((r) => r.status === "failed").map((r) => r.name),
+        stoppedBy,
+    });
+    const plan = await opts.plan(values);
+    ({ subtasks, dropped } = capped(plan, opts.maxWorkers));
+    steps.push(planStep(steps.length, plan, dropped));
+    opts.onPlan?.(plan, dropped);
+    if (subtasks.length === 0) {
+        return finish("completed", "planned-nothing", plan.output ?? "");
+    }
+    const spentOnPlan = (plan.inputTokens ?? 0) + (plan.outputTokens ?? 0);
+    if (opts.maxTokens !== undefined && spentOnPlan >= opts.maxTokens) {
+        return finish("failed", "tokens", "", "The plan alone reached the token ceiling.");
+    }
+    const done = await work(subtasks, values, opts);
+    steps.push(...done.steps.map((step, at) => ({ ...step, index: steps.length + at })));
+    results = done.results;
+    if (done.status === "failed") {
+        return finish("failed", "workers-failed", "", done.error);
+    }
+    const { inputTokens, outputTokens } = tokens(steps);
+    if (opts.maxTokens !== undefined && inputTokens + outputTokens >= opts.maxTokens) {
+        return finish("failed", "tokens", "", "The workers reached the token ceiling before synthesis.");
+    }
+    const synthesis = await opts.synthesise(results.filter((r) => r.status === "completed"), values, (text, stage) => opts.emit?.(text, stage ?? "synthesis"));
+    steps.push(synthesisStep(steps.length, synthesis));
+    return finish("completed", "completed", synthesis.output);
+}

package/dist/orchestrate/plan.d.ts

@@ -0,0 +1,18 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Plan, Subtask } from "./types.js";
+/**
+ * The subtasks that will run, and the ones the cap refused.
+ *
+ * Dropping the tail rather than the head: a lead asked for the most important
+ * piece first usually gives it first, and truncating from the front would throw
+ * away exactly what it thought mattered.
+ *
+ * Duplicated names are made unique rather than rejected — a lead naming two
+ * sections "summary" is a wording problem, not a reason to lose the run.
+ */
+export declare function capped(plan: Plan, max?: number): {
+    subtasks: Subtask[];
+    dropped: Subtask[];
+};
+/** The lead's own call as a step, because deciding is not free. */
+export declare function planStep(index: number, plan: Plan, dropped: Subtask[]): RolloutStep;

package/dist/orchestrate/plan.js

@@ -0,0 +1,27 @@
+const DEFAULT_MAX_WORKERS = 5;
+export function capped(plan, max = DEFAULT_MAX_WORKERS) {
+    const limit = Math.max(1, max);
+    const seen = new Map();
+    const named = plan.subtasks.map((subtask) => {
+        const count = seen.get(subtask.name) ?? 0;
+        seen.set(subtask.name, count + 1);
+        return count === 0 ? subtask : { ...subtask, name: `${subtask.name}_${count}` };
+    });
+    return { subtasks: named.slice(0, limit), dropped: named.slice(limit) };
+}
+export function planStep(index, plan, dropped) {
+    return {
+        index,
+        source: "plan",
+        model: plan.model,
+        output: plan.output ?? JSON.stringify(plan.subtasks.map((s) => s.name)),
+        status: "completed",
+        inputTokens: plan.inputTokens,
+        outputTokens: plan.outputTokens,
+        ...(dropped.length > 0
+            ? {
+                error: `${dropped.length} subtask(s) past the worker cap were dropped: ${dropped.map((s) => s.name).join(", ")}`,
+            }
+            : {}),
+    };
+}

package/dist/orchestrate/types.d.ts

@@ -0,0 +1,75 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { TaskResult } from "../parallel/index.js";
+import type { Emit } from "../stream/index.js";
+/** One piece of work the lead invented. Anything else on it is yours. */
+export interface Subtask {
+    /** Unique in the run, and the name its step is recorded under. */
+    name: string;
+    [key: string]: unknown;
+}
+/** What the lead decided to do, and what deciding cost. */
+export interface Plan {
+    subtasks: Subtask[];
+    output?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}
+/** What a worker or the synthesis produced. */
+export interface Outcome {
+    output: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+}
+export interface OrchestrateOptions {
+    /** The lead reads the input and says what the pieces are. */
+    plan(values: Record<string, unknown>): Promise<Plan> | Plan;
+    /** One piece of work. */
+    work(subtask: Subtask, values: Record<string, unknown>, emit: Emit): Promise<Outcome> | Outcome;
+    /** The lead turns what came back into the answer. */
+    synthesise(results: TaskResult[], values: Record<string, unknown>, emit: Emit): Promise<Outcome> | Outcome;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /**
+     * Most workers to run. Default 5.
+     *
+     * The failure this exists for: a lead asked to decompose a large input
+     * invents twelve subtasks, and twelve model calls leave before anyone has
+     * decided that was reasonable. Anything past the cap is dropped and **said**,
+     * never dropped quietly.
+     */
+    maxWorkers?: number;
+    /** Most workers in flight at once. Default 4. */
+    concurrency?: number;
+    /** How many must succeed before synthesising. Defaults to all that ran. */
+    minSuccess?: number;
+    /** Checked before the workers and before the synthesis. */
+    maxTokens?: number;
+    onPlan?: (plan: Plan, dropped: Subtask[]) => void;
+    onWork?: (result: TaskResult) => void;
+}
+/** Why it ended. `planned-nothing` is the lead finding no work to do. */
+export type OrchestrateEnd = "completed" | "planned-nothing" | "workers-failed" | "tokens" | "failed";
+export interface OrchestrateResult {
+    status: "completed" | "failed";
+    output: string;
+    error?: string;
+    steps: RolloutStep[];
+    outputs: Record<string, unknown>;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    /** What the lead asked for, after the cap. */
+    subtasks: Subtask[];
+    /** What the cap refused. Empty when the lead stayed within it. */
+    dropped: Subtask[];
+    ran: string[];
+    failed: string[];
+    stoppedBy: OrchestrateEnd;
+}

package/dist/orchestrate/types.js

@@ -0,0 +1 @@
+export {};

package/dist/orchestrate/work.d.ts

@@ -0,0 +1,13 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import { type ParallelResult } from "../parallel/index.js";
+import type { OrchestrateOptions, Outcome, Subtask } from "./types.js";
+/**
+ * Every worker at once.
+ *
+ * Delegated to `parallel` rather than fanned out again here: it already caps the
+ * concurrency, already turns a thrown worker into a result so the others land,
+ * and already numbers the steps by declaration so two runs can be compared.
+ */
+export declare function work(subtasks: Subtask[], values: Record<string, unknown>, opts: OrchestrateOptions): Promise<ParallelResult>;
+/** The lead's second call as a step. */
+export declare function synthesisStep(index: number, outcome: Outcome): RolloutStep;

package/dist/orchestrate/work.js

@@ -0,0 +1,25 @@
+import { parallel } from "../parallel/index.js";
+export function work(subtasks, values, opts) {
+    return parallel(values, {
+        tasks: subtasks.map((subtask) => ({
+            name: subtask.name,
+            run: (_values, emit) => opts.work(subtask, values, emit),
+        })),
+        emit: opts.emit,
+        concurrency: opts.concurrency,
+        minSuccess: opts.minSuccess ?? subtasks.length,
+        onTask: opts.onWork,
+        merge: (results) => results,
+    });
+}
+export function synthesisStep(index, outcome) {
+    return {
+        index,
+        source: "synthesise",
+        model: outcome.model,
+        output: outcome.output,
+        status: "completed",
+        inputTokens: outcome.inputTokens,
+        outputTokens: outcome.outputTokens,
+    };
+}

package/dist/parallel/index.d.ts

@@ -0,0 +1,4 @@
+export { majority, repeat, type Vote } from "./merge.js";
+export { parallel } from "./parallel.js";
+export { runTask, taskStep } from "./task.js";
+export type { ParallelOptions, ParallelResult, Task, TaskResult } from "./types.js";

package/dist/parallel/index.js

@@ -0,0 +1,3 @@
+export { majority, repeat } from "./merge.js";
+export { parallel } from "./parallel.js";
+export { runTask, taskStep } from "./task.js";

package/dist/parallel/merge.d.ts

@@ -0,0 +1,26 @@
+import type { Task, TaskResult } from "./types.js";
+/**
+ * The same task, N times over, for voting.
+ *
+ * An odd N on purpose — an even one ties, and a tie has no answer that is not
+ * arbitrary. Ask for an even number and you get the next one up.
+ */
+export declare function repeat(name: string, times: number, run: (values: Record<string, unknown>, at: number) => Promise<unknown> | unknown): Task[];
+export interface Vote<T = unknown> {
+    value: T;
+    /** How many of the tasks agreed on it. */
+    count: number;
+    /** Of the ones that succeeded. */
+    of: number;
+}
+/**
+ * The answer most of them gave.
+ *
+ * Voting is worth the tokens where a model is right most of the time and wrong
+ * differently each time — the wrong answers scatter and the right one repeats.
+ * It buys nothing where the model is confidently wrong the same way every time,
+ * which is the failure it is easiest to mistake it for a fix for.
+ *
+ * Ties go to the first declared, so the result is the same on every run.
+ */
+export declare function majority<T = unknown>(results: TaskResult[]): Vote<T> | undefined;

package/dist/parallel/merge.js

@@ -0,0 +1,35 @@
+export function repeat(name, times, run) {
+    const odd = Math.max(1, times % 2 === 0 ? times + 1 : times);
+    return Array.from({ length: odd }, (_, at) => ({
+        name: `${name}_${at}`,
+        run: (values) => run(values, at),
+    }));
+}
+function ballot(value) {
+    if (value &&
+        typeof value === "object" &&
+        typeof value.output === "string") {
+        return value.output;
+    }
+    return typeof value === "string" ? value : JSON.stringify(value);
+}
+export function majority(results) {
+    const done = results.filter((r) => r.status === "completed");
+    if (done.length === 0)
+        return undefined;
+    const tally = new Map();
+    for (const result of done) {
+        const key = ballot(result.value);
+        const seen = tally.get(key);
+        if (seen)
+            seen.count++;
+        else
+            tally.set(key, { value: result.value, count: 1 });
+    }
+    let best = { value: done[0].value, count: 0 };
+    for (const entry of tally.values()) {
+        if (entry.count > best.count)
+            best = entry;
+    }
+    return { value: best.value, count: best.count, of: done.length };
+}

package/dist/parallel/parallel.d.ts

@@ -0,0 +1,12 @@
+import type { ParallelOptions, ParallelResult } from "./types.js";
+/**
+ * Every task at once, then one answer out of what came back.
+ *
+ * Not compiled to a graph like `route` and `chain`: `@spendgraph/graph` walks one
+ * node at a time and picks a single edge, so a fan-out is not a shape it can
+ * express. The concurrency comes from `mapLimit`, which already exists.
+ *
+ * A task that throws is a result, not an exception — the point of running five
+ * is that the other four still land.
+ */
+export declare function parallel(values: Record<string, unknown>, opts: ParallelOptions): Promise<ParallelResult>;

package/dist/parallel/parallel.js

@@ -0,0 +1,56 @@
+import { mapLimit } from "@spendgraph/prompt/internals";
+import { runTask, taskStep } from "./task.js";
+const DEFAULT_CONCURRENCY = 4;
+function totals(results) {
+    let inputTokens = 0;
+    let outputTokens = 0;
+    for (const result of results) {
+        inputTokens += result.inputTokens;
+        outputTokens += result.outputTokens;
+    }
+    return { inputTokens, outputTokens };
+}
+function stringify(value) {
+    if (value === undefined || value === null)
+        return "";
+    return typeof value === "string" ? value : JSON.stringify(value);
+}
+export async function parallel(values, opts) {
+    if (opts.tasks.length === 0)
+        throw new Error("A parallel run needs at least one task.");
+    const names = new Set();
+    for (const task of opts.tasks) {
+        if (names.has(task.name))
+            throw new Error(`Two tasks are called "${task.name}".`);
+        names.add(task.name);
+    }
+    const startedAt = Date.now();
+    const results = await mapLimit(opts.tasks, opts.concurrency ?? DEFAULT_CONCURRENCY, async (task) => {
+        const result = await runTask(task, values, opts.emit);
+        opts.onTask?.(result);
+        return result;
+    });
+    const steps = results.map((result, at) => taskStep(at, result));
+    const succeeded = results.filter((r) => r.status === "completed");
+    const needed = opts.minSuccess ?? opts.tasks.length;
+    const outputs = Object.fromEntries(results.map((r) => [r.name, r.value]));
+    const base = {
+        steps,
+        outputs,
+        latencyMs: Date.now() - startedAt,
+        ...totals(results),
+        results,
+        succeeded: succeeded.map((r) => r.name),
+        failed: results.filter((r) => r.status === "failed").map((r) => r.name),
+    };
+    if (succeeded.length < needed) {
+        return {
+            ...base,
+            status: "failed",
+            output: "",
+            error: `${succeeded.length} of ${opts.tasks.length} tasks succeeded, and ${needed} were needed.`,
+        };
+    }
+    const merged = opts.merge(succeeded, values);
+    return { ...base, status: "completed", output: stringify(merged), merged };
+}

package/dist/parallel/task.d.ts

@@ -0,0 +1,13 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+import type { Task, TaskResult } from "./types.js";
+/** Runs one task. Never throws — a failure is a result, so the others still land. */
+export declare function runTask(task: Task, values: Record<string, unknown>, emit?: Emit): Promise<TaskResult>;
+/**
+ * One task as a step.
+ *
+ * Indexed by where it was declared rather than when it landed: a fan-out that
+ * numbered its steps by completion order would record a different run every
+ * time for the same work, which makes two runs impossible to compare.
+ */
+export declare function taskStep(index: number, result: TaskResult): RolloutStep;

package/dist/parallel/task.js

@@ -0,0 +1,37 @@
+function counted(value) {
+    return value && typeof value === "object" ? value : {};
+}
+export async function runTask(task, values, emit) {
+    try {
+        const value = await task.run(values, (text, stage) => emit?.(text, stage ?? task.name));
+        const { inputTokens = 0, outputTokens = 0 } = counted(value);
+        return { name: task.name, status: "completed", value, inputTokens, outputTokens };
+    }
+    catch (err) {
+        return {
+            name: task.name,
+            status: "failed",
+            error: err instanceof Error ? err.message : String(err),
+            inputTokens: 0,
+            outputTokens: 0,
+        };
+    }
+}
+export function taskStep(index, result) {
+    const { output, model } = counted(result.value);
+    return {
+        index,
+        source: result.name,
+        model,
+        output: result.status === "completed" ? (output ?? stringify(result.value)) : "",
+        status: result.status,
+        error: result.error,
+        inputTokens: result.inputTokens,
+        outputTokens: result.outputTokens,
+    };
+}
+function stringify(value) {
+    if (value === undefined || value === null)
+        return "";
+    return typeof value === "string" ? value : JSON.stringify(value);
+}

package/dist/parallel/types.d.ts

@@ -0,0 +1,61 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Emit } from "../stream/index.js";
+/** One branch of the fan-out. */
+export interface Task {
+    /** Unique in the run, and the name its step is recorded under. */
+    name: string;
+    run(values: Record<string, unknown>, emit: Emit): Promise<unknown> | unknown;
+}
+/** What one task produced, or why it did not. */
+export interface TaskResult {
+    name: string;
+    status: "completed" | "failed";
+    value?: unknown;
+    error?: string;
+    inputTokens: number;
+    outputTokens: number;
+}
+export interface ParallelOptions {
+    tasks: Task[];
+    /**
+     * Combines what came back into the answer.
+     *
+     * Only the tasks that succeeded are passed. Sectioning usually joins them;
+     * voting usually counts them — `majority` is exported for that.
+     */
+    merge(results: TaskResult[], values: Record<string, unknown>): unknown;
+    /**
+     * Where the tokens go while this runs.
+     *
+     * Passed down to whatever makes the model call; omit it and nothing is
+     * emitted. `streamed()` supplies one.
+     */
+    emit?: Emit;
+    /** Most tasks in flight at once. Default 4. */
+    concurrency?: number;
+    /**
+     * How many must succeed before merging is worth doing. Defaults to all.
+     *
+     * Sectioning wants all of them: a missing section is a hole in the answer.
+     * Voting wants a majority, and is right to ignore the one call that timed out.
+     */
+    minSuccess?: number;
+    /** Called as each task lands, so a slow fan-out is not silent. */
+    onTask?: (result: TaskResult) => void;
+}
+export interface ParallelResult {
+    status: "completed" | "failed";
+    output: string;
+    error?: string;
+    steps: RolloutStep[];
+    outputs: Record<string, unknown>;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    /** Every task, in the order they were declared — never the order they landed. */
+    results: TaskResult[];
+    succeeded: string[];
+    failed: string[];
+    /** What `merge` returned. Undefined when too few tasks succeeded. */
+    merged?: unknown;
+}

package/dist/parallel/types.js

@@ -0,0 +1 @@
+export {};

package/dist/refine/attempt.d.ts

@@ -0,0 +1,22 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { Attempt } from "./types.js";
+/**
+ * What one attempt consumed.
+ *
+ * Both halves, because a refine loop's cost is dominated by the input it resends
+ * every round — the draft, the critique and the instruction to try again.
+ */
+export declare function tokensOf(attempt: Attempt): number;
+/**
+ * One attempt as a step.
+ *
+ * Recorded whether it was accepted or not: the rejected drafts are the evidence
+ * for whether refining paid at all, and a loop that keeps only the winner cannot
+ * say what the answer cost.
+ */
+export declare function attemptStep(index: number, round: number, attempt: Attempt): RolloutStep;
+/** Both halves of every attempt so far, for the result's totals. */
+export declare function totalTokens(steps: RolloutStep[]): {
+    inputTokens: number;
+    outputTokens: number;
+};

package/dist/refine/attempt.js

@@ -0,0 +1,24 @@
+export function tokensOf(attempt) {
+    return (attempt.inputTokens ?? 0) + (attempt.outputTokens ?? 0);
+}
+export function attemptStep(index, round, attempt) {
+    return {
+        index,
+        source: `attempt_${round}`,
+        model: attempt.model,
+        output: attempt.output,
+        status: attempt.status ?? "completed",
+        error: attempt.error,
+        inputTokens: attempt.inputTokens,
+        outputTokens: attempt.outputTokens,
+    };
+}
+export function totalTokens(steps) {
+    let inputTokens = 0;
+    let outputTokens = 0;
+    for (const step of steps) {
+        inputTokens += step.inputTokens ?? 0;
+        outputTokens += step.outputTokens ?? 0;
+    }
+    return { inputTokens, outputTokens };
+}

package/dist/refine/index.d.ts

@@ -0,0 +1,4 @@
+export { attemptStep, tokensOf } from "./attempt.js";
+export { best, judgeStep } from "./judge.js";
+export { refine } from "./refine.js";
+export type { Attempt, RefineOptions, RefineResult, RoundRecord, Verdict } from "./types.js";

package/dist/refine/index.js

@@ -0,0 +1,3 @@
+export { attemptStep, tokensOf } from "./attempt.js";
+export { best, judgeStep } from "./judge.js";
+export { refine } from "./refine.js";

package/dist/refine/judge.d.ts

@@ -0,0 +1,15 @@
+import type { RolloutStep } from "@spendgraph/sdk";
+import type { RoundRecord, Verdict } from "./types.js";
+/** One verdict as a step, so a run says why each draft was turned down. */
+export declare function judgeStep(index: number, round: number, verdict: Verdict): RolloutStep;
+/**
+ * The attempt to hand back when nothing was accepted.
+ *
+ * Highest score wins, and the last attempt wins when nothing was scored — a loop
+ * that improves each round has its best work at the end, and returning the first
+ * draft after three revisions would throw the work away.
+ *
+ * Here rather than in the loop because the choice is the judge's: it is made on
+ * the verdicts, and a judge that never scores gets the fallback.
+ */
+export declare function best(history: RoundRecord[]): RoundRecord | undefined;

package/dist/refine/judge.js

@@ -0,0 +1,14 @@
+export function judgeStep(index, round, verdict) {
+    return {
+        index,
+        source: `judge_${round}`,
+        output: JSON.stringify(verdict),
+        status: "completed",
+    };
+}
+export function best(history) {
+    const scored = history.filter((r) => typeof r.verdict.score === "number");
+    if (scored.length === 0)
+        return history.at(-1);
+    return scored.reduce((a, b) => ((b.verdict.score ?? 0) > (a.verdict.score ?? 0) ? b : a));
+}

package/dist/refine/refine.d.ts

@@ -0,0 +1,10 @@
+import type { RefineOptions, RefineResult } from "./types.js";
+/**
+ * Draft, judge, revise, until it is good enough or the budget is out.
+ *
+ * The pattern worth naming when there is a clear criterion and a first draft
+ * rarely meets it. Every attempt and every judgement is recorded, the rejected
+ * ones included: a loop that keeps only the winner cannot say what the answer
+ * cost, and the rejected drafts are the evidence for whether refining paid.
+ */
+export declare function refine(opts: RefineOptions): Promise<RefineResult>;