@spendgraph/harness 0.1.0 → 0.2.0

package/dist/prompt/render.js

@@ -1,66 +0,0 @@
-/**
- * Substitutes `{name}` placeholders. An unknown one is left verbatim — silently
- * emptying it would send a prompt with a hole the caller cannot see.
- *
- * Mirrors `lib/prompt.ts` in the app, deliberately duplicated: this package is
- * published and depends on nothing. The two are kept in step by
- * `rendered` being recorded on every rollout, which makes a divergence visible
- * rather than theoretical.
- */
-export function applyVariables(text, values) {
-    if (!text || !values)
-        return text;
-    return text.replace(/\{(\w+)\}/g, (whole, name) => name in values ? values[name] : whole);
-}
-/** Placeholder names used anywhere in the prompt, in first-seen order. */
-export function findVariables(blocks, question) {
-    const seen = [];
-    const scan = (text) => {
-        for (const m of text.matchAll(/\{(\w+)\}/g)) {
-            if (!seen.includes(m[1]))
-                seen.push(m[1]);
-        }
-    };
-    // titles too — a heading can carry a placeholder as readily as a body
-    for (const b of blocks ?? []) {
-        scan(b.title ?? "");
-        scan(b.body ?? "");
-    }
-    scan(question);
-    return seen;
-}
-/**
- * Blocks in the order they were arranged.
- *
- * An empty body drops the block entirely — a heading over nothing is noise in
- * the prompt. A body with no title is emitted bare rather than under a blank
- * `##`, so an unnamed block still says what it says.
- */
-export function compileSystemPrompt(blocks, values) {
-    const out = [];
-    for (const b of blocks ?? []) {
-        const body = b.body?.trim();
-        if (!body)
-            continue;
-        const title = b.title?.trim();
-        const text = applyVariables(body, values);
-        out.push(title ? `## ${applyVariables(title, values)}\n${text}` : text);
-    }
-    return out.join("\n\n");
-}
-/**
- * Every message the request will send, in order.
- *
- * An empty system message is a different request from no system message at all,
- * so a prompt with no blocks sends one message rather than two.
- */
-export function renderMessages(blocks, question, values) {
-    const system = compileSystemPrompt(blocks, values);
-    const user = applyVariables(question, values);
-    return system
-        ? [
-            { role: "system", content: system },
-            { role: "user", content: user },
-        ]
-        : [{ role: "user", content: user }];
-}

package/dist/prompt/run.d.ts

@@ -1,38 +0,0 @@
-import type { Budget } from "../core/budget.js";
-import type { Client } from "../core/client.js";
-import type { RunOptions, RunResult } from "./types.js";
-/** Generated per attempt so a retry of the same call lands on one row. */
-export declare function newRolloutId(): string;
-/**
- * Runs `fn` over `items` with at most `limit` in flight.
- *
- * Bounded on purpose. Firing a whole dataset at once turns most of it into 429s
- * that you still wait out, so the unbounded version finishes later and flakier
- * than the bounded one.
- */
-export declare function mapLimit<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
-export declare function runOnce(client: Client, promptId: string, values: Record<string, unknown>, opts?: RunOptions, budget?: Budget): Promise<RunResult>;
-/**
- * k repetitions of one case, so `pass^k` is computable.
- *
- * Always returns k results. A seed that failed is present with
- * `status: "failed"` rather than missing: returning three results for k = 5
- * would have the reliability metric compute over three and report better
- * consistency than the run actually showed.
- *
- * A budget stop is the exception, and it throws. Money running out says nothing
- * about how reliable the prompt is, so recording those seeds as failures would
- * push a number into pass^k that is about the wallet rather than the model.
- *
- * `seed` is metadata, not a provider parameter. Anthropic exposes none and
- * OpenAI's is best-effort; it records which repetition a row is, which is all
- * pass^k needs.
- */
-export declare function sampleRuns(client: Client, promptId: string, values: Record<string, unknown>, opts?: RunOptions & {
-    k?: number;
-    concurrency?: number;
-}, budget?: Budget): Promise<RunResult[]>;
-/** One rollout per model — the playground's comparison, headless. */
-export declare function runAcrossModels(client: Client, promptId: string, values: Record<string, unknown>, models: string[], opts?: RunOptions & {
-    concurrency?: number;
-}, budget?: Budget): Promise<RunResult[]>;

package/dist/prompt/run.js

@@ -1,96 +0,0 @@
-import { BudgetExceededError } from "../core/budget.js";
-/** Generated per attempt so a retry of the same call lands on one row. */
-export function newRolloutId() {
-    return `ro_${crypto.randomUUID().replace(/-/g, "")}`;
-}
-/**
- * Runs `fn` over `items` with at most `limit` in flight.
- *
- * Bounded on purpose. Firing a whole dataset at once turns most of it into 429s
- * that you still wait out, so the unbounded version finishes later and flakier
- * than the bounded one.
- */
-export async function mapLimit(items, limit, fn) {
-    const out = new Array(items.length);
-    let next = 0;
-    const workers = Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
-        while (next < items.length) {
-            const i = next++;
-            out[i] = await fn(items[i], i);
-        }
-    });
-    await Promise.all(workers);
-    return out;
-}
-export async function runOnce(client, promptId, values, opts = {}, budget) {
-    budget?.assertAffordable();
-    const rolloutId = opts.rolloutId ?? newRolloutId();
-    const res = await client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}/run`, {
-        method: "POST",
-        body: JSON.stringify({
-            rolloutId,
-            values,
-            model: opts.model,
-            temperature: opts.temperature,
-            maxTokens: opts.maxTokens,
-            seed: opts.seed ?? 0,
-            caseId: opts.caseId,
-            candidateId: opts.candidateId,
-            parentId: opts.parentId,
-            generation: opts.generation,
-            record: opts.record ?? true,
-        }),
-    });
-    // A deduped reply costs nothing new, but its cost is already in the total from
-    // the attempt that wrote it, so counting it again would overstate the spend.
-    if (!res.deduped)
-        budget?.add(res.rollout.costMicros ?? 0);
-    return res.rollout;
-}
-/**
- * k repetitions of one case, so `pass^k` is computable.
- *
- * Always returns k results. A seed that failed is present with
- * `status: "failed"` rather than missing: returning three results for k = 5
- * would have the reliability metric compute over three and report better
- * consistency than the run actually showed.
- *
- * A budget stop is the exception, and it throws. Money running out says nothing
- * about how reliable the prompt is, so recording those seeds as failures would
- * push a number into pass^k that is about the wallet rather than the model.
- *
- * `seed` is metadata, not a provider parameter. Anthropic exposes none and
- * OpenAI's is best-effort; it records which repetition a row is, which is all
- * pass^k needs.
- */
-export async function sampleRuns(client, promptId, values, opts = {}, budget) {
-    const k = Math.max(1, opts.k ?? 5);
-    const seeds = Array.from({ length: k }, (_, i) => i);
-    return mapLimit(seeds, opts.concurrency ?? 4, async (seed) => {
-        try {
-            return await runOnce(client, promptId, values, { ...opts, seed, rolloutId: undefined }, budget);
-        }
-        catch (err) {
-            if (err instanceof BudgetExceededError)
-                throw err;
-            // A transport failure is still an outcome for this seed. Throwing would
-            // discard the k-1 results that did come back.
-            return {
-                id: null,
-                model: opts.model ?? "",
-                status: "failed",
-                output: "",
-                error: err.message,
-                costMicros: 0,
-                latencyMs: 0,
-                inputTokens: 0,
-                outputTokens: 0,
-                seed,
-            };
-        }
-    });
-}
-/** One rollout per model — the playground's comparison, headless. */
-export async function runAcrossModels(client, promptId, values, models, opts = {}, budget) {
-    return mapLimit(models, opts.concurrency ?? 4, (model) => runOnce(client, promptId, values, { ...opts, model, rolloutId: undefined }, budget));
-}

package/dist/prompt/types.d.ts

@@ -1,156 +0,0 @@
-import type { FieldSpec } from "../core/schema/types.js";
-import type { Block, Message } from "./render.js";
-/** What `GET /api/v1/prompts/:id` returns, narrowed to what this package uses. */
-export interface PromptPayload {
-    id: string;
-    name: string;
-    /** The handle this prompt can also be pulled by. Absent before the server
-     * that adds it; null on a row the backfill could not slug. */
-    slug?: string | null;
-    blocks: Block[];
-    question: string;
-    models: string[];
-    temperature: number;
-    maxTokens: number;
-    fieldSpec: FieldSpec[];
-    currentVersionId: string | null;
-}
-/** A pulled prompt: its declarations, its version, and how to render it. */
-export interface Prompt {
-    id: string;
-    name: string;
-    /** What `pull` accepts besides the id. Null when the prompt has none. */
-    slug: string | null;
-    fields: FieldSpec[];
-    /** What a rollout recorded from this prompt should reference. */
-    versionId: string | null;
-    models: string[];
-    /** Validates and substitutes, throwing on a missing required field. */
-    render(values?: Record<string, unknown>): Message[];
-    /** The serialised field map that `render` used — recorded on the rollout. */
-    serialize(values?: Record<string, unknown>): Record<string, string>;
-}
-/** What the caller hands back after running the prompt themselves. */
-export interface ReportInput {
-    /** Dedupe key. Generated if omitted; supply your own to make a retry safe. */
-    rolloutId?: string;
-    versionId?: string | null;
-    model: string;
-    fields: Record<string, string>;
-    rendered: Message[];
-    output: string;
-    status?: "completed" | "failed";
-    error?: string;
-    inputTokens?: number;
-    outputTokens?: number;
-    cacheReadTokens?: number;
-    cacheWriteTokens?: number;
-    latencyMs?: number;
-    /** Set only for evaluation rollouts. Production traffic leaves it unset. */
-    caseId?: string;
-    seed?: number;
-    candidateId?: string;
-    parentId?: string;
-    generation?: number;
-}
-export interface Rollout {
-    rolloutId: string;
-    status: "completed" | "failed";
-    output: string;
-    costMicros: number;
-    latencyMs: number;
-    inputTokens: number;
-    outputTokens: number;
-}
-/** Run configuration. Field values are the second argument, not part of this. */
-export interface RunOptions {
-    /** Overrides the prompt's first saved model. Never silently defaulted. */
-    model?: string;
-    temperature?: number;
-    maxTokens?: number;
-    /** Which repetition this is. Metadata for pass^k, not a provider parameter. */
-    seed?: number;
-    /** Links the run to a dataset case; the server resolves the split from it. */
-    caseId?: string;
-    candidateId?: string;
-    parentId?: string;
-    generation?: number;
-    /** Supply to make your own retry idempotent. Generated when omitted. */
-    rolloutId?: string;
-    /** false runs the model but writes nothing — for debugging, not datasets. */
-    record?: boolean;
-}
-/**
- * Prompt id → the values that prompt expects.
- *
- * Empty here and filled in by the file `harness codegen` writes, through
- * declaration merging. Left empty, every id falls back to a loose value map —
- * so the package works with no codegen step, and a stale generated file
- * degrades to permissive rather than wrong.
- */
-export interface Prompts {
-}
-/**
- * The values type for a known prompt id, or a loose map for anything else.
- *
- * A conditional rather than an overload: overloads would let a call with the
- * wrong values silently fall through to the permissive signature, which is
- * exactly the mistake codegen exists to catch.
- */
-export type ValuesFor<K extends string> = K extends keyof Prompts ? Prompts[K] : Record<string, unknown>;
-/** One wording a prompt has had. */
-export interface PromptVersion {
-    id: string;
-    promptId: string;
-    hash: string;
-    blocks: Block[];
-    question: string;
-    fieldSpec: FieldSpec[] | null;
-    /** "user" is a wording a person wrote; "assay" is one the optimizer proposed. */
-    origin: "user" | "assay";
-    candidateId: string | null;
-    parentVersionId: string | null;
-    /** Last time this wording was put live; null means it never has been. */
-    servedAt?: string | null;
-    /** How many stints it has had live. */
-    servedCount?: number;
-    createdAt: string;
-    isCurrent: boolean;
-}
-export type Split = "feedback" | "held_out";
-/** One input the prompt is evaluated against. */
-export interface DatasetCase {
-    caseId: string;
-    fieldValues: Record<string, string>;
-    expected?: string | null;
-    /**
-     * Omit it. The server derives the split from the case id so a case never moves
-     * between sides; supply one only when importing a set that already carries it.
-     */
-    split?: Split;
-}
-export interface DatasetSummary {
-    cases: (DatasetCase & {
-        split: Split;
-    })[];
-    counts: {
-        total: number;
-        feedback: number;
-        held_out: number;
-    };
-    /** Why this dataset cannot be optimized against yet, or null. */
-    problem: string | null;
-}
-/** A rollout as the run route returns it. `id` is null for a dry run. */
-export interface RunResult {
-    id: string | null;
-    model: string;
-    status: "completed" | "failed";
-    output: string;
-    error?: string | null;
-    costMicros: number;
-    latencyMs: number;
-    inputTokens: number;
-    outputTokens: number;
-    seed?: number;
-}