@spendgraph/harness 0.1.0

package/dist/core/client.d.ts

@@ -0,0 +1,33 @@
+export interface ClientOptions {
+    /** spendgraph API key (sg_…). Without it every call is refused locally. */
+    apiKey: string | undefined;
+    /** Base URL of your deployed spendgraph app. */
+    baseUrl: string;
+    /** Injected for tests and for runtimes with a non-global fetch. */
+    fetch?: typeof fetch;
+    /** Injected so tests do not actually wait out a backoff. */
+    sleep?: (ms: number) => Promise<void>;
+    /** Total attempts including the first. Default 3. */
+    attempts?: number;
+    /** Never wait longer than this for one backoff. Default 5000. */
+    maxWaitMs?: number;
+}
+/**
+ * The one thing that talks to spendgraph.
+ *
+ * Deliberately the only module in this package that knows a network exists —
+ * everything else is given data. That is what makes the rest of it testable
+ * with no server, and what keeps the boundary rule honest: this package speaks
+ * HTTP and never touches a database.
+ */
+export declare class Client {
+    private readonly apiKey;
+    private readonly baseUrl;
+    private readonly doFetch;
+    private readonly sleep;
+    private readonly attempts;
+    private readonly maxWaitMs;
+    constructor(opts: ClientOptions);
+    request<T>(path: string, init?: RequestInit): Promise<T>;
+    private toError;
+}

package/dist/core/client.js

@@ -0,0 +1,98 @@
+import { HarnessError } from "./errors.js";
+const DEFAULT_ATTEMPTS = 3;
+const DEFAULT_MAX_WAIT_MS = 5_000;
+/**
+ * `retry-after` in milliseconds, or null when the server did not say.
+ *
+ * Sent as whole seconds per the HTTP spec; spendgraph's rate limiter always
+ * sets it alongside a 429, and honouring it beats guessing — a backoff shorter
+ * than the window just burns another attempt.
+ */
+function retryAfterMs(res) {
+    const raw = res.headers.get("retry-after");
+    if (!raw)
+        return null;
+    const seconds = Number(raw);
+    return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : null;
+}
+/**
+ * The one thing that talks to spendgraph.
+ *
+ * Deliberately the only module in this package that knows a network exists —
+ * everything else is given data. That is what makes the rest of it testable
+ * with no server, and what keeps the boundary rule honest: this package speaks
+ * HTTP and never touches a database.
+ */
+export class Client {
+    apiKey;
+    baseUrl;
+    doFetch;
+    sleep;
+    attempts;
+    maxWaitMs;
+    constructor(opts) {
+        this.apiKey = opts.apiKey;
+        this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
+        this.doFetch = opts.fetch ?? globalThis.fetch;
+        this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+        this.attempts = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS);
+        this.maxWaitMs = opts.maxWaitMs ?? DEFAULT_MAX_WAIT_MS;
+    }
+    async request(path, init = {}) {
+        if (!this.apiKey) {
+            // Refused here rather than sent and rejected: a missing key is a
+            // configuration mistake, and a 401 from the server describes it worse.
+            throw new HarnessError(401, "no_api_key", "No spendgraph API key configured.");
+        }
+        const url = `${this.baseUrl}${path.startsWith("/") ? path : `/${path}`}`;
+        let last;
+        for (let attempt = 1; attempt <= this.attempts; attempt++) {
+            let res;
+            try {
+                res = await this.doFetch(url, {
+                    ...init,
+                    headers: {
+                        "x-api-key": this.apiKey,
+                        "content-type": "application/json",
+                        ...init.headers,
+                    },
+                });
+            }
+            catch (err) {
+                // A dropped connection is worth another attempt; a bad request is not.
+                last = new HarnessError(0, "network_error", err.message, { attempts: attempt });
+                if (attempt < this.attempts) {
+                    await this.sleep(Math.min(this.maxWaitMs, 250 * 2 ** (attempt - 1)));
+                    continue;
+                }
+                throw last;
+            }
+            if (res.ok)
+                return (await res.json());
+            last = await this.toError(res, attempt);
+            if (!last.retryable || attempt === this.attempts)
+                throw last;
+            // Honour the server's own number when it sent one; otherwise back off
+            // exponentially from 250ms.
+            const wait = retryAfterMs(res) ?? 250 * 2 ** (attempt - 1);
+            await this.sleep(Math.min(this.maxWaitMs, wait));
+        }
+        // Unreachable: the loop either returns or throws.
+        throw last ?? new HarnessError(0, "unknown", "Request failed.");
+    }
+    async toError(res, attempt) {
+        let code = "http_error";
+        let message = `${res.status} ${res.statusText}`.trim();
+        try {
+            const body = (await res.json());
+            if (body?.error?.code)
+                code = body.error.code;
+            if (body?.error?.message)
+                message = body.error.message;
+        }
+        catch {
+            // A non-JSON body (a proxy's HTML error page) is not worth failing over.
+        }
+        return new HarnessError(res.status, code, message, { attempts: attempt });
+    }
+}

package/dist/core/errors.d.ts

@@ -0,0 +1,22 @@
+/**
+ * One error type for everything the API can refuse, carrying the status.
+ *
+ * A caller retrying blind is the failure this prevents: 429 is worth waiting
+ * out, 401 never will be, and a message alone cannot tell them apart.
+ */
+export declare class HarnessError extends Error {
+    readonly status: number;
+    /** The API's machine-readable code, when it sent one. */
+    readonly code: string;
+    /** How many attempts were made, so a log line says "gave up after 4". */
+    readonly attempts: number;
+    constructor(status: number, code: string, message: string, opts?: {
+        attempts?: number;
+    });
+    /**
+     * Worth trying again: rate limits, server faults, and a connection that never
+     * got far enough to have a status. Never a bad request — retrying a 401 or a
+     * 422 just spends the same attempt budget on the same answer.
+     */
+    get retryable(): boolean;
+}

package/dist/core/errors.js

@@ -0,0 +1,28 @@
+/**
+ * One error type for everything the API can refuse, carrying the status.
+ *
+ * A caller retrying blind is the failure this prevents: 429 is worth waiting
+ * out, 401 never will be, and a message alone cannot tell them apart.
+ */
+export class HarnessError extends Error {
+    status;
+    /** The API's machine-readable code, when it sent one. */
+    code;
+    /** How many attempts were made, so a log line says "gave up after 4". */
+    attempts;
+    constructor(status, code, message, opts = {}) {
+        super(message);
+        this.name = "HarnessError";
+        this.status = status;
+        this.code = code;
+        this.attempts = opts.attempts ?? 1;
+    }
+    /**
+     * Worth trying again: rate limits, server faults, and a connection that never
+     * got far enough to have a status. Never a bad request — retrying a 401 or a
+     * 422 just spends the same attempt budget on the same answer.
+     */
+    get retryable() {
+        return this.status === 0 || this.status === 429 || this.status >= 500;
+    }
+}

package/dist/core/index.d.ts

@@ -0,0 +1,5 @@
+export { HarnessError } from "./errors.js";
+export { Budget, BudgetExceededError, resetBudgetWarning } from "./budget.js";
+export { Client, type ClientOptions } from "./client.js";
+export { PullCache, type CacheHit, type CacheOptions } from "./cache.js";
+export { pullWithCache } from "./pull.js";

package/dist/core/index.js

@@ -0,0 +1,5 @@
+export { HarnessError } from "./errors.js";
+export { Budget, BudgetExceededError, resetBudgetWarning } from "./budget.js";
+export { Client } from "./client.js";
+export { PullCache } from "./cache.js";
+export { pullWithCache } from "./pull.js";

package/dist/core/pull.d.ts

@@ -0,0 +1,17 @@
+import type { PullCache } from "./cache.js";
+/**
+ * Fetch-by-id with stale-while-revalidate.
+ *
+ * Generic over the payload because tools and graphs will be pulled the same
+ * way: the mechanic is identical, only the type differs.
+ *
+ * The three paths, and why the middle one exists:
+ *   fresh  → return it, no request
+ *   stale  → return it NOW, refresh behind the caller
+ *   miss   → fetch, and only then answer
+ *
+ * Serving stale is the difference between a prompt edit taking effect within a
+ * TTL and every request unlucky enough to arrive after expiry paying for a round
+ * trip. A production request path cannot afford the second.
+ */
+export declare function pullWithCache<T>(cache: PullCache<T>, key: string, fetchOne: () => Promise<T>): Promise<T>;

package/dist/core/pull.js

@@ -0,0 +1,38 @@
+/**
+ * Fetch-by-id with stale-while-revalidate.
+ *
+ * Generic over the payload because tools and graphs will be pulled the same
+ * way: the mechanic is identical, only the type differs.
+ *
+ * The three paths, and why the middle one exists:
+ *   fresh  → return it, no request
+ *   stale  → return it NOW, refresh behind the caller
+ *   miss   → fetch, and only then answer
+ *
+ * Serving stale is the difference between a prompt edit taking effect within a
+ * TTL and every request unlucky enough to arrive after expiry paying for a round
+ * trip. A production request path cannot afford the second.
+ */
+export async function pullWithCache(cache, key, fetchOne) {
+    const hit = cache.get(key);
+    if (hit && !hit.stale)
+        return hit.value;
+    if (hit) {
+        cache.refreshInBackground(key, fetchOne);
+        return hit.value;
+    }
+    // Guarded like the background refresh, and for the same reason: an
+    // invalidation can land while this fetch is in flight, and storing the result
+    // afterwards would mark pre-invalidation state fresh for a whole TTL. This is
+    // the path a cold isolate takes, so it is the common one, not the rare one.
+    const startedAt = cache.beginFetch();
+    try {
+        const value = await fetchOne();
+        cache.settle(key, value, startedAt);
+        return value;
+    }
+    catch (err) {
+        cache.abandon(key);
+        throw err;
+    }
+}

package/dist/core/schema/serialize.d.ts

@@ -0,0 +1,12 @@
+import type { FieldSpec } from "./types.js";
+/**
+ * Validated values as the flat string map the renderer substitutes.
+ *
+ * Throws rather than rendering something approximate. Headless there is no
+ * human reading the reply, so a missing required field would otherwise buy a
+ * paid-for response with a hole in it that nothing downstream can detect.
+ *
+ * An optional field left unset is omitted, so its `{name}` stays visible — the
+ * same behaviour the prompt builder has always had.
+ */
+export declare function serializeFields(values: Record<string, unknown>, spec: FieldSpec[]): Record<string, string>;

package/dist/core/schema/serialize.js

@@ -0,0 +1,42 @@
+import { FieldValidationError, isMissing, toBoolean, toNumber, validateFields, } from "./validate.js";
+/** One declared value as the text that replaces its placeholder. */
+function renderOne(field, value) {
+    switch (field.type) {
+        case "number":
+            return String(toNumber(value));
+        case "boolean":
+            return toBoolean(value) ? (field.trueText ?? "true") : (field.falseText ?? "false");
+        case "list":
+            return value.map(String).join(field.separator ?? ", ");
+        case "json":
+            return JSON.stringify(value, null, 2);
+        default:
+            return String(value);
+    }
+}
+/**
+ * Validated values as the flat string map the renderer substitutes.
+ *
+ * Throws rather than rendering something approximate. Headless there is no
+ * human reading the reply, so a missing required field would otherwise buy a
+ * paid-for response with a hole in it that nothing downstream can detect.
+ *
+ * An optional field left unset is omitted, so its `{name}` stays visible — the
+ * same behaviour the prompt builder has always had.
+ */
+export function serializeFields(values, spec) {
+    const errors = validateFields(values, spec);
+    if (errors.length)
+        throw new FieldValidationError(errors);
+    const out = {};
+    for (const field of spec) {
+        const value = values[field.name];
+        if (isMissing(value)) {
+            if (field.default !== undefined)
+                out[field.name] = field.default;
+            continue;
+        }
+        out[field.name] = renderOne(field, value);
+    }
+    return out;
+}

package/dist/core/schema/types.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Typed prompt fields — what a `{placeholder}` accepts, and how it renders.
+ *
+ * Deliberately a copy of `lib/fields.ts` rather than an import. This package is
+ * published and depends on nothing; reaching into the app for a type would make
+ * every installed copy break the next time the app moved a file. The two are
+ * kept in step by the wire format, which is the only contract that matters.
+ *
+ * A field type is two things, and the second is the one that gets forgotten: an
+ * input contract, and a rendering rule. Everything ends up interpolated into a
+ * string, so `{max_words}` given `50` has to become `"50"` and `{tags}` given
+ * `["a","b"]` has to become something the prompt author chose rather than
+ * whatever the first implementation happened to do.
+ *
+ * Deliberately not called `Variable`: a tool's arguments and an agent node's
+ * inputs are the same problem, and naming this after prompts guarantees it gets
+ * written twice more with three different meanings for `required`.
+ */
+export type FieldType = "string" | "text" | "number" | "boolean" | "enum" | "list" | "json";
+export interface FieldSpec {
+    /** Matches the `{placeholder}` in the blocks or question. */
+    name: string;
+    type: FieldType;
+    /** A missing required field throws before anything is sent. */
+    required: boolean;
+    /**
+     * Substituted verbatim when the field is absent — already-rendered text, not
+     * a value to coerce. A `list` default is the joined string, a `json` default
+     * is the printed JSON. Authored once by a human rather than typed per run, so
+     * asking for it in final form is simpler than parsing it back per type.
+     */
+    default?: string;
+    /**
+     * What the field means. Shown in the UI, and given to the optimizer's
+     * reflector — knowing what a field is *for* is the difference between a
+     * useful prompt mutation and a guess.
+     */
+    description?: string;
+    /** `enum` — the permitted values. */
+    options?: string[];
+    /** `list` — what joins the items. Defaults to ", ". */
+    separator?: string;
+    /** `number` — inclusive bounds. */
+    min?: number;
+    max?: number;
+    /** `string` | `text` — rejected above this length. */
+    maxLength?: number;
+    /** `boolean` — words to render instead of "true"/"false". */
+    trueText?: string;
+    falseText?: string;
+    /**
+     * This is the field a dataset varies.
+     *
+     * A prompt declaring `{question}` plus 200 stored values for it *is* a
+     * 200-case dataset; this flag is what says which field those values fill.
+     */
+    datasetKey?: boolean;
+}

package/dist/core/schema/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/schema/validate.d.ts

@@ -0,0 +1,28 @@
+import type { FieldSpec } from "./types.js";
+export type FieldError = {
+    field: string;
+    message: string;
+};
+export declare class FieldValidationError extends Error {
+    readonly errors: FieldError[];
+    constructor(errors: FieldError[]);
+}
+/** Absent means absent. An empty string is a value someone chose to send. */
+export declare function isMissing(value: unknown): boolean;
+/**
+ * Coercions shared with the renderer.
+ *
+ * Shared deliberately: if the checker and the renderer disagreed, a value could
+ * pass validation and then render as something else — a request you paid for
+ * and cannot explain.
+ */
+export declare function toNumber(value: unknown): number;
+export declare function toBoolean(value: unknown): boolean;
+/**
+ * Everything wrong with these values, rather than the first thing.
+ *
+ * Values with no matching field are ignored, not reported: a stored prompt keeps
+ * values for placeholders that were later edited out, and treating a stale key
+ * as an error would fail a prompt that renders perfectly well.
+ */
+export declare function validateFields(values: Record<string, unknown>, spec: FieldSpec[]): FieldError[];

package/dist/core/schema/validate.js

@@ -0,0 +1,98 @@
+export class FieldValidationError extends Error {
+    errors;
+    constructor(errors) {
+        super(errors.map((e) => `${e.field}: ${e.message}`).join("; "));
+        this.name = "FieldValidationError";
+        this.errors = errors;
+    }
+}
+/** Absent means absent. An empty string is a value someone chose to send. */
+export function isMissing(value) {
+    return value === undefined || value === null;
+}
+/**
+ * Coercions shared with the renderer.
+ *
+ * Shared deliberately: if the checker and the renderer disagreed, a value could
+ * pass validation and then render as something else — a request you paid for
+ * and cannot explain.
+ */
+export function toNumber(value) {
+    return typeof value === "number" ? value : Number(String(value).trim());
+}
+export function toBoolean(value) {
+    return typeof value === "boolean" ? value : String(value).toLowerCase() === "true";
+}
+function checkOne(field, value) {
+    switch (field.type) {
+        case "string":
+        case "text": {
+            const s = String(value);
+            return field.maxLength !== undefined && s.length > field.maxLength
+                ? `is ${s.length} characters, over the limit of ${field.maxLength}`
+                : null;
+        }
+        case "number": {
+            const n = toNumber(value);
+            if (!Number.isFinite(n))
+                return `expects a number, got ${JSON.stringify(value)}`;
+            if (field.min !== undefined && n < field.min)
+                return `is ${n}, below the minimum of ${field.min}`;
+            if (field.max !== undefined && n > field.max)
+                return `is ${n}, above the maximum of ${field.max}`;
+            return null;
+        }
+        case "boolean": {
+            if (typeof value === "boolean")
+                return null;
+            const s = String(value).toLowerCase();
+            return s === "true" || s === "false"
+                ? null
+                : `expects true or false, got ${JSON.stringify(value)}`;
+        }
+        case "enum": {
+            const options = field.options ?? [];
+            if (options.length === 0)
+                return "is an enum with no options declared";
+            return options.includes(String(value))
+                ? null
+                : `must be one of ${options.join(" | ")}, got ${JSON.stringify(value)}`;
+        }
+        case "list":
+            return Array.isArray(value) ? null : `expects an array, got ${typeof value}`;
+        case "json":
+            try {
+                // Catches cycles and BigInt, which otherwise reach the provider as a
+                // crash — at which point the request has already been paid for.
+                JSON.stringify(value);
+                return null;
+            }
+            catch (err) {
+                return `is not serialisable: ${err.message}`;
+            }
+    }
+}
+/**
+ * Everything wrong with these values, rather than the first thing.
+ *
+ * Values with no matching field are ignored, not reported: a stored prompt keeps
+ * values for placeholders that were later edited out, and treating a stale key
+ * as an error would fail a prompt that renders perfectly well.
+ */
+export function validateFields(values, spec) {
+    const errors = [];
+    for (const field of spec) {
+        const value = values[field.name];
+        if (isMissing(value)) {
+            if (field.default !== undefined)
+                continue;
+            if (field.required)
+                errors.push({ field: field.name, message: "is required" });
+            continue; // optional and unset — the placeholder stays visible
+        }
+        const problem = checkOne(field, value);
+        if (problem)
+            errors.push({ field: field.name, message: problem });
+    }
+    return errors;
+}

package/dist/harness.d.ts

@@ -0,0 +1,128 @@
+import { type ClientOptions } from "./core/client.js";
+import { type CacheOptions } from "./core/cache.js";
+import type { DatasetCase, DatasetSummary, Prompt, ReportInput, PromptVersion, RunOptions, RunResult, ValuesFor } from "./prompt/types.js";
+export interface HarnessOptions extends Omit<ClientOptions, "apiKey" | "baseUrl"> {
+    /** spendgraph API key (sg_…). Without it, `pull` throws and `report` no-ops. */
+    apiKey: string | undefined;
+    baseUrl: string;
+    cache?: CacheOptions;
+    /** Max requests in flight for sample() and runAll(). Default 4. */
+    concurrency?: number;
+    /**
+     * Stop spending past this, in micro-dollars. Omit and the client warns once
+     * that it is unbounded; pass `null` to say you meant it.
+     */
+    maxCostMicros?: number | null;
+    /**
+     * Called when a `report` is dropped. Reporting never throws, so without this
+     * a misconfigured key is invisible until somebody notices the empty table.
+     */
+    onReportError?: (err: unknown) => void;
+    /**
+     * Called when a background prompt refresh fails.
+     *
+     * The case worth wiring: renaming a prompt mints a new slug, so a service
+     * pulling by the old one keeps serving the wording it last saw. That is the
+     * safe behaviour, but it should not be a silent one.
+     */
+    onPullError?: (err: unknown, handle: string) => void;
+}
+export declare class Harness {
+    private readonly client;
+    private readonly cache;
+    private readonly hasKey;
+    private readonly onReportError?;
+    private readonly concurrency;
+    private readonly budget;
+    constructor(opts: HarnessOptions);
+    /**
+     * A prompt, from cache when one is warm.
+     *
+     * Serves a stale copy immediately and refreshes behind the caller, so an edit
+     * takes effect within a TTL without any request ever paying for the fetch.
+     *
+     * Takes an id or a slug — the server resolves either. Cached under whatever
+     * you passed, so pulling the same prompt both ways keeps two entries.
+     *
+     * A slug is only as stable as the prompt's name: renaming it server-side
+     * mints a new one and the old stops resolving, at which point the background
+     * refresh 404s and this keeps serving the wording it last saw. Pull by id
+     * where a handle has to outlive an edit to the name.
+     */
+    pull(promptId: string): Promise<Prompt>;
+    /**
+     * Runs the prompt server-side and records the rollout.
+     *
+     * The batch path. A production request should pull, render and report
+     * instead — this adds a hop and makes spendgraph a dependency of the caller's
+     * uptime, which is the right trade for evaluation and the wrong one for a
+     * user waiting on a response.
+     */
+    run<K extends string>(promptId: K, values?: ValuesFor<K>, opts?: RunOptions): Promise<RunResult>;
+    /** k repetitions of one case. Always k results, failures included. */
+    sample<K extends string>(promptId: K, values?: ValuesFor<K>, opts?: RunOptions & {
+        k?: number;
+        concurrency?: number;
+    }): Promise<RunResult[]>;
+    /** One rollout per model — the playground's comparison, headless. */
+    runAll<K extends string>(promptId: K, values?: ValuesFor<K>, opts?: RunOptions & {
+        models?: string[];
+        concurrency?: number;
+    }): Promise<RunResult[]>;
+    /** The dataset a prompt is evaluated against, with its split counts. */
+    cases(promptId: string): Promise<DatasetSummary>;
+    /**
+     * Replaces the whole dataset.
+     *
+     * A dataset is a set: "which cases am I evaluating against" has one answer at
+     * a time, and a merge would leave no way to remove a case or to know from the
+     * outside what the set currently is.
+     */
+    setCases(promptId: string, cases: DatasetCase[]): Promise<Omit<DatasetSummary, "cases">>;
+    /** Micro-dollars this client has spent on runs it initiated. */
+    spent(): number;
+    /** What is left of the ceiling, or Infinity when none was set. */
+    remaining(): number;
+    /** Every wording this prompt has had, newest first. */
+    versions(promptId: string, opts?: {
+        origin?: "user" | "assay";
+        limit?: number;
+    }): Promise<{
+        versions: PromptVersion[];
+        currentVersionId: string | null;
+    }>;
+    /**
+     * Serves this version from now on.
+     *
+     * Invalidates the cache: a promotion that left a warm entry in place would
+     * keep sending the old wording for up to a TTL, which is the one moment a
+     * caller is actively watching for the change.
+     */
+    promote(promptId: string, versionId: string): Promise<{
+        promoted: string;
+        previous?: string | null;
+        unchanged?: boolean;
+    }>;
+    /**
+     * Drops whatever is cached, so the next pull refetches.
+     *
+     * Both handles, not just the one passed. `pull` keys the cache on whatever
+     * string it was given, so the same prompt can sit under its uuid and its
+     * slug at once — and `promote` is only reachable with a uuid while the
+     * dashboard teaches pulling by slug. Dropping one key left the other warm
+     * for a full TTL, still rendering the wording the promotion replaced, which
+     * is the exact case this method exists to prevent.
+     */
+    invalidate(promptId?: string): void;
+    /**
+     * Records a rollout the caller executed.
+     *
+     * Never throws and never rejects. This sits beside a user-facing request that
+     * has already been answered; telemetry that can break the thing it measures
+     * is worse than no telemetry. Failures go to `onReportError` and are dropped.
+     *
+     * Returns the rollout id it used, so a caller that wants to retry the exact
+     * same record can pass it back and land on one row.
+     */
+    report(promptId: string, input: ReportInput): Promise<string>;
+}