@spendgraph/harness 0.1.0

package/dist/prompt/types.d.ts

@@ -0,0 +1,156 @@
+import type { FieldSpec } from "../core/schema/types.js";
+import type { Block, Message } from "./render.js";
+/** What `GET /api/v1/prompts/:id` returns, narrowed to what this package uses. */
+export interface PromptPayload {
+    id: string;
+    name: string;
+    /** The handle this prompt can also be pulled by. Absent before the server
+     * that adds it; null on a row the backfill could not slug. */
+    slug?: string | null;
+    blocks: Block[];
+    question: string;
+    models: string[];
+    temperature: number;
+    maxTokens: number;
+    fieldSpec: FieldSpec[];
+    currentVersionId: string | null;
+}
+/** A pulled prompt: its declarations, its version, and how to render it. */
+export interface Prompt {
+    id: string;
+    name: string;
+    /** What `pull` accepts besides the id. Null when the prompt has none. */
+    slug: string | null;
+    fields: FieldSpec[];
+    /** What a rollout recorded from this prompt should reference. */
+    versionId: string | null;
+    models: string[];
+    /** Validates and substitutes, throwing on a missing required field. */
+    render(values?: Record<string, unknown>): Message[];
+    /** The serialised field map that `render` used — recorded on the rollout. */
+    serialize(values?: Record<string, unknown>): Record<string, string>;
+}
+/** What the caller hands back after running the prompt themselves. */
+export interface ReportInput {
+    /** Dedupe key. Generated if omitted; supply your own to make a retry safe. */
+    rolloutId?: string;
+    versionId?: string | null;
+    model: string;
+    fields: Record<string, string>;
+    rendered: Message[];
+    output: string;
+    status?: "completed" | "failed";
+    error?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    cacheReadTokens?: number;
+    cacheWriteTokens?: number;
+    latencyMs?: number;
+    /** Set only for evaluation rollouts. Production traffic leaves it unset. */
+    caseId?: string;
+    seed?: number;
+    candidateId?: string;
+    parentId?: string;
+    generation?: number;
+}
+export interface Rollout {
+    rolloutId: string;
+    status: "completed" | "failed";
+    output: string;
+    costMicros: number;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+}
+/** Run configuration. Field values are the second argument, not part of this. */
+export interface RunOptions {
+    /** Overrides the prompt's first saved model. Never silently defaulted. */
+    model?: string;
+    temperature?: number;
+    maxTokens?: number;
+    /** Which repetition this is. Metadata for pass^k, not a provider parameter. */
+    seed?: number;
+    /** Links the run to a dataset case; the server resolves the split from it. */
+    caseId?: string;
+    candidateId?: string;
+    parentId?: string;
+    generation?: number;
+    /** Supply to make your own retry idempotent. Generated when omitted. */
+    rolloutId?: string;
+    /** false runs the model but writes nothing — for debugging, not datasets. */
+    record?: boolean;
+}
+/**
+ * Prompt id → the values that prompt expects.
+ *
+ * Empty here and filled in by the file `harness codegen` writes, through
+ * declaration merging. Left empty, every id falls back to a loose value map —
+ * so the package works with no codegen step, and a stale generated file
+ * degrades to permissive rather than wrong.
+ */
+export interface Prompts {
+}
+/**
+ * The values type for a known prompt id, or a loose map for anything else.
+ *
+ * A conditional rather than an overload: overloads would let a call with the
+ * wrong values silently fall through to the permissive signature, which is
+ * exactly the mistake codegen exists to catch.
+ */
+export type ValuesFor<K extends string> = K extends keyof Prompts ? Prompts[K] : Record<string, unknown>;
+/** One wording a prompt has had. */
+export interface PromptVersion {
+    id: string;
+    promptId: string;
+    hash: string;
+    blocks: Block[];
+    question: string;
+    fieldSpec: FieldSpec[] | null;
+    /** "user" is a wording a person wrote; "assay" is one the optimizer proposed. */
+    origin: "user" | "assay";
+    candidateId: string | null;
+    parentVersionId: string | null;
+    /** Last time this wording was put live; null means it never has been. */
+    servedAt?: string | null;
+    /** How many stints it has had live. */
+    servedCount?: number;
+    createdAt: string;
+    isCurrent: boolean;
+}
+export type Split = "feedback" | "held_out";
+/** One input the prompt is evaluated against. */
+export interface DatasetCase {
+    caseId: string;
+    fieldValues: Record<string, string>;
+    expected?: string | null;
+    /**
+     * Omit it. The server derives the split from the case id so a case never moves
+     * between sides; supply one only when importing a set that already carries it.
+     */
+    split?: Split;
+}
+export interface DatasetSummary {
+    cases: (DatasetCase & {
+        split: Split;
+    })[];
+    counts: {
+        total: number;
+        feedback: number;
+        held_out: number;
+    };
+    /** Why this dataset cannot be optimized against yet, or null. */
+    problem: string | null;
+}
+/** A rollout as the run route returns it. `id` is null for a dry run. */
+export interface RunResult {
+    id: string | null;
+    model: string;
+    status: "completed" | "failed";
+    output: string;
+    error?: string | null;
+    costMicros: number;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    seed?: number;
+}

package/dist/prompt/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@spendgraph/harness",
+  "version": "0.1.0",
+  "description": "Pull a stored prompt, render it, report what it cost. Zero dependencies.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/fnLog0/spendgraph.git",
+    "directory": "packages/harness"
+  },
+  "keywords": [
+    "llm",
+    "prompt",
+    "prompt-management",
+    "cost",
+    "evals"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./core": {
+      "types": "./dist/core/index.d.ts",
+      "import": "./dist/core/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "bin",
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "npm run build && vitest run"
+  },
+  "devDependencies": {
+    "typescript": "^5"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "bin": {
+    "harness": "bin/harness.mjs"
+  }
+}