@spendgraph/harness 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -160
- package/dist/cascade/cascade.d.ts +13 -0
- package/dist/cascade/cascade.js +91 -0
- package/dist/cascade/index.d.ts +3 -0
- package/dist/cascade/index.js +2 -0
- package/dist/cascade/tier.d.ts +31 -0
- package/dist/cascade/tier.js +53 -0
- package/dist/cascade/types.d.ts +98 -0
- package/dist/chain/chain.d.ts +9 -0
- package/dist/chain/chain.js +58 -0
- package/dist/chain/gate.d.ts +16 -0
- package/dist/chain/gate.js +33 -0
- package/dist/chain/index.d.ts +3 -0
- package/dist/chain/index.js +2 -0
- package/dist/chain/step.d.ts +9 -0
- package/dist/chain/step.js +14 -0
- package/dist/chain/types.d.ts +50 -0
- package/dist/index.d.ts +16 -9
- package/dist/index.js +8 -7
- package/dist/loop/act.d.ts +21 -0
- package/dist/loop/act.js +43 -0
- package/dist/loop/compact.d.ts +49 -0
- package/dist/loop/compact.js +59 -0
- package/dist/loop/hooks.d.ts +43 -0
- package/dist/loop/hooks.js +41 -0
- package/dist/loop/index.d.ts +6 -0
- package/dist/loop/index.js +5 -0
- package/dist/loop/loop.d.ts +13 -0
- package/dist/loop/loop.js +120 -0
- package/dist/loop/turn.d.ts +6 -0
- package/dist/loop/turn.js +17 -0
- package/dist/loop/types.d.ts +140 -0
- package/dist/loop/types.js +1 -0
- package/dist/orchestrate/index.d.ts +4 -0
- package/dist/orchestrate/index.js +3 -0
- package/dist/orchestrate/orchestrate.d.ts +12 -0
- package/dist/orchestrate/orchestrate.js +68 -0
- package/dist/orchestrate/plan.d.ts +18 -0
- package/dist/orchestrate/plan.js +38 -0
- package/dist/orchestrate/types.d.ts +75 -0
- package/dist/orchestrate/types.js +1 -0
- package/dist/orchestrate/work.d.ts +13 -0
- package/dist/orchestrate/work.js +35 -0
- package/dist/parallel/index.d.ts +4 -0
- package/dist/parallel/index.js +3 -0
- package/dist/parallel/merge.d.ts +26 -0
- package/dist/parallel/merge.js +52 -0
- package/dist/parallel/parallel.d.ts +12 -0
- package/dist/parallel/parallel.js +66 -0
- package/dist/parallel/task.d.ts +13 -0
- package/dist/parallel/task.js +45 -0
- package/dist/parallel/types.d.ts +61 -0
- package/dist/parallel/types.js +1 -0
- package/dist/refine/attempt.d.ts +22 -0
- package/dist/refine/attempt.js +38 -0
- package/dist/refine/index.d.ts +4 -0
- package/dist/refine/index.js +3 -0
- package/dist/refine/judge.d.ts +15 -0
- package/dist/refine/judge.js +25 -0
- package/dist/refine/refine.d.ts +10 -0
- package/dist/refine/refine.js +67 -0
- package/dist/refine/types.d.ts +90 -0
- package/dist/refine/types.js +1 -0
- package/dist/route/classify.d.ts +52 -0
- package/dist/route/classify.js +57 -0
- package/dist/route/dispatch.d.ts +12 -0
- package/dist/route/dispatch.js +33 -0
- package/dist/route/index.d.ts +3 -0
- package/dist/route/index.js +2 -0
- package/dist/route/route.d.ts +11 -0
- package/dist/route/route.js +31 -0
- package/dist/route/types.d.ts +53 -0
- package/dist/route/types.js +1 -0
- package/dist/stream/graph.d.ts +9 -0
- package/dist/stream/graph.js +16 -0
- package/dist/stream/index.d.ts +2 -0
- package/dist/stream/index.js +1 -0
- package/dist/stream/stream.d.ts +19 -0
- package/dist/stream/stream.js +65 -0
- package/dist/stream/types.d.ts +31 -0
- package/dist/stream/types.js +1 -0
- package/docs/README.md +41 -0
- package/docs/cascade.md +165 -0
- package/docs/chain.md +134 -0
- package/docs/loop.md +318 -0
- package/docs/orchestrate.md +128 -0
- package/docs/parallel.md +139 -0
- package/docs/refine.md +206 -0
- package/docs/route.md +212 -0
- package/package.json +18 -14
- package/bin/harness.mjs +0 -76
- package/dist/core/budget.d.ts +0 -46
- package/dist/core/budget.js +0 -72
- package/dist/core/cache.d.ts +0 -119
- package/dist/core/cache.js +0 -206
- package/dist/core/client.d.ts +0 -33
- package/dist/core/client.js +0 -98
- package/dist/core/errors.d.ts +0 -22
- package/dist/core/errors.js +0 -28
- package/dist/core/index.d.ts +0 -5
- package/dist/core/index.js +0 -5
- package/dist/core/pull.d.ts +0 -17
- package/dist/core/pull.js +0 -38
- package/dist/core/schema/serialize.d.ts +0 -12
- package/dist/core/schema/serialize.js +0 -42
- package/dist/core/schema/types.d.ts +0 -58
- package/dist/core/schema/validate.d.ts +0 -28
- package/dist/core/schema/validate.js +0 -98
- package/dist/harness.d.ts +0 -128
- package/dist/harness.js +0 -187
- package/dist/prompt/codegen.d.ts +0 -37
- package/dist/prompt/codegen.js +0 -97
- package/dist/prompt/render.d.ts +0 -35
- package/dist/prompt/render.js +0 -66
- package/dist/prompt/run.d.ts +0 -38
- package/dist/prompt/run.js +0 -96
- package/dist/prompt/types.d.ts +0 -156
- /package/dist/{core/schema → cascade}/types.js +0 -0
- /package/dist/{prompt → chain}/types.js +0 -0
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Typed prompt fields — what a `{placeholder}` accepts, and how it renders.
|
|
3
|
-
*
|
|
4
|
-
* Deliberately a copy of `lib/fields.ts` rather than an import. This package is
|
|
5
|
-
* published and depends on nothing; reaching into the app for a type would make
|
|
6
|
-
* every installed copy break the next time the app moved a file. The two are
|
|
7
|
-
* kept in step by the wire format, which is the only contract that matters.
|
|
8
|
-
*
|
|
9
|
-
* A field type is two things, and the second is the one that gets forgotten: an
|
|
10
|
-
* input contract, and a rendering rule. Everything ends up interpolated into a
|
|
11
|
-
* string, so `{max_words}` given `50` has to become `"50"` and `{tags}` given
|
|
12
|
-
* `["a","b"]` has to become something the prompt author chose rather than
|
|
13
|
-
* whatever the first implementation happened to do.
|
|
14
|
-
*
|
|
15
|
-
* Deliberately not called `Variable`: a tool's arguments and an agent node's
|
|
16
|
-
* inputs are the same problem, and naming this after prompts guarantees it gets
|
|
17
|
-
* written twice more with three different meanings for `required`.
|
|
18
|
-
*/
|
|
19
|
-
export type FieldType = "string" | "text" | "number" | "boolean" | "enum" | "list" | "json";
|
|
20
|
-
export interface FieldSpec {
|
|
21
|
-
/** Matches the `{placeholder}` in the blocks or question. */
|
|
22
|
-
name: string;
|
|
23
|
-
type: FieldType;
|
|
24
|
-
/** A missing required field throws before anything is sent. */
|
|
25
|
-
required: boolean;
|
|
26
|
-
/**
|
|
27
|
-
* Substituted verbatim when the field is absent — already-rendered text, not
|
|
28
|
-
* a value to coerce. A `list` default is the joined string, a `json` default
|
|
29
|
-
* is the printed JSON. Authored once by a human rather than typed per run, so
|
|
30
|
-
* asking for it in final form is simpler than parsing it back per type.
|
|
31
|
-
*/
|
|
32
|
-
default?: string;
|
|
33
|
-
/**
|
|
34
|
-
* What the field means. Shown in the UI, and given to the optimizer's
|
|
35
|
-
* reflector — knowing what a field is *for* is the difference between a
|
|
36
|
-
* useful prompt mutation and a guess.
|
|
37
|
-
*/
|
|
38
|
-
description?: string;
|
|
39
|
-
/** `enum` — the permitted values. */
|
|
40
|
-
options?: string[];
|
|
41
|
-
/** `list` — what joins the items. Defaults to ", ". */
|
|
42
|
-
separator?: string;
|
|
43
|
-
/** `number` — inclusive bounds. */
|
|
44
|
-
min?: number;
|
|
45
|
-
max?: number;
|
|
46
|
-
/** `string` | `text` — rejected above this length. */
|
|
47
|
-
maxLength?: number;
|
|
48
|
-
/** `boolean` — words to render instead of "true"/"false". */
|
|
49
|
-
trueText?: string;
|
|
50
|
-
falseText?: string;
|
|
51
|
-
/**
|
|
52
|
-
* This is the field a dataset varies.
|
|
53
|
-
*
|
|
54
|
-
* A prompt declaring `{question}` plus 200 stored values for it *is* a
|
|
55
|
-
* 200-case dataset; this flag is what says which field those values fill.
|
|
56
|
-
*/
|
|
57
|
-
datasetKey?: boolean;
|
|
58
|
-
}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import type { FieldSpec } from "./types.js";
|
|
2
|
-
export type FieldError = {
|
|
3
|
-
field: string;
|
|
4
|
-
message: string;
|
|
5
|
-
};
|
|
6
|
-
export declare class FieldValidationError extends Error {
|
|
7
|
-
readonly errors: FieldError[];
|
|
8
|
-
constructor(errors: FieldError[]);
|
|
9
|
-
}
|
|
10
|
-
/** Absent means absent. An empty string is a value someone chose to send. */
|
|
11
|
-
export declare function isMissing(value: unknown): boolean;
|
|
12
|
-
/**
|
|
13
|
-
* Coercions shared with the renderer.
|
|
14
|
-
*
|
|
15
|
-
* Shared deliberately: if the checker and the renderer disagreed, a value could
|
|
16
|
-
* pass validation and then render as something else — a request you paid for
|
|
17
|
-
* and cannot explain.
|
|
18
|
-
*/
|
|
19
|
-
export declare function toNumber(value: unknown): number;
|
|
20
|
-
export declare function toBoolean(value: unknown): boolean;
|
|
21
|
-
/**
|
|
22
|
-
* Everything wrong with these values, rather than the first thing.
|
|
23
|
-
*
|
|
24
|
-
* Values with no matching field are ignored, not reported: a stored prompt keeps
|
|
25
|
-
* values for placeholders that were later edited out, and treating a stale key
|
|
26
|
-
* as an error would fail a prompt that renders perfectly well.
|
|
27
|
-
*/
|
|
28
|
-
export declare function validateFields(values: Record<string, unknown>, spec: FieldSpec[]): FieldError[];
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
export class FieldValidationError extends Error {
|
|
2
|
-
errors;
|
|
3
|
-
constructor(errors) {
|
|
4
|
-
super(errors.map((e) => `${e.field}: ${e.message}`).join("; "));
|
|
5
|
-
this.name = "FieldValidationError";
|
|
6
|
-
this.errors = errors;
|
|
7
|
-
}
|
|
8
|
-
}
|
|
9
|
-
/** Absent means absent. An empty string is a value someone chose to send. */
|
|
10
|
-
export function isMissing(value) {
|
|
11
|
-
return value === undefined || value === null;
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* Coercions shared with the renderer.
|
|
15
|
-
*
|
|
16
|
-
* Shared deliberately: if the checker and the renderer disagreed, a value could
|
|
17
|
-
* pass validation and then render as something else — a request you paid for
|
|
18
|
-
* and cannot explain.
|
|
19
|
-
*/
|
|
20
|
-
export function toNumber(value) {
|
|
21
|
-
return typeof value === "number" ? value : Number(String(value).trim());
|
|
22
|
-
}
|
|
23
|
-
export function toBoolean(value) {
|
|
24
|
-
return typeof value === "boolean" ? value : String(value).toLowerCase() === "true";
|
|
25
|
-
}
|
|
26
|
-
function checkOne(field, value) {
|
|
27
|
-
switch (field.type) {
|
|
28
|
-
case "string":
|
|
29
|
-
case "text": {
|
|
30
|
-
const s = String(value);
|
|
31
|
-
return field.maxLength !== undefined && s.length > field.maxLength
|
|
32
|
-
? `is ${s.length} characters, over the limit of ${field.maxLength}`
|
|
33
|
-
: null;
|
|
34
|
-
}
|
|
35
|
-
case "number": {
|
|
36
|
-
const n = toNumber(value);
|
|
37
|
-
if (!Number.isFinite(n))
|
|
38
|
-
return `expects a number, got ${JSON.stringify(value)}`;
|
|
39
|
-
if (field.min !== undefined && n < field.min)
|
|
40
|
-
return `is ${n}, below the minimum of ${field.min}`;
|
|
41
|
-
if (field.max !== undefined && n > field.max)
|
|
42
|
-
return `is ${n}, above the maximum of ${field.max}`;
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
case "boolean": {
|
|
46
|
-
if (typeof value === "boolean")
|
|
47
|
-
return null;
|
|
48
|
-
const s = String(value).toLowerCase();
|
|
49
|
-
return s === "true" || s === "false"
|
|
50
|
-
? null
|
|
51
|
-
: `expects true or false, got ${JSON.stringify(value)}`;
|
|
52
|
-
}
|
|
53
|
-
case "enum": {
|
|
54
|
-
const options = field.options ?? [];
|
|
55
|
-
if (options.length === 0)
|
|
56
|
-
return "is an enum with no options declared";
|
|
57
|
-
return options.includes(String(value))
|
|
58
|
-
? null
|
|
59
|
-
: `must be one of ${options.join(" | ")}, got ${JSON.stringify(value)}`;
|
|
60
|
-
}
|
|
61
|
-
case "list":
|
|
62
|
-
return Array.isArray(value) ? null : `expects an array, got ${typeof value}`;
|
|
63
|
-
case "json":
|
|
64
|
-
try {
|
|
65
|
-
// Catches cycles and BigInt, which otherwise reach the provider as a
|
|
66
|
-
// crash — at which point the request has already been paid for.
|
|
67
|
-
JSON.stringify(value);
|
|
68
|
-
return null;
|
|
69
|
-
}
|
|
70
|
-
catch (err) {
|
|
71
|
-
return `is not serialisable: ${err.message}`;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Everything wrong with these values, rather than the first thing.
|
|
77
|
-
*
|
|
78
|
-
* Values with no matching field are ignored, not reported: a stored prompt keeps
|
|
79
|
-
* values for placeholders that were later edited out, and treating a stale key
|
|
80
|
-
* as an error would fail a prompt that renders perfectly well.
|
|
81
|
-
*/
|
|
82
|
-
export function validateFields(values, spec) {
|
|
83
|
-
const errors = [];
|
|
84
|
-
for (const field of spec) {
|
|
85
|
-
const value = values[field.name];
|
|
86
|
-
if (isMissing(value)) {
|
|
87
|
-
if (field.default !== undefined)
|
|
88
|
-
continue;
|
|
89
|
-
if (field.required)
|
|
90
|
-
errors.push({ field: field.name, message: "is required" });
|
|
91
|
-
continue; // optional and unset — the placeholder stays visible
|
|
92
|
-
}
|
|
93
|
-
const problem = checkOne(field, value);
|
|
94
|
-
if (problem)
|
|
95
|
-
errors.push({ field: field.name, message: problem });
|
|
96
|
-
}
|
|
97
|
-
return errors;
|
|
98
|
-
}
|
package/dist/harness.d.ts
DELETED
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
import { type ClientOptions } from "./core/client.js";
|
|
2
|
-
import { type CacheOptions } from "./core/cache.js";
|
|
3
|
-
import type { DatasetCase, DatasetSummary, Prompt, ReportInput, PromptVersion, RunOptions, RunResult, ValuesFor } from "./prompt/types.js";
|
|
4
|
-
export interface HarnessOptions extends Omit<ClientOptions, "apiKey" | "baseUrl"> {
|
|
5
|
-
/** spendgraph API key (sg_…). Without it, `pull` throws and `report` no-ops. */
|
|
6
|
-
apiKey: string | undefined;
|
|
7
|
-
baseUrl: string;
|
|
8
|
-
cache?: CacheOptions;
|
|
9
|
-
/** Max requests in flight for sample() and runAll(). Default 4. */
|
|
10
|
-
concurrency?: number;
|
|
11
|
-
/**
|
|
12
|
-
* Stop spending past this, in micro-dollars. Omit and the client warns once
|
|
13
|
-
* that it is unbounded; pass `null` to say you meant it.
|
|
14
|
-
*/
|
|
15
|
-
maxCostMicros?: number | null;
|
|
16
|
-
/**
|
|
17
|
-
* Called when a `report` is dropped. Reporting never throws, so without this
|
|
18
|
-
* a misconfigured key is invisible until somebody notices the empty table.
|
|
19
|
-
*/
|
|
20
|
-
onReportError?: (err: unknown) => void;
|
|
21
|
-
/**
|
|
22
|
-
* Called when a background prompt refresh fails.
|
|
23
|
-
*
|
|
24
|
-
* The case worth wiring: renaming a prompt mints a new slug, so a service
|
|
25
|
-
* pulling by the old one keeps serving the wording it last saw. That is the
|
|
26
|
-
* safe behaviour, but it should not be a silent one.
|
|
27
|
-
*/
|
|
28
|
-
onPullError?: (err: unknown, handle: string) => void;
|
|
29
|
-
}
|
|
30
|
-
export declare class Harness {
|
|
31
|
-
private readonly client;
|
|
32
|
-
private readonly cache;
|
|
33
|
-
private readonly hasKey;
|
|
34
|
-
private readonly onReportError?;
|
|
35
|
-
private readonly concurrency;
|
|
36
|
-
private readonly budget;
|
|
37
|
-
constructor(opts: HarnessOptions);
|
|
38
|
-
/**
|
|
39
|
-
* A prompt, from cache when one is warm.
|
|
40
|
-
*
|
|
41
|
-
* Serves a stale copy immediately and refreshes behind the caller, so an edit
|
|
42
|
-
* takes effect within a TTL without any request ever paying for the fetch.
|
|
43
|
-
*
|
|
44
|
-
* Takes an id or a slug — the server resolves either. Cached under whatever
|
|
45
|
-
* you passed, so pulling the same prompt both ways keeps two entries.
|
|
46
|
-
*
|
|
47
|
-
* A slug is only as stable as the prompt's name: renaming it server-side
|
|
48
|
-
* mints a new one and the old stops resolving, at which point the background
|
|
49
|
-
* refresh 404s and this keeps serving the wording it last saw. Pull by id
|
|
50
|
-
* where a handle has to outlive an edit to the name.
|
|
51
|
-
*/
|
|
52
|
-
pull(promptId: string): Promise<Prompt>;
|
|
53
|
-
/**
|
|
54
|
-
* Runs the prompt server-side and records the rollout.
|
|
55
|
-
*
|
|
56
|
-
* The batch path. A production request should pull, render and report
|
|
57
|
-
* instead — this adds a hop and makes spendgraph a dependency of the caller's
|
|
58
|
-
* uptime, which is the right trade for evaluation and the wrong one for a
|
|
59
|
-
* user waiting on a response.
|
|
60
|
-
*/
|
|
61
|
-
run<K extends string>(promptId: K, values?: ValuesFor<K>, opts?: RunOptions): Promise<RunResult>;
|
|
62
|
-
/** k repetitions of one case. Always k results, failures included. */
|
|
63
|
-
sample<K extends string>(promptId: K, values?: ValuesFor<K>, opts?: RunOptions & {
|
|
64
|
-
k?: number;
|
|
65
|
-
concurrency?: number;
|
|
66
|
-
}): Promise<RunResult[]>;
|
|
67
|
-
/** One rollout per model — the playground's comparison, headless. */
|
|
68
|
-
runAll<K extends string>(promptId: K, values?: ValuesFor<K>, opts?: RunOptions & {
|
|
69
|
-
models?: string[];
|
|
70
|
-
concurrency?: number;
|
|
71
|
-
}): Promise<RunResult[]>;
|
|
72
|
-
/** The dataset a prompt is evaluated against, with its split counts. */
|
|
73
|
-
cases(promptId: string): Promise<DatasetSummary>;
|
|
74
|
-
/**
|
|
75
|
-
* Replaces the whole dataset.
|
|
76
|
-
*
|
|
77
|
-
* A dataset is a set: "which cases am I evaluating against" has one answer at
|
|
78
|
-
* a time, and a merge would leave no way to remove a case or to know from the
|
|
79
|
-
* outside what the set currently is.
|
|
80
|
-
*/
|
|
81
|
-
setCases(promptId: string, cases: DatasetCase[]): Promise<Omit<DatasetSummary, "cases">>;
|
|
82
|
-
/** Micro-dollars this client has spent on runs it initiated. */
|
|
83
|
-
spent(): number;
|
|
84
|
-
/** What is left of the ceiling, or Infinity when none was set. */
|
|
85
|
-
remaining(): number;
|
|
86
|
-
/** Every wording this prompt has had, newest first. */
|
|
87
|
-
versions(promptId: string, opts?: {
|
|
88
|
-
origin?: "user" | "assay";
|
|
89
|
-
limit?: number;
|
|
90
|
-
}): Promise<{
|
|
91
|
-
versions: PromptVersion[];
|
|
92
|
-
currentVersionId: string | null;
|
|
93
|
-
}>;
|
|
94
|
-
/**
|
|
95
|
-
* Serves this version from now on.
|
|
96
|
-
*
|
|
97
|
-
* Invalidates the cache: a promotion that left a warm entry in place would
|
|
98
|
-
* keep sending the old wording for up to a TTL, which is the one moment a
|
|
99
|
-
* caller is actively watching for the change.
|
|
100
|
-
*/
|
|
101
|
-
promote(promptId: string, versionId: string): Promise<{
|
|
102
|
-
promoted: string;
|
|
103
|
-
previous?: string | null;
|
|
104
|
-
unchanged?: boolean;
|
|
105
|
-
}>;
|
|
106
|
-
/**
|
|
107
|
-
* Drops whatever is cached, so the next pull refetches.
|
|
108
|
-
*
|
|
109
|
-
* Both handles, not just the one passed. `pull` keys the cache on whatever
|
|
110
|
-
* string it was given, so the same prompt can sit under its uuid and its
|
|
111
|
-
* slug at once — and `promote` is only reachable with a uuid while the
|
|
112
|
-
* dashboard teaches pulling by slug. Dropping one key left the other warm
|
|
113
|
-
* for a full TTL, still rendering the wording the promotion replaced, which
|
|
114
|
-
* is the exact case this method exists to prevent.
|
|
115
|
-
*/
|
|
116
|
-
invalidate(promptId?: string): void;
|
|
117
|
-
/**
|
|
118
|
-
* Records a rollout the caller executed.
|
|
119
|
-
*
|
|
120
|
-
* Never throws and never rejects. This sits beside a user-facing request that
|
|
121
|
-
* has already been answered; telemetry that can break the thing it measures
|
|
122
|
-
* is worse than no telemetry. Failures go to `onReportError` and are dropped.
|
|
123
|
-
*
|
|
124
|
-
* Returns the rollout id it used, so a caller that wants to retry the exact
|
|
125
|
-
* same record can pass it back and land on one row.
|
|
126
|
-
*/
|
|
127
|
-
report(promptId: string, input: ReportInput): Promise<string>;
|
|
128
|
-
}
|
package/dist/harness.js
DELETED
|
@@ -1,187 +0,0 @@
|
|
|
1
|
-
import { Client } from "./core/client.js";
|
|
2
|
-
import { Budget } from "./core/budget.js";
|
|
3
|
-
import { PullCache } from "./core/cache.js";
|
|
4
|
-
import { pullWithCache } from "./core/pull.js";
|
|
5
|
-
import { serializeFields } from "./core/schema/serialize.js";
|
|
6
|
-
import { renderMessages } from "./prompt/render.js";
|
|
7
|
-
import { newRolloutId, runAcrossModels, runOnce, sampleRuns } from "./prompt/run.js";
|
|
8
|
-
export class Harness {
|
|
9
|
-
client;
|
|
10
|
-
cache;
|
|
11
|
-
hasKey;
|
|
12
|
-
onReportError;
|
|
13
|
-
concurrency;
|
|
14
|
-
budget;
|
|
15
|
-
constructor(opts) {
|
|
16
|
-
const { cache, onReportError, onPullError, concurrency, maxCostMicros, ...clientOpts } = opts;
|
|
17
|
-
this.client = new Client(clientOpts);
|
|
18
|
-
// Spread first, then the wiring — the other order let a `cache` object
|
|
19
|
-
// carrying `onRefreshError: undefined` silently discard `onPullError`.
|
|
20
|
-
this.cache = new PullCache({
|
|
21
|
-
...cache,
|
|
22
|
-
onRefreshError: cache?.onRefreshError ?? onPullError,
|
|
23
|
-
// What a cached prompt answers to, so an invalidation by uuid also
|
|
24
|
-
// catches the slug production actually pulled by, and vice versa.
|
|
25
|
-
identify: (p) => [p.id, p.slug].filter((v) => Boolean(v)),
|
|
26
|
-
});
|
|
27
|
-
this.hasKey = Boolean(opts.apiKey);
|
|
28
|
-
this.onReportError = onReportError;
|
|
29
|
-
this.concurrency = Math.max(1, concurrency ?? 4);
|
|
30
|
-
this.budget = new Budget("maxCostMicros" in opts ? maxCostMicros : undefined);
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* A prompt, from cache when one is warm.
|
|
34
|
-
*
|
|
35
|
-
* Serves a stale copy immediately and refreshes behind the caller, so an edit
|
|
36
|
-
* takes effect within a TTL without any request ever paying for the fetch.
|
|
37
|
-
*
|
|
38
|
-
* Takes an id or a slug — the server resolves either. Cached under whatever
|
|
39
|
-
* you passed, so pulling the same prompt both ways keeps two entries.
|
|
40
|
-
*
|
|
41
|
-
* A slug is only as stable as the prompt's name: renaming it server-side
|
|
42
|
-
* mints a new one and the old stops resolving, at which point the background
|
|
43
|
-
* refresh 404s and this keeps serving the wording it last saw. Pull by id
|
|
44
|
-
* where a handle has to outlive an edit to the name.
|
|
45
|
-
*/
|
|
46
|
-
async pull(promptId) {
|
|
47
|
-
const payload = await pullWithCache(this.cache, promptId, async () => {
|
|
48
|
-
const res = await this.client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}`);
|
|
49
|
-
return res.prompt;
|
|
50
|
-
});
|
|
51
|
-
const serialize = (values = {}) => serializeFields(values, payload.fieldSpec ?? []);
|
|
52
|
-
return {
|
|
53
|
-
id: payload.id,
|
|
54
|
-
name: payload.name,
|
|
55
|
-
slug: payload.slug ?? null,
|
|
56
|
-
fields: payload.fieldSpec ?? [],
|
|
57
|
-
versionId: payload.currentVersionId ?? null,
|
|
58
|
-
models: payload.models ?? [],
|
|
59
|
-
serialize,
|
|
60
|
-
render: (values = {}) => renderMessages(payload.blocks, payload.question, serialize(values)),
|
|
61
|
-
};
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Runs the prompt server-side and records the rollout.
|
|
65
|
-
*
|
|
66
|
-
* The batch path. A production request should pull, render and report
|
|
67
|
-
* instead — this adds a hop and makes spendgraph a dependency of the caller's
|
|
68
|
-
* uptime, which is the right trade for evaluation and the wrong one for a
|
|
69
|
-
* user waiting on a response.
|
|
70
|
-
*/
|
|
71
|
-
async run(promptId, values = {}, opts = {}) {
|
|
72
|
-
return runOnce(this.client, promptId, values, opts, this.budget);
|
|
73
|
-
}
|
|
74
|
-
/** k repetitions of one case. Always k results, failures included. */
|
|
75
|
-
async sample(promptId, values = {}, opts = {}) {
|
|
76
|
-
return sampleRuns(this.client, promptId, values, { concurrency: this.concurrency, ...opts }, this.budget);
|
|
77
|
-
}
|
|
78
|
-
/** One rollout per model — the playground's comparison, headless. */
|
|
79
|
-
async runAll(promptId, values = {}, opts = {}) {
|
|
80
|
-
const models = opts.models ?? (await this.pull(promptId)).models;
|
|
81
|
-
if (models.length === 0) {
|
|
82
|
-
throw new Error("No models given and the prompt has none saved.");
|
|
83
|
-
}
|
|
84
|
-
return runAcrossModels(this.client, promptId, values, models, { concurrency: this.concurrency, ...opts }, this.budget);
|
|
85
|
-
}
|
|
86
|
-
/** The dataset a prompt is evaluated against, with its split counts. */
|
|
87
|
-
async cases(promptId) {
|
|
88
|
-
return this.client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}/cases`);
|
|
89
|
-
}
|
|
90
|
-
/**
|
|
91
|
-
* Replaces the whole dataset.
|
|
92
|
-
*
|
|
93
|
-
* A dataset is a set: "which cases am I evaluating against" has one answer at
|
|
94
|
-
* a time, and a merge would leave no way to remove a case or to know from the
|
|
95
|
-
* outside what the set currently is.
|
|
96
|
-
*/
|
|
97
|
-
async setCases(promptId, cases) {
|
|
98
|
-
return this.client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}/cases`, { method: "PUT", body: JSON.stringify({ cases }) });
|
|
99
|
-
}
|
|
100
|
-
/** Micro-dollars this client has spent on runs it initiated. */
|
|
101
|
-
spent() {
|
|
102
|
-
return this.budget.spent();
|
|
103
|
-
}
|
|
104
|
-
/** What is left of the ceiling, or Infinity when none was set. */
|
|
105
|
-
remaining() {
|
|
106
|
-
return this.budget.remaining();
|
|
107
|
-
}
|
|
108
|
-
/** Every wording this prompt has had, newest first. */
|
|
109
|
-
async versions(promptId, opts = {}) {
|
|
110
|
-
const q = new URLSearchParams();
|
|
111
|
-
if (opts.origin)
|
|
112
|
-
q.set("origin", opts.origin);
|
|
113
|
-
if (opts.limit)
|
|
114
|
-
q.set("limit", String(opts.limit));
|
|
115
|
-
const suffix = q.size ? `?${q}` : "";
|
|
116
|
-
return this.client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}/versions${suffix}`);
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Serves this version from now on.
|
|
120
|
-
*
|
|
121
|
-
* Invalidates the cache: a promotion that left a warm entry in place would
|
|
122
|
-
* keep sending the old wording for up to a TTL, which is the one moment a
|
|
123
|
-
* caller is actively watching for the change.
|
|
124
|
-
*/
|
|
125
|
-
async promote(promptId, versionId) {
|
|
126
|
-
const res = await this.client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}/versions`, {
|
|
127
|
-
method: "POST",
|
|
128
|
-
body: JSON.stringify({ versionId }),
|
|
129
|
-
});
|
|
130
|
-
this.invalidate(promptId);
|
|
131
|
-
return res;
|
|
132
|
-
}
|
|
133
|
-
/**
|
|
134
|
-
* Drops whatever is cached, so the next pull refetches.
|
|
135
|
-
*
|
|
136
|
-
* Both handles, not just the one passed. `pull` keys the cache on whatever
|
|
137
|
-
* string it was given, so the same prompt can sit under its uuid and its
|
|
138
|
-
* slug at once — and `promote` is only reachable with a uuid while the
|
|
139
|
-
* dashboard teaches pulling by slug. Dropping one key left the other warm
|
|
140
|
-
* for a full TTL, still rendering the wording the promotion replaced, which
|
|
141
|
-
* is the exact case this method exists to prevent.
|
|
142
|
-
*/
|
|
143
|
-
invalidate(promptId) {
|
|
144
|
-
if (!promptId) {
|
|
145
|
-
this.cache.clear();
|
|
146
|
-
return;
|
|
147
|
-
}
|
|
148
|
-
// Matched on the payload, not on the key handed in. Reading the entry under
|
|
149
|
-
// `promptId` only finds the sibling when that key was the cached one — and
|
|
150
|
-
// the case this exists for is the opposite: production pulled by slug, so
|
|
151
|
-
// the slug is the only key, while `promote` is reachable only with a uuid.
|
|
152
|
-
for (const key of this.cache.keysWhere((p) => p.id === promptId || p.slug === promptId)) {
|
|
153
|
-
this.cache.delete(key);
|
|
154
|
-
}
|
|
155
|
-
// Always, even with nothing stored under it: a cold isolate has a fetch in
|
|
156
|
-
// flight under this handle and no entry yet, and delete is what bumps the
|
|
157
|
-
// epoch that stops the in-flight result being stored after the promotion.
|
|
158
|
-
this.cache.delete(promptId);
|
|
159
|
-
// Recorded against the name as well, so a fetch still in flight — which
|
|
160
|
-
// carries no payload yet to say which prompt it is for — is judged when it
|
|
161
|
-
// lands rather than guessed at now. This is what replaced dropping every
|
|
162
|
-
// unrelated outstanding fetch on any promote.
|
|
163
|
-
this.cache.invalidate(promptId);
|
|
164
|
-
}
|
|
165
|
-
/**
|
|
166
|
-
* Records a rollout the caller executed.
|
|
167
|
-
*
|
|
168
|
-
* Never throws and never rejects. This sits beside a user-facing request that
|
|
169
|
-
* has already been answered; telemetry that can break the thing it measures
|
|
170
|
-
* is worse than no telemetry. Failures go to `onReportError` and are dropped.
|
|
171
|
-
*
|
|
172
|
-
* Returns the rollout id it used, so a caller that wants to retry the exact
|
|
173
|
-
* same record can pass it back and land on one row.
|
|
174
|
-
*/
|
|
175
|
-
async report(promptId, input) {
|
|
176
|
-
const rolloutId = input.rolloutId ?? newRolloutId();
|
|
177
|
-
if (!this.hasKey)
|
|
178
|
-
return rolloutId;
|
|
179
|
-
try {
|
|
180
|
-
await this.client.request(`/api/v1/prompts/${encodeURIComponent(promptId)}/rollouts`, { method: "POST", body: JSON.stringify({ ...input, rolloutId }) });
|
|
181
|
-
}
|
|
182
|
-
catch (err) {
|
|
183
|
-
this.onReportError?.(err);
|
|
184
|
-
}
|
|
185
|
-
return rolloutId;
|
|
186
|
-
}
|
|
187
|
-
}
|
package/dist/prompt/codegen.d.ts
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import type { FieldSpec } from "../core/schema/types.js";
|
|
2
|
-
/** The shape codegen needs from a prompt — a narrow slice of the payload. */
|
|
3
|
-
export interface CodegenPrompt {
|
|
4
|
-
id: string;
|
|
5
|
-
/** The other handle `pull` and `run` accept, so it needs a key too. */
|
|
6
|
-
slug?: string | null;
|
|
7
|
-
name: string;
|
|
8
|
-
fieldSpec: FieldSpec[];
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* The TypeScript type a declared field accepts.
|
|
12
|
-
*
|
|
13
|
-
* `enum` becomes a union of its options rather than `string`, which is the whole
|
|
14
|
-
* reason a caller feels the difference: a typo in a tone or a format is caught
|
|
15
|
-
* where it is written instead of at the other end of a paid request.
|
|
16
|
-
*/
|
|
17
|
-
export declare function typeOf(field: FieldSpec): string;
|
|
18
|
-
/** `{ name: type }`, or `Record<string, never>` when the prompt takes nothing. */
|
|
19
|
-
export declare function valuesTypeFor(fields: FieldSpec[], indent?: string): string;
|
|
20
|
-
export interface EmitOptions {
|
|
21
|
-
/** The package specifier being augmented. Overridable for tests and forks. */
|
|
22
|
-
module?: string;
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* A `.d.ts` that teaches TypeScript what each stored prompt expects.
|
|
26
|
-
*
|
|
27
|
-
* Declaration merging rather than a generated client: the caller keeps importing
|
|
28
|
-
* `Harness` from the package, and the generated file only sharpens the types it
|
|
29
|
-
* already has. Nothing breaks if the file is deleted or goes stale — calls fall
|
|
30
|
-
* back to `Record<string, unknown>`, and the runtime validation that always runs
|
|
31
|
-
* catches what the compiler then cannot.
|
|
32
|
-
*
|
|
33
|
-
* `export {}` at the end is load-bearing: without it the file is a script rather
|
|
34
|
-
* than a module, and `declare module` would replace the package's types instead
|
|
35
|
-
* of adding to them.
|
|
36
|
-
*/
|
|
37
|
-
export declare function emitPromptTypes(prompts: CodegenPrompt[], opts?: EmitOptions): string;
|
package/dist/prompt/codegen.js
DELETED
|
@@ -1,97 +0,0 @@
|
|
|
1
|
-
/** JSON string literal, so a quote or newline in an option cannot break the file. */
|
|
2
|
-
function lit(value) {
|
|
3
|
-
return JSON.stringify(value);
|
|
4
|
-
}
|
|
5
|
-
/**
|
|
6
|
-
* The TypeScript type a declared field accepts.
|
|
7
|
-
*
|
|
8
|
-
* `enum` becomes a union of its options rather than `string`, which is the whole
|
|
9
|
-
* reason a caller feels the difference: a typo in a tone or a format is caught
|
|
10
|
-
* where it is written instead of at the other end of a paid request.
|
|
11
|
-
*/
|
|
12
|
-
export function typeOf(field) {
|
|
13
|
-
switch (field.type) {
|
|
14
|
-
case "number":
|
|
15
|
-
return "number";
|
|
16
|
-
case "boolean":
|
|
17
|
-
return "boolean";
|
|
18
|
-
case "enum": {
|
|
19
|
-
const options = field.options ?? [];
|
|
20
|
-
// An enum with no options declared cannot be narrowed to anything useful,
|
|
21
|
-
// and `never` would make the prompt uncallable.
|
|
22
|
-
return options.length ? options.map(lit).join(" | ") : "string";
|
|
23
|
-
}
|
|
24
|
-
case "list":
|
|
25
|
-
return "string[]";
|
|
26
|
-
case "json":
|
|
27
|
-
return "unknown";
|
|
28
|
-
default:
|
|
29
|
-
return "string";
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
/** A field may be omitted when it is optional or carries a default. */
|
|
33
|
-
function isOptional(field) {
|
|
34
|
-
return !field.required || field.default !== undefined;
|
|
35
|
-
}
|
|
36
|
-
/** `{ name: type }`, or `Record<string, never>` when the prompt takes nothing. */
|
|
37
|
-
export function valuesTypeFor(fields, indent = " ") {
|
|
38
|
-
if (fields.length === 0)
|
|
39
|
-
return "Record<string, never>";
|
|
40
|
-
const lines = fields.map((f) => {
|
|
41
|
-
const doc = f.description ? `${indent} /** ${f.description.replace(/\*\//g, "*\\/")} */\n` : "";
|
|
42
|
-
const key = /^[A-Za-z_$][\w$]*$/.test(f.name) ? f.name : lit(f.name);
|
|
43
|
-
return `${doc}${indent} ${key}${isOptional(f) ? "?" : ""}: ${typeOf(f)};`;
|
|
44
|
-
});
|
|
45
|
-
return `{\n${lines.join("\n")}\n${indent}}`;
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* A `.d.ts` that teaches TypeScript what each stored prompt expects.
|
|
49
|
-
*
|
|
50
|
-
* Declaration merging rather than a generated client: the caller keeps importing
|
|
51
|
-
* `Harness` from the package, and the generated file only sharpens the types it
|
|
52
|
-
* already has. Nothing breaks if the file is deleted or goes stale — calls fall
|
|
53
|
-
* back to `Record<string, unknown>`, and the runtime validation that always runs
|
|
54
|
-
* catches what the compiler then cannot.
|
|
55
|
-
*
|
|
56
|
-
* `export {}` at the end is load-bearing: without it the file is a script rather
|
|
57
|
-
* than a module, and `declare module` would replace the package's types instead
|
|
58
|
-
* of adding to them.
|
|
59
|
-
*/
|
|
60
|
-
export function emitPromptTypes(prompts, opts = {}) {
|
|
61
|
-
// Not named `module`: the app's eslint config forbids assigning that
|
|
62
|
-
// identifier, and this package is linted under it.
|
|
63
|
-
const importFrom = opts.module ?? "@spendgraph/harness";
|
|
64
|
-
// Sorted by id so regenerating an unchanged project produces no diff.
|
|
65
|
-
const sorted = [...prompts].sort((a, b) => a.id.localeCompare(b.id));
|
|
66
|
-
const entries = sorted.map((p) => {
|
|
67
|
-
const name = p.name ? `${indentDoc(p.name)}\n` : "";
|
|
68
|
-
const values = valuesTypeFor(p.fieldSpec ?? []);
|
|
69
|
-
// A key per handle. `pull` and `run` accept an id or a slug, and the
|
|
70
|
-
// dashboard's Use-in-code tab hands out the slug — so keying on the id
|
|
71
|
-
// alone meant every snippet the product prints fell through
|
|
72
|
-
// `ValuesFor<K>` to `Record<string, unknown>` and lost the checking this
|
|
73
|
-
// file exists to provide, silently.
|
|
74
|
-
const keys = p.slug && p.slug !== p.id ? [p.id, p.slug] : [p.id];
|
|
75
|
-
return keys.map((k) => `${name} ${lit(k)}: ${values};`).join("\n");
|
|
76
|
-
});
|
|
77
|
-
const body = entries.length
|
|
78
|
-
? entries.join("\n")
|
|
79
|
-
: " // No prompts found. Run `harness codegen` again after saving one.";
|
|
80
|
-
return `// Generated by \`harness codegen\`. Do not edit.
|
|
81
|
-
//
|
|
82
|
-
// Sharpens the id and values arguments of Harness methods. Deleting this file
|
|
83
|
-
// is safe: calls fall back to a loose \`Record<string, unknown>\`, and the
|
|
84
|
-
// runtime field validation is unchanged either way.
|
|
85
|
-
|
|
86
|
-
declare module ${lit(importFrom)} {
|
|
87
|
-
interface Prompts {
|
|
88
|
-
${body}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
export {};
|
|
93
|
-
`;
|
|
94
|
-
}
|
|
95
|
-
function indentDoc(text) {
|
|
96
|
-
return ` /** ${text.replace(/\*\//g, "*\\/")} */`;
|
|
97
|
-
}
|
package/dist/prompt/render.d.ts
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
export type Block = {
|
|
2
|
-
title: string;
|
|
3
|
-
body: string;
|
|
4
|
-
};
|
|
5
|
-
export type Message = {
|
|
6
|
-
role: "system" | "user";
|
|
7
|
-
content: string;
|
|
8
|
-
};
|
|
9
|
-
/**
|
|
10
|
-
* Substitutes `{name}` placeholders. An unknown one is left verbatim — silently
|
|
11
|
-
* emptying it would send a prompt with a hole the caller cannot see.
|
|
12
|
-
*
|
|
13
|
-
* Mirrors `lib/prompt.ts` in the app, deliberately duplicated: this package is
|
|
14
|
-
* published and depends on nothing. The two are kept in step by
|
|
15
|
-
* `rendered` being recorded on every rollout, which makes a divergence visible
|
|
16
|
-
* rather than theoretical.
|
|
17
|
-
*/
|
|
18
|
-
export declare function applyVariables(text: string, values: Record<string, string> | undefined): string;
|
|
19
|
-
/** Placeholder names used anywhere in the prompt, in first-seen order. */
|
|
20
|
-
export declare function findVariables(blocks: Block[], question: string): string[];
|
|
21
|
-
/**
|
|
22
|
-
* Blocks in the order they were arranged.
|
|
23
|
-
*
|
|
24
|
-
* An empty body drops the block entirely — a heading over nothing is noise in
|
|
25
|
-
* the prompt. A body with no title is emitted bare rather than under a blank
|
|
26
|
-
* `##`, so an unnamed block still says what it says.
|
|
27
|
-
*/
|
|
28
|
-
export declare function compileSystemPrompt(blocks: Block[], values?: Record<string, string>): string;
|
|
29
|
-
/**
|
|
30
|
-
* Every message the request will send, in order.
|
|
31
|
-
*
|
|
32
|
-
* An empty system message is a different request from no system message at all,
|
|
33
|
-
* so a prompt with no blocks sends one message rather than two.
|
|
34
|
-
*/
|
|
35
|
-
export declare function renderMessages(blocks: Block[], question: string, values?: Record<string, string>): Message[];
|