@spendgraph/harness 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -111
- package/dist/cascade/cascade.d.ts +13 -0
- package/dist/cascade/cascade.js +80 -0
- package/dist/cascade/index.d.ts +3 -0
- package/dist/cascade/index.js +2 -0
- package/dist/cascade/tier.d.ts +31 -0
- package/dist/cascade/tier.js +32 -0
- package/dist/cascade/types.d.ts +98 -0
- package/dist/chain/chain.d.ts +9 -0
- package/dist/chain/chain.js +50 -0
- package/dist/chain/gate.d.ts +16 -0
- package/dist/chain/gate.js +25 -0
- package/dist/chain/index.d.ts +3 -0
- package/dist/chain/index.js +2 -0
- package/dist/chain/step.d.ts +9 -0
- package/dist/chain/step.js +7 -0
- package/dist/chain/types.d.ts +50 -0
- package/dist/index.d.ts +16 -9
- package/dist/index.js +8 -7
- package/dist/loop/act.d.ts +21 -0
- package/dist/loop/act.js +29 -0
- package/dist/loop/compact.d.ts +49 -0
- package/dist/loop/compact.js +41 -0
- package/dist/loop/hooks.d.ts +43 -0
- package/dist/loop/hooks.js +31 -0
- package/dist/loop/index.d.ts +6 -0
- package/dist/loop/index.js +5 -0
- package/dist/loop/loop.d.ts +13 -0
- package/dist/loop/loop.js +99 -0
- package/dist/loop/turn.d.ts +6 -0
- package/dist/loop/turn.js +15 -0
- package/dist/loop/types.d.ts +140 -0
- package/dist/loop/types.js +1 -0
- package/dist/orchestrate/index.d.ts +4 -0
- package/dist/orchestrate/index.js +3 -0
- package/dist/orchestrate/orchestrate.d.ts +12 -0
- package/dist/orchestrate/orchestrate.js +56 -0
- package/dist/orchestrate/plan.d.ts +18 -0
- package/dist/orchestrate/plan.js +27 -0
- package/dist/orchestrate/types.d.ts +75 -0
- package/dist/orchestrate/types.js +1 -0
- package/dist/orchestrate/work.d.ts +13 -0
- package/dist/orchestrate/work.js +25 -0
- package/dist/parallel/index.d.ts +4 -0
- package/dist/parallel/index.js +3 -0
- package/dist/parallel/merge.d.ts +26 -0
- package/dist/parallel/merge.js +35 -0
- package/dist/parallel/parallel.d.ts +12 -0
- package/dist/parallel/parallel.js +56 -0
- package/dist/parallel/task.d.ts +13 -0
- package/dist/parallel/task.js +37 -0
- package/dist/parallel/types.d.ts +61 -0
- package/dist/parallel/types.js +1 -0
- package/dist/refine/attempt.d.ts +22 -0
- package/dist/refine/attempt.js +24 -0
- package/dist/refine/index.d.ts +4 -0
- package/dist/refine/index.js +3 -0
- package/dist/refine/judge.d.ts +15 -0
- package/dist/refine/judge.js +14 -0
- package/dist/refine/refine.d.ts +10 -0
- package/dist/refine/refine.js +59 -0
- package/dist/refine/types.d.ts +90 -0
- package/dist/refine/types.js +1 -0
- package/dist/route/classify.d.ts +52 -0
- package/dist/route/classify.js +33 -0
- package/dist/route/dispatch.d.ts +12 -0
- package/dist/route/dispatch.js +25 -0
- package/dist/route/index.d.ts +3 -0
- package/dist/route/index.js +2 -0
- package/dist/route/route.d.ts +11 -0
- package/dist/route/route.js +22 -0
- package/dist/route/types.d.ts +53 -0
- package/dist/route/types.js +1 -0
- package/dist/stream/graph.d.ts +9 -0
- package/dist/stream/graph.js +10 -0
- package/dist/stream/index.d.ts +2 -0
- package/dist/stream/index.js +1 -0
- package/dist/stream/stream.d.ts +19 -0
- package/dist/stream/stream.js +48 -0
- package/dist/stream/types.d.ts +31 -0
- package/dist/stream/types.js +1 -0
- package/docs/README.md +41 -0
- package/docs/cascade.md +165 -0
- package/docs/chain.md +134 -0
- package/docs/loop.md +318 -0
- package/docs/orchestrate.md +128 -0
- package/docs/parallel.md +139 -0
- package/docs/refine.md +206 -0
- package/docs/route.md +212 -0
- package/package.json +19 -15
- package/bin/harness.mjs +0 -76
- package/dist/core/budget.d.ts +0 -46
- package/dist/core/budget.js +0 -72
- package/dist/core/cache.d.ts +0 -119
- package/dist/core/cache.js +0 -206
- package/dist/core/client.d.ts +0 -33
- package/dist/core/client.js +0 -98
- package/dist/core/errors.d.ts +0 -22
- package/dist/core/errors.js +0 -28
- package/dist/core/index.d.ts +0 -5
- package/dist/core/index.js +0 -5
- package/dist/core/pull.d.ts +0 -17
- package/dist/core/pull.js +0 -38
- package/dist/core/schema/serialize.d.ts +0 -12
- package/dist/core/schema/serialize.js +0 -42
- package/dist/core/schema/types.d.ts +0 -58
- package/dist/core/schema/validate.d.ts +0 -28
- package/dist/core/schema/validate.js +0 -98
- package/dist/harness.d.ts +0 -128
- package/dist/harness.js +0 -187
- package/dist/prompt/codegen.d.ts +0 -37
- package/dist/prompt/codegen.js +0 -97
- package/dist/prompt/render.d.ts +0 -35
- package/dist/prompt/render.js +0 -66
- package/dist/prompt/run.d.ts +0 -38
- package/dist/prompt/run.js +0 -96
- package/dist/prompt/types.d.ts +0 -156
- /package/dist/{core/schema → cascade}/types.js +0 -0
- /package/dist/{prompt → chain}/types.js +0 -0
package/README.md
CHANGED
|
@@ -1,161 +1,191 @@
|
|
|
1
1
|
# @spendgraph/harness
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
**Agent = Model + Harness.** The model reasons. The harness is everything around
|
|
4
|
+
it that turns text into work — choosing what runs next, calling tools, holding
|
|
5
|
+
the result together, and knowing what to do when a step fails.
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
7
|
+
This is the *outer* harness: the part you assemble, on top of whatever agent SDK
|
|
8
|
+
your provider ships. It is built on `@spendgraph/prompt`, `@spendgraph/tools`
|
|
9
|
+
and `@spendgraph/graph`, and none of it reasons — every workflow here directs a
|
|
10
|
+
model call you supply.
|
|
8
11
|
|
|
9
12
|
```sh
|
|
10
13
|
npm install @spendgraph/harness
|
|
11
14
|
```
|
|
12
15
|
|
|
13
|
-
|
|
16
|
+
Seven shapes, all built. Each is a graph you would otherwise hand-roll, with the
|
|
17
|
+
pricing and the rollout already attached, so a run is recorded rather than
|
|
18
|
+
reconstructed afterwards.
|
|
14
19
|
|
|
15
|
-
|
|
16
|
-
|
|
20
|
+
| Workflow | What it is | When it wins |
|
|
21
|
+
| --- | --- | --- |
|
|
22
|
+
| **`chain`** | steps in sequence, each on the last's output, with a gate between | a task that decomposes cleanly and reads better as stages |
|
|
23
|
+
| **`route`** | classify first, then dispatch to a specialist | inputs that fall into kinds a cheaper model can tell apart |
|
|
24
|
+
| **`parallel`** | the same call N times, or N sections at once | voting for confidence, sectioning for latency |
|
|
25
|
+
| **`orchestrate`** | a lead decomposes and delegates to workers | the subtasks are not known until the input is read |
|
|
26
|
+
| **`refine`** | generate, critique, revise until a bar is met | there is a clear criterion and a first draft rarely meets it |
|
|
27
|
+
| **`loop`** | tools in a loop until the model stops asking | open-ended work with a tool surface and a step ceiling |
|
|
28
|
+
| **`cascade`** | try the cheap model, escalate only when it will not do | most inputs are easy and a few are not, and you can tell which |
|
|
29
|
+
|
|
30
|
+
## Two of them, in full
|
|
31
|
+
|
|
32
|
+
`refine` is generate and critique until it passes:
|
|
17
33
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
34
|
+
```ts
|
|
35
|
+
import { refine } from "@spendgraph/harness";
|
|
36
|
+
|
|
37
|
+
const result = await refine({
|
|
38
|
+
attempt: (feedback) =>
|
|
39
|
+
llm.call([{ role: "user", content: feedback ? `${brief}\n\nFix: ${feedback}` : brief }]),
|
|
40
|
+
judge: (attempt) =>
|
|
41
|
+
attempt.output.length < 900
|
|
42
|
+
? { accepted: true }
|
|
43
|
+
: { accepted: false, feedback: "Cut it to under 900 characters." },
|
|
44
|
+
rounds: 3,
|
|
45
|
+
maxTokens: 40_000,
|
|
21
46
|
});
|
|
22
47
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const messages = prompt.render(values);
|
|
28
|
-
|
|
29
|
-
// Call the provider yourself. spendgraph is never between you and the model.
|
|
30
|
-
const startedAt = Date.now();
|
|
31
|
-
const completion = await yourModelCall(messages);
|
|
32
|
-
|
|
33
|
-
await harness.report(prompt.id, {
|
|
34
|
-
versionId: prompt.versionId,
|
|
35
|
-
model: "anthropic/claude-sonnet-5",
|
|
36
|
-
fields: prompt.serialize(values),
|
|
37
|
-
rendered: messages,
|
|
38
|
-
output: completion.text,
|
|
39
|
-
inputTokens: completion.usage.inputTokens,
|
|
40
|
-
outputTokens: completion.usage.outputTokens,
|
|
41
|
-
cacheReadTokens: completion.usage.cacheReadTokens ?? 0,
|
|
42
|
-
cacheWriteTokens: completion.usage.cacheWriteTokens ?? 0,
|
|
43
|
-
latencyMs: Date.now() - startedAt,
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
return completion.text;
|
|
47
|
-
}
|
|
48
|
+
result.output; // the winning draft
|
|
49
|
+
result.accepted; // false when the rounds or the ceiling ran out first
|
|
50
|
+
result.stoppedBy; // "accepted" | "rounds" | "tokens" | "failed"
|
|
51
|
+
result.history; // every attempt, including the rejected ones
|
|
48
52
|
```
|
|
49
53
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
trip.
|
|
54
|
+
One `attempt` callback rather than separate generate and revise, because in
|
|
55
|
+
practice they are the same prompt with one extra paragraph. `feedback` is null
|
|
56
|
+
on the first round and the judge's note after it.
|
|
54
57
|
|
|
55
|
-
`
|
|
56
|
-
already been answered, and telemetry that can break the thing it measures is
|
|
57
|
-
worse than no telemetry. Pass `onReportError` if you want to know when one is
|
|
58
|
-
dropped.
|
|
58
|
+
`route` classifies, then dispatches:
|
|
59
59
|
|
|
60
|
-
|
|
60
|
+
```ts
|
|
61
|
+
import { route } from "@spendgraph/harness";
|
|
62
|
+
|
|
63
|
+
const result = await route(
|
|
64
|
+
{ subject, body },
|
|
65
|
+
{
|
|
66
|
+
classify: ({ subject }) => (/refund|charge/i.test(String(subject)) ? "billing" : "general"),
|
|
67
|
+
routes: {
|
|
68
|
+
billing: (values) => billingAgent(values),
|
|
69
|
+
general: (values) => generalAgent(values),
|
|
70
|
+
},
|
|
71
|
+
fallback: (values) => humanQueue(values),
|
|
72
|
+
}
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
result.route; // the branch that ran, or null when the fallback did
|
|
76
|
+
result.classified; // what the classifier said, whether or not it was used
|
|
77
|
+
```
|
|
61
78
|
|
|
62
|
-
`
|
|
79
|
+
**`fallback` is required.** A router that can fail to route fails in production
|
|
80
|
+
at 3am, and "unknown" is a class every classifier eventually returns. `classify`
|
|
81
|
+
is often better as plain code than a model — the regex above is free and never
|
|
82
|
+
invents a category.
|
|
63
83
|
|
|
64
|
-
|
|
65
|
-
one and the old stops resolving**. The uuid never changes. Pull by slug for
|
|
66
|
-
something readable; pull by id where the handle has to outlive an edit to the
|
|
67
|
-
name.
|
|
84
|
+
## Calling convention
|
|
68
85
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
call. Pass `onPullError` to be told, because otherwise that is silent:
|
|
86
|
+
Five take the run's values first; two have nothing to substitute into and take
|
|
87
|
+
options alone.
|
|
72
88
|
|
|
73
89
|
```ts
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
90
|
+
await chain(values, opts);
|
|
91
|
+
await route(values, opts);
|
|
92
|
+
await parallel(values, opts);
|
|
93
|
+
await orchestrate(values, opts);
|
|
94
|
+
await cascade(values, opts);
|
|
95
|
+
|
|
96
|
+
await refine(opts);
|
|
97
|
+
await loop(opts);
|
|
79
98
|
```
|
|
80
99
|
|
|
81
|
-
##
|
|
100
|
+
## What every one of them owes you
|
|
82
101
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
the network. So an edit in the dashboard reaches production within a TTL without
|
|
86
|
-
any request paying for the fetch.
|
|
102
|
+
Each result is a `GraphResult` first, so it reports as a rollout without
|
|
103
|
+
translation:
|
|
87
104
|
|
|
88
105
|
```ts
|
|
89
|
-
|
|
106
|
+
result.status; // "completed" | "failed"
|
|
107
|
+
result.output;
|
|
108
|
+
result.steps; // every step, the rejected ones included
|
|
109
|
+
result.inputTokens; // summed across the whole run
|
|
110
|
+
result.outputTokens;
|
|
111
|
+
result.stoppedBy; // never the same value as "it finished"
|
|
90
112
|
```
|
|
91
113
|
|
|
92
|
-
|
|
93
|
-
|
|
114
|
+
- Every step recorded, including the ones that were rejected — a refine loop that
|
|
115
|
+
keeps only the winner cannot say what it cost.
|
|
116
|
+
- A budget ceiling, because each of these spends more than one call and the
|
|
117
|
+
interesting ones spend an unbounded amount.
|
|
118
|
+
- A `stoppedBy`, because "it finished" and "it gave up" must never look the same.
|
|
94
119
|
|
|
95
|
-
|
|
120
|
+
Tokens are summed across **every** attempt rather than the winning one. The only
|
|
121
|
+
question a refine loop has to answer is whether refining paid, and a total that
|
|
122
|
+
counted the winner alone would always say yes.
|
|
96
123
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
124
|
+
## Watching one run
|
|
125
|
+
|
|
126
|
+
`streamed()` wraps any workflow and hands it an `emit`, so the tokens go
|
|
127
|
+
somewhere while it runs instead of arriving all at once at the end.
|
|
101
128
|
|
|
102
129
|
```ts
|
|
103
|
-
|
|
104
|
-
const k = await harness.sample(promptId, values, { k: 5 }); // pass^k
|
|
105
|
-
const across = await harness.runAll(promptId, values); // one per model
|
|
106
|
-
```
|
|
130
|
+
import { streamed } from "@spendgraph/harness";
|
|
107
131
|
|
|
108
|
-
|
|
132
|
+
const running = streamed((emit) => refine({ attempt, judge, emit }));
|
|
109
133
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
134
|
+
for await (const event of running) {
|
|
135
|
+
if (event.type === "token") res.write(event.text);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const result = await running.result;
|
|
114
139
|
```
|
|
115
140
|
|
|
116
|
-
|
|
117
|
-
you meant it.
|
|
141
|
+
## Inner and outer
|
|
118
142
|
|
|
119
|
-
|
|
143
|
+
The provider's SDK is the **inner** harness: it drives one model call, and on
|
|
144
|
+
Anthropic or OpenAI it will loop over tool calls for you. `@spendgraph/llms`
|
|
145
|
+
wraps that.
|
|
120
146
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
147
|
+
What it does not do is decide *which* call to make, what to do with the answer,
|
|
148
|
+
when to stop, or what any of it cost. That is the **outer** harness, and it is
|
|
149
|
+
the part that is written again in every project — usually inline, usually
|
|
150
|
+
without a ceiling, and usually recording nothing.
|
|
125
151
|
|
|
126
|
-
|
|
152
|
+
| | |
|
|
153
|
+
| --- | --- |
|
|
154
|
+
| Inner | one call, tool loop, streaming — `@spendgraph/llms` |
|
|
155
|
+
| Outer | which call, what next, when to stop, what it cost — here |
|
|
127
156
|
|
|
128
|
-
|
|
129
|
-
npx harness codegen --url https://your-spendgraph.example.com
|
|
130
|
-
# usage: harness codegen [--out <file>] [--project <id>] [--module <specifier>]
|
|
131
|
-
```
|
|
157
|
+
## Why here and not in `graph`
|
|
132
158
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
159
|
+
`graph` is the mechanism: nodes, edges, a run. These are the seven arrangements of
|
|
160
|
+
it worth naming, each with its own failure mode — a chain needs a gate or a bad
|
|
161
|
+
step poisons the rest, voting needs an odd N, a loop needs a ceiling and every
|
|
162
|
+
one of them needs a budget. Putting them beside the mechanism would suggest the
|
|
163
|
+
mechanism is opinionated. It is not.
|
|
138
164
|
|
|
139
|
-
##
|
|
165
|
+
## What is missing
|
|
140
166
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
167
|
+
Every workflow here is **single-run**: it starts, it finishes, and it holds
|
|
168
|
+
nothing afterwards. A harness in the fuller sense also carries state across a
|
|
169
|
+
pause — a run that waits for a person, resumes tomorrow, or picks up where a
|
|
170
|
+
crash left it.
|
|
145
171
|
|
|
146
|
-
|
|
147
|
-
|
|
172
|
+
That needs somewhere to persist a run, which is a decision about the product and
|
|
173
|
+
not a wiring pattern. It is written up in [`PLAN.md`](./PLAN.md) rather than
|
|
174
|
+
quietly folded in.
|
|
148
175
|
|
|
149
|
-
##
|
|
176
|
+
## Documentation
|
|
150
177
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
await harness.setCases(promptId, cases);
|
|
154
|
-
```
|
|
178
|
+
Each built workflow has a page with worked examples, the cases it suits, and the
|
|
179
|
+
cases it does not.
|
|
155
180
|
|
|
156
|
-
`
|
|
157
|
-
|
|
158
|
-
|
|
181
|
+
- [`docs/refine.md`](./docs/refine.md) — draft, judge, revise
|
|
182
|
+
- [`docs/route.md`](./docs/route.md) — classify, then dispatch
|
|
183
|
+
- [`docs/chain.md`](./docs/chain.md) — steps in sequence, with a gate between
|
|
184
|
+
- [`docs/parallel.md`](./docs/parallel.md) — sectioning for latency, voting for confidence
|
|
185
|
+
- [`docs/loop.md`](./docs/loop.md) — tools until the model stops asking
|
|
186
|
+
- [`docs/orchestrate.md`](./docs/orchestrate.md) — a lead decomposes and delegates
|
|
187
|
+
- [`docs/cascade.md`](./docs/cascade.md) — cheap first, escalate on rejection
|
|
188
|
+
- [`docs/README.md`](./docs) — the seven patterns, and what each owes you
|
|
159
189
|
|
|
160
190
|
## License
|
|
161
191
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { CascadeOptions, CascadeResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Try the cheap model, and escalate only when its answer will not do.
|
|
4
|
+
*
|
|
5
|
+
* Not `route`, which chooses before it has seen an answer. Not `refine`, which
|
|
6
|
+
* revises with the same model. This is the one lever that spends less by
|
|
7
|
+
* default and more only on the inputs that turn out to need it.
|
|
8
|
+
*
|
|
9
|
+
* Whether it pays depends entirely on how often the first tier is accepted, so
|
|
10
|
+
* every tier tried is recorded and its tokens counted — a total that hid the
|
|
11
|
+
* rejected attempts would report a saving on every run.
|
|
12
|
+
*/
|
|
13
|
+
export declare function cascade(values: Record<string, unknown>, opts: CascadeOptions): Promise<CascadeResult>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { attempt, fallback, tierStep, tokensOf } from "./tier.js";
|
|
2
|
+
function checkTiers(tiers) {
|
|
3
|
+
if (tiers.length === 0)
|
|
4
|
+
throw new Error("cascade needs at least one tier.");
|
|
5
|
+
const seen = new Set();
|
|
6
|
+
for (const tier of tiers) {
|
|
7
|
+
if (!tier.name.trim())
|
|
8
|
+
throw new Error("Every cascade tier needs a name.");
|
|
9
|
+
if (seen.has(tier.name))
|
|
10
|
+
throw new Error(`Two cascade tiers are called "${tier.name}".`);
|
|
11
|
+
seen.add(tier.name);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export async function cascade(values, opts) {
|
|
15
|
+
const startedAt = Date.now();
|
|
16
|
+
checkTiers(opts.tiers);
|
|
17
|
+
const steps = [];
|
|
18
|
+
const history = [];
|
|
19
|
+
let spent = 0;
|
|
20
|
+
let accepted;
|
|
21
|
+
let stoppedBy = "exhausted";
|
|
22
|
+
for (const [index, tier] of opts.tiers.entries()) {
|
|
23
|
+
if (opts.maxTokens !== undefined && spent >= opts.maxTokens) {
|
|
24
|
+
stoppedBy = "tokens";
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
const { answer, error } = await attempt(tier.run, values, (text, stage) => opts.emit?.(text, stage ?? tier.name));
|
|
28
|
+
const passed = answer !== null && !error && (await opts.accept(answer, index));
|
|
29
|
+
const record = {
|
|
30
|
+
index,
|
|
31
|
+
name: tier.name,
|
|
32
|
+
answer,
|
|
33
|
+
accepted: passed,
|
|
34
|
+
...(passed ? {} : { escalatedBy: error ? "failed" : "rejected" }),
|
|
35
|
+
error,
|
|
36
|
+
tokens: tokensOf(answer),
|
|
37
|
+
};
|
|
38
|
+
steps.push(tierStep(steps.length, record));
|
|
39
|
+
spent += record.tokens;
|
|
40
|
+
history.push(record);
|
|
41
|
+
opts.onTier?.(record);
|
|
42
|
+
if (passed) {
|
|
43
|
+
accepted = record;
|
|
44
|
+
stoppedBy = "accepted";
|
|
45
|
+
break;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const winner = accepted ?? fallback(history);
|
|
49
|
+
const totals = history.reduce((sum, record) => ({
|
|
50
|
+
inputTokens: sum.inputTokens + (record.answer?.inputTokens ?? 0),
|
|
51
|
+
outputTokens: sum.outputTokens + (record.answer?.outputTokens ?? 0),
|
|
52
|
+
}), { inputTokens: 0, outputTokens: 0 });
|
|
53
|
+
const shared = {
|
|
54
|
+
steps,
|
|
55
|
+
outputs: { history },
|
|
56
|
+
latencyMs: Date.now() - startedAt,
|
|
57
|
+
...totals,
|
|
58
|
+
accepted: accepted !== undefined,
|
|
59
|
+
escalated: history.filter((r) => r !== winner).map((r) => r.name),
|
|
60
|
+
history,
|
|
61
|
+
};
|
|
62
|
+
if (!winner) {
|
|
63
|
+
return {
|
|
64
|
+
...shared,
|
|
65
|
+
status: "failed",
|
|
66
|
+
output: "",
|
|
67
|
+
error: history.at(-1)?.error ?? "No tier produced an answer.",
|
|
68
|
+
tier: null,
|
|
69
|
+
stoppedBy: stoppedBy === "tokens" ? "tokens" : "failed",
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
...shared,
|
|
74
|
+
status: "completed",
|
|
75
|
+
output: winner.answer?.output ?? "",
|
|
76
|
+
tier: winner.name,
|
|
77
|
+
stoppedBy,
|
|
78
|
+
...(winner.answer?.data === undefined ? {} : { data: winner.answer.data }),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { RolloutStep } from "@spendgraph/sdk";
|
|
2
|
+
import type { Emit } from "../stream/index.js";
|
|
3
|
+
import type { Answer, TierRecord } from "./types.js";
|
|
4
|
+
/** What one tier consumed, both halves, whether or not its answer was used. */
|
|
5
|
+
export declare function tokensOf(answer: Answer | null): number;
|
|
6
|
+
/**
|
|
7
|
+
* One tier as a step, named for the tier rather than its position.
|
|
8
|
+
*
|
|
9
|
+
* A rollout that read `tier_0` would need the config beside it to mean anything,
|
|
10
|
+
* and the config is the thing most likely to have changed since.
|
|
11
|
+
*/
|
|
12
|
+
export declare function tierStep(index: number, record: TierRecord): RolloutStep;
|
|
13
|
+
/**
|
|
14
|
+
* Runs one tier, turning a throw into an escalation.
|
|
15
|
+
*
|
|
16
|
+
* An overloaded cheap model is the case this workflow exists for, so a tier that
|
|
17
|
+
* falls over must hand on rather than end the run. The next rung is the whole
|
|
18
|
+
* point of having one.
|
|
19
|
+
*/
|
|
20
|
+
export declare function attempt(run: (values: Record<string, unknown>, emit: Emit) => Promise<Answer> | Answer, values: Record<string, unknown>, emit: Emit): Promise<{
|
|
21
|
+
answer: Answer | null;
|
|
22
|
+
error?: string;
|
|
23
|
+
}>;
|
|
24
|
+
/**
|
|
25
|
+
* The tier to fall back to when nothing was accepted.
|
|
26
|
+
*
|
|
27
|
+
* The last that produced anything, because the tiers are ordered by capability
|
|
28
|
+
* and the most capable answer is the closest thing to a best effort. A tier that
|
|
29
|
+
* threw produced nothing, so it cannot be the fallback however late it ran.
|
|
30
|
+
*/
|
|
31
|
+
export declare function fallback(history: TierRecord[]): TierRecord | undefined;
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export function tokensOf(answer) {
|
|
2
|
+
if (!answer)
|
|
3
|
+
return 0;
|
|
4
|
+
return (answer.inputTokens ?? 0) + (answer.outputTokens ?? 0);
|
|
5
|
+
}
|
|
6
|
+
export function tierStep(index, record) {
|
|
7
|
+
return {
|
|
8
|
+
index,
|
|
9
|
+
source: record.name,
|
|
10
|
+
model: record.answer?.model,
|
|
11
|
+
output: record.answer?.output ?? "",
|
|
12
|
+
status: record.escalatedBy === "failed" ? "failed" : "completed",
|
|
13
|
+
error: record.error,
|
|
14
|
+
inputTokens: record.answer?.inputTokens,
|
|
15
|
+
outputTokens: record.answer?.outputTokens,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
export async function attempt(run, values, emit) {
|
|
19
|
+
try {
|
|
20
|
+
const answer = await run(values, emit);
|
|
21
|
+
if (answer.status === "failed") {
|
|
22
|
+
return { answer, error: answer.error ?? "The tier failed." };
|
|
23
|
+
}
|
|
24
|
+
return { answer };
|
|
25
|
+
}
|
|
26
|
+
catch (cause) {
|
|
27
|
+
return { answer: null, error: cause instanceof Error ? cause.message : String(cause) };
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function fallback(history) {
|
|
31
|
+
return [...history].reverse().find((record) => record.answer !== null);
|
|
32
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type { RolloutStep } from "@spendgraph/sdk";
|
|
2
|
+
import type { Emit } from "../stream/index.js";
|
|
3
|
+
/** What one tier produced. Structurally the `LlmReply` a provider read gives back. */
|
|
4
|
+
export interface Answer {
|
|
5
|
+
output: string;
|
|
6
|
+
model: string;
|
|
7
|
+
status?: "completed" | "failed";
|
|
8
|
+
error?: string;
|
|
9
|
+
inputTokens?: number;
|
|
10
|
+
outputTokens?: number;
|
|
11
|
+
/** A parsed reply, where the tier asked for a shape. What `accept` usually reads. */
|
|
12
|
+
data?: unknown;
|
|
13
|
+
}
|
|
14
|
+
/** One rung: a name for the record, and the call it makes. */
|
|
15
|
+
export interface Tier {
|
|
16
|
+
/** How it appears in `result.tier`, `escalated` and the steps. Unique. */
|
|
17
|
+
name: string;
|
|
18
|
+
run(values: Record<string, unknown>, emit: Emit): Promise<Answer> | Answer;
|
|
19
|
+
}
|
|
20
|
+
/** Why a tier did not answer. */
|
|
21
|
+
export type Escalation = "rejected" | "failed";
|
|
22
|
+
/** What one tier did, whether it answered or not. */
|
|
23
|
+
export interface TierRecord {
|
|
24
|
+
index: number;
|
|
25
|
+
name: string;
|
|
26
|
+
/** What it produced, or null when the call threw. */
|
|
27
|
+
answer: Answer | null;
|
|
28
|
+
accepted: boolean;
|
|
29
|
+
/**
|
|
30
|
+
* Absent when it answered.
|
|
31
|
+
*
|
|
32
|
+
* `rejected` is `accept` saying no; `failed` is the call falling over. They
|
|
33
|
+
* want opposite fixes — a bar set too high against a model that is down — and
|
|
34
|
+
* a record that collapsed them could not tell you which you have.
|
|
35
|
+
*/
|
|
36
|
+
escalatedBy?: Escalation;
|
|
37
|
+
error?: string;
|
|
38
|
+
tokens: number;
|
|
39
|
+
}
|
|
40
|
+
export interface CascadeOptions {
|
|
41
|
+
/**
|
|
42
|
+
* Rungs in order, cheapest first.
|
|
43
|
+
*
|
|
44
|
+
* Ordered by capability rather than price where the two disagree: the last is
|
|
45
|
+
* what a run falls back to, so it should be the one most likely to cope.
|
|
46
|
+
*/
|
|
47
|
+
tiers: Tier[];
|
|
48
|
+
/**
|
|
49
|
+
* Whether that answer will do.
|
|
50
|
+
*
|
|
51
|
+
* Required. Without it there is no cascade — only a list of models and no
|
|
52
|
+
* reason to stop at any of them.
|
|
53
|
+
*/
|
|
54
|
+
accept(answer: Answer, index: number): Promise<boolean> | boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Where the tokens go while this runs.
|
|
57
|
+
*
|
|
58
|
+
* Passed down to whatever makes the model call; omit it and nothing is
|
|
59
|
+
* emitted. `streamed()` supplies one.
|
|
60
|
+
*/
|
|
61
|
+
emit?: Emit;
|
|
62
|
+
/**
|
|
63
|
+
* Stop once the tiers have consumed this many tokens.
|
|
64
|
+
*
|
|
65
|
+
* Checked before each tier, so the ceiling is never crossed rather than merely
|
|
66
|
+
* noticed. Tokens rather than money because this package cannot price a call.
|
|
67
|
+
*/
|
|
68
|
+
maxTokens?: number;
|
|
69
|
+
/** Called after each tier, so a long escalation is not silent. */
|
|
70
|
+
onTier?: (record: TierRecord) => void;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* What the cascade produced, in the shape of a rollout.
|
|
74
|
+
*
|
|
75
|
+
* `inputTokens` and `outputTokens` cover **every** tier tried, not the winner
|
|
76
|
+
* alone. The only question a cascade has to answer is whether it paid, and a
|
|
77
|
+
* total that hid the rejected attempts would always say yes.
|
|
78
|
+
*/
|
|
79
|
+
export interface CascadeResult {
|
|
80
|
+
status: "completed" | "failed";
|
|
81
|
+
output: string;
|
|
82
|
+
error?: string;
|
|
83
|
+
steps: RolloutStep[];
|
|
84
|
+
outputs: Record<string, unknown>;
|
|
85
|
+
latencyMs: number;
|
|
86
|
+
inputTokens: number;
|
|
87
|
+
outputTokens: number;
|
|
88
|
+
/** True when a tier passed `accept`. False when the answer is a fallback. */
|
|
89
|
+
accepted: boolean;
|
|
90
|
+
/** Whose answer this is: the one accepted, or the last that produced one. */
|
|
91
|
+
tier: string | null;
|
|
92
|
+
/** Tiers tried that did not supply the answer, in the order they were tried. */
|
|
93
|
+
escalated: string[];
|
|
94
|
+
stoppedBy: "accepted" | "exhausted" | "tokens" | "failed";
|
|
95
|
+
/** The winning tier's parsed reply, where it had one. */
|
|
96
|
+
data?: unknown;
|
|
97
|
+
history: TierRecord[];
|
|
98
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ChainOptions, ChainResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Steps in sequence, each on the last one's output, with a gate between.
|
|
4
|
+
*
|
|
5
|
+
* Compiled to a graph, which already stops at the node that failed and records
|
|
6
|
+
* every node that ran before it — which is the whole reason to gate a chain
|
|
7
|
+
* rather than let it run to the end and hand back something plausible.
|
|
8
|
+
*/
|
|
9
|
+
export declare function chain(values: Record<string, unknown>, opts: ChainOptions): Promise<ChainResult>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { edge, end, graph } from "@spendgraph/graph";
|
|
2
|
+
import { runGraph } from "../stream/graph.js";
|
|
3
|
+
import { gateName, gateNode } from "./gate.js";
|
|
4
|
+
import { stepNode } from "./step.js";
|
|
5
|
+
function assertNamed(steps) {
|
|
6
|
+
if (steps.length === 0)
|
|
7
|
+
throw new Error("A chain needs at least one step.");
|
|
8
|
+
const seen = new Set();
|
|
9
|
+
for (const step of steps) {
|
|
10
|
+
if (seen.has(step.name))
|
|
11
|
+
throw new Error(`Two steps are called "${step.name}".`);
|
|
12
|
+
seen.add(step.name);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
function nodesFor(steps) {
|
|
16
|
+
const nodes = [];
|
|
17
|
+
const order = [];
|
|
18
|
+
let previous = null;
|
|
19
|
+
for (const step of steps) {
|
|
20
|
+
nodes.push(stepNode(step, previous));
|
|
21
|
+
order.push(step.name);
|
|
22
|
+
if (step.gate) {
|
|
23
|
+
nodes.push(gateNode(step));
|
|
24
|
+
order.push(gateName(step.name));
|
|
25
|
+
}
|
|
26
|
+
previous = step.name;
|
|
27
|
+
}
|
|
28
|
+
return { nodes, order };
|
|
29
|
+
}
|
|
30
|
+
export async function chain(values, opts) {
|
|
31
|
+
assertNamed(opts.steps);
|
|
32
|
+
const { nodes, order } = nodesFor(opts.steps);
|
|
33
|
+
const flow = graph({
|
|
34
|
+
entry: order[0],
|
|
35
|
+
maxSteps: opts.maxSteps,
|
|
36
|
+
nodes,
|
|
37
|
+
edges: [
|
|
38
|
+
...order.slice(0, -1).map((from, at) => edge(from, order[at + 1])),
|
|
39
|
+
end(order.at(-1)),
|
|
40
|
+
],
|
|
41
|
+
}, { now: opts.now });
|
|
42
|
+
const result = await runGraph(flow, values, opts.emit);
|
|
43
|
+
const stopped = result.status === "failed" ? result.steps.at(-1) : undefined;
|
|
44
|
+
return {
|
|
45
|
+
...result,
|
|
46
|
+
ran: result.steps.filter((s) => !s.source.startsWith("gate_")).map((s) => s.source),
|
|
47
|
+
stoppedAt: stopped ? stopped.source.replace(/^gate_/, "") : null,
|
|
48
|
+
reason: stopped?.error,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Step } from "./types.js";
|
|
2
|
+
/** The gate node's name for a stage, and the key its verdict is stored under. */
|
|
3
|
+
export declare function gateName(step: string): string;
|
|
4
|
+
export declare class GateRefused extends Error {
|
|
5
|
+
readonly step: string;
|
|
6
|
+
readonly reason: string;
|
|
7
|
+
constructor(step: string, reason: string);
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* A gate as its own node, so a refusal is a step of its own.
|
|
11
|
+
*
|
|
12
|
+
* Folding the check into the stage would lose the output it refused — and the
|
|
13
|
+
* rejected output is the evidence for why the gate fired at all. This way the
|
|
14
|
+
* stage records what it produced and the gate records what was wrong with it.
|
|
15
|
+
*/
|
|
16
|
+
export declare function gateNode(step: Step): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>;
|