@spendgraph/harness 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -17
- package/dist/cascade/cascade.js +0 -11
- package/dist/cascade/tier.js +0 -21
- package/dist/chain/chain.js +0 -8
- package/dist/chain/gate.js +0 -8
- package/dist/chain/step.js +0 -7
- package/dist/loop/act.js +0 -14
- package/dist/loop/compact.js +0 -18
- package/dist/loop/hooks.js +0 -10
- package/dist/loop/loop.js +0 -21
- package/dist/loop/turn.js +0 -2
- package/dist/orchestrate/orchestrate.js +0 -12
- package/dist/orchestrate/plan.js +0 -11
- package/dist/orchestrate/work.js +0 -10
- package/dist/parallel/merge.js +0 -17
- package/dist/parallel/parallel.js +0 -10
- package/dist/parallel/task.js +0 -8
- package/dist/refine/attempt.js +0 -14
- package/dist/refine/judge.js +0 -11
- package/dist/refine/refine.js +0 -8
- package/dist/route/classify.js +0 -24
- package/dist/route/dispatch.js +0 -8
- package/dist/route/route.js +0 -9
- package/dist/stream/graph.js +0 -6
- package/dist/stream/stream.js +0 -17
- package/package.json +7 -7
package/README.md
CHANGED
|
@@ -9,7 +9,11 @@ your provider ships. It is built on `@spendgraph/prompt`, `@spendgraph/tools`
|
|
|
9
9
|
and `@spendgraph/graph`, and none of it reasons — every workflow here directs a
|
|
10
10
|
model call you supply.
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
```sh
|
|
13
|
+
npm install @spendgraph/harness
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Seven shapes, all built. Each is a graph you would otherwise hand-roll, with the
|
|
13
17
|
pricing and the rollout already attached, so a run is recorded rather than
|
|
14
18
|
reconstructed afterwards.
|
|
15
19
|
|
|
@@ -23,6 +27,117 @@ reconstructed afterwards.
|
|
|
23
27
|
| **`loop`** | tools in a loop until the model stops asking | open-ended work with a tool surface and a step ceiling |
|
|
24
28
|
| **`cascade`** | try the cheap model, escalate only when it will not do | most inputs are easy and a few are not, and you can tell which |
|
|
25
29
|
|
|
30
|
+
## Two of them, in full
|
|
31
|
+
|
|
32
|
+
`refine` is generate and critique until it passes:
|
|
33
|
+
|
|
34
|
+
```ts
|
|
35
|
+
import { refine } from "@spendgraph/harness";
|
|
36
|
+
|
|
37
|
+
const result = await refine({
|
|
38
|
+
attempt: (feedback) =>
|
|
39
|
+
llm.call([{ role: "user", content: feedback ? `${brief}\n\nFix: ${feedback}` : brief }]),
|
|
40
|
+
judge: (attempt) =>
|
|
41
|
+
attempt.output.length < 900
|
|
42
|
+
? { accepted: true }
|
|
43
|
+
: { accepted: false, feedback: "Cut it to under 900 characters." },
|
|
44
|
+
rounds: 3,
|
|
45
|
+
maxTokens: 40_000,
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
result.output; // the winning draft
|
|
49
|
+
result.accepted; // false when the rounds or the ceiling ran out first
|
|
50
|
+
result.stoppedBy; // "accepted" | "rounds" | "tokens" | "failed"
|
|
51
|
+
result.history; // every attempt, including the rejected ones
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
One `attempt` callback rather than separate generate and revise, because in
|
|
55
|
+
practice they are the same prompt with one extra paragraph. `feedback` is null
|
|
56
|
+
on the first round and the judge's note after it.
|
|
57
|
+
|
|
58
|
+
`route` classifies, then dispatches:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
import { route } from "@spendgraph/harness";
|
|
62
|
+
|
|
63
|
+
const result = await route(
|
|
64
|
+
{ subject, body },
|
|
65
|
+
{
|
|
66
|
+
classify: ({ subject }) => (/refund|charge/i.test(String(subject)) ? "billing" : "general"),
|
|
67
|
+
routes: {
|
|
68
|
+
billing: (values) => billingAgent(values),
|
|
69
|
+
general: (values) => generalAgent(values),
|
|
70
|
+
},
|
|
71
|
+
fallback: (values) => humanQueue(values),
|
|
72
|
+
}
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
result.route; // the branch that ran, or null when the fallback did
|
|
76
|
+
result.classified; // what the classifier said, whether or not it was used
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**`fallback` is required.** A router that can fail to route fails in production
|
|
80
|
+
at 3am, and "unknown" is a class every classifier eventually returns. `classify`
|
|
81
|
+
is often better as plain code than a model — the regex above is free and never
|
|
82
|
+
invents a category.
|
|
83
|
+
|
|
84
|
+
## Calling convention
|
|
85
|
+
|
|
86
|
+
Five take the run's values first; two have nothing to substitute into and take
|
|
87
|
+
options alone.
|
|
88
|
+
|
|
89
|
+
```ts
|
|
90
|
+
await chain(values, opts);
|
|
91
|
+
await route(values, opts);
|
|
92
|
+
await parallel(values, opts);
|
|
93
|
+
await orchestrate(values, opts);
|
|
94
|
+
await cascade(values, opts);
|
|
95
|
+
|
|
96
|
+
await refine(opts);
|
|
97
|
+
await loop(opts);
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## What every one of them owes you
|
|
101
|
+
|
|
102
|
+
Each result is a `GraphResult` first, so it reports as a rollout without
|
|
103
|
+
translation:
|
|
104
|
+
|
|
105
|
+
```ts
|
|
106
|
+
result.status; // "completed" | "failed"
|
|
107
|
+
result.output;
|
|
108
|
+
result.steps; // every step, the rejected ones included
|
|
109
|
+
result.inputTokens; // summed across the whole run
|
|
110
|
+
result.outputTokens;
|
|
111
|
+
result.stoppedBy; // never the same value as "it finished"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
- Every step recorded, including the ones that were rejected — a refine loop that
|
|
115
|
+
keeps only the winner cannot say what it cost.
|
|
116
|
+
- A budget ceiling, because each of these spends more than one call and the
|
|
117
|
+
interesting ones spend an unbounded amount.
|
|
118
|
+
- A `stoppedBy`, because "it finished" and "it gave up" must never look the same.
|
|
119
|
+
|
|
120
|
+
Tokens are summed across **every** attempt rather than the winning one. The only
|
|
121
|
+
question a refine loop has to answer is whether refining paid, and a total that
|
|
122
|
+
counted the winner alone would always say yes.
|
|
123
|
+
|
|
124
|
+
## Watching one run
|
|
125
|
+
|
|
126
|
+
`streamed()` wraps any workflow and hands it an `emit`, so the tokens go
|
|
127
|
+
somewhere while it runs instead of arriving all at once at the end.
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
import { streamed } from "@spendgraph/harness";
|
|
131
|
+
|
|
132
|
+
const running = streamed((emit) => refine({ attempt, judge, emit }));
|
|
133
|
+
|
|
134
|
+
for await (const event of running) {
|
|
135
|
+
if (event.type === "token") res.write(event.text);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const result = await running.result;
|
|
139
|
+
```
|
|
140
|
+
|
|
26
141
|
## Inner and outer
|
|
27
142
|
|
|
28
143
|
The provider's SDK is the **inner** harness: it drives one model call, and on
|
|
@@ -47,15 +162,6 @@ step poisons the rest, voting needs an odd N, a loop needs a ceiling and every
|
|
|
47
162
|
one of them needs a budget. Putting them beside the mechanism would suggest the
|
|
48
163
|
mechanism is opinionated. It is not.
|
|
49
164
|
|
|
50
|
-
## What every one of them owes you
|
|
51
|
-
|
|
52
|
-
- A `GraphResult`, so the run reports as a rollout without translation.
|
|
53
|
-
- Every step recorded, including the ones that were rejected — a refine loop that
|
|
54
|
-
keeps only the winner cannot say what it cost.
|
|
55
|
-
- A budget ceiling, because each of these spends more than one call and the
|
|
56
|
-
interesting ones spend an unbounded amount.
|
|
57
|
-
- A `stoppedBy`, because "it finished" and "it gave up" must never look the same.
|
|
58
|
-
|
|
59
165
|
## What is missing
|
|
60
166
|
|
|
61
167
|
Every workflow here is **single-run**: it starts, it finishes, and it holds
|
|
@@ -67,13 +173,6 @@ That needs somewhere to persist a run, which is a decision about the product and
|
|
|
67
173
|
not a wiring pattern. It is written up in [`PLAN.md`](./PLAN.md) rather than
|
|
68
174
|
quietly folded in.
|
|
69
175
|
|
|
70
|
-
## What is next
|
|
71
|
-
|
|
72
|
-
Phase one is done: the context manager (`compact`), lifecycle hooks, tool
|
|
73
|
-
annotations, budget signalling and `cascade`. What is left is **durable state** —
|
|
74
|
-
a run that can wait for a person and be resumed — which [`PLAN.md`](./PLAN.md)
|
|
75
|
-
plans with the decisions and the tests it needs.
|
|
76
|
-
|
|
77
176
|
## Documentation
|
|
78
177
|
|
|
79
178
|
Each built workflow has a page with worked examples, the cases it suits, and the
|
|
@@ -87,3 +186,7 @@ cases it does not.
|
|
|
87
186
|
- [`docs/orchestrate.md`](./docs/orchestrate.md) — a lead decomposes and delegates
|
|
88
187
|
- [`docs/cascade.md`](./docs/cascade.md) — cheap first, escalate on rejection
|
|
89
188
|
- [`docs/README.md`](./docs) — the seven patterns, and what each owes you
|
|
189
|
+
|
|
190
|
+
## License
|
|
191
|
+
|
|
192
|
+
MIT
|
package/dist/cascade/cascade.js
CHANGED
|
@@ -11,17 +11,6 @@ function checkTiers(tiers) {
|
|
|
11
11
|
seen.add(tier.name);
|
|
12
12
|
}
|
|
13
13
|
}
|
|
14
|
-
/**
|
|
15
|
-
* Try the cheap model, and escalate only when its answer will not do.
|
|
16
|
-
*
|
|
17
|
-
* Not `route`, which chooses before it has seen an answer. Not `refine`, which
|
|
18
|
-
* revises with the same model. This is the one lever that spends less by
|
|
19
|
-
* default and more only on the inputs that turn out to need it.
|
|
20
|
-
*
|
|
21
|
-
* Whether it pays depends entirely on how often the first tier is accepted, so
|
|
22
|
-
* every tier tried is recorded and its tokens counted — a total that hid the
|
|
23
|
-
* rejected attempts would report a saving on every run.
|
|
24
|
-
*/
|
|
25
14
|
export async function cascade(values, opts) {
|
|
26
15
|
const startedAt = Date.now();
|
|
27
16
|
checkTiers(opts.tiers);
|
package/dist/cascade/tier.js
CHANGED
|
@@ -1,15 +1,8 @@
|
|
|
1
|
-
/** What one tier consumed, both halves, whether or not its answer was used. */
|
|
2
1
|
export function tokensOf(answer) {
|
|
3
2
|
if (!answer)
|
|
4
3
|
return 0;
|
|
5
4
|
return (answer.inputTokens ?? 0) + (answer.outputTokens ?? 0);
|
|
6
5
|
}
|
|
7
|
-
/**
|
|
8
|
-
* One tier as a step, named for the tier rather than its position.
|
|
9
|
-
*
|
|
10
|
-
* A rollout that read `tier_0` would need the config beside it to mean anything,
|
|
11
|
-
* and the config is the thing most likely to have changed since.
|
|
12
|
-
*/
|
|
13
6
|
export function tierStep(index, record) {
|
|
14
7
|
return {
|
|
15
8
|
index,
|
|
@@ -22,13 +15,6 @@ export function tierStep(index, record) {
|
|
|
22
15
|
outputTokens: record.answer?.outputTokens,
|
|
23
16
|
};
|
|
24
17
|
}
|
|
25
|
-
/**
|
|
26
|
-
* Runs one tier, turning a throw into an escalation.
|
|
27
|
-
*
|
|
28
|
-
* An overloaded cheap model is the case this workflow exists for, so a tier that
|
|
29
|
-
* falls over must hand on rather than end the run. The next rung is the whole
|
|
30
|
-
* point of having one.
|
|
31
|
-
*/
|
|
32
18
|
export async function attempt(run, values, emit) {
|
|
33
19
|
try {
|
|
34
20
|
const answer = await run(values, emit);
|
|
@@ -41,13 +27,6 @@ export async function attempt(run, values, emit) {
|
|
|
41
27
|
return { answer: null, error: cause instanceof Error ? cause.message : String(cause) };
|
|
42
28
|
}
|
|
43
29
|
}
|
|
44
|
-
/**
|
|
45
|
-
* The tier to fall back to when nothing was accepted.
|
|
46
|
-
*
|
|
47
|
-
* The last that produced anything, because the tiers are ordered by capability
|
|
48
|
-
* and the most capable answer is the closest thing to a best effort. A tier that
|
|
49
|
-
* threw produced nothing, so it cannot be the fallback however late it ran.
|
|
50
|
-
*/
|
|
51
30
|
export function fallback(history) {
|
|
52
31
|
return [...history].reverse().find((record) => record.answer !== null);
|
|
53
32
|
}
|
package/dist/chain/chain.js
CHANGED
|
@@ -12,7 +12,6 @@ function assertNamed(steps) {
|
|
|
12
12
|
seen.add(step.name);
|
|
13
13
|
}
|
|
14
14
|
}
|
|
15
|
-
/** Every node in order: a stage, then its gate when it has one. */
|
|
16
15
|
function nodesFor(steps) {
|
|
17
16
|
const nodes = [];
|
|
18
17
|
const order = [];
|
|
@@ -28,13 +27,6 @@ function nodesFor(steps) {
|
|
|
28
27
|
}
|
|
29
28
|
return { nodes, order };
|
|
30
29
|
}
|
|
31
|
-
/**
|
|
32
|
-
* Steps in sequence, each on the last one's output, with a gate between.
|
|
33
|
-
*
|
|
34
|
-
* Compiled to a graph, which already stops at the node that failed and records
|
|
35
|
-
* every node that ran before it — which is the whole reason to gate a chain
|
|
36
|
-
* rather than let it run to the end and hand back something plausible.
|
|
37
|
-
*/
|
|
38
30
|
export async function chain(values, opts) {
|
|
39
31
|
assertNamed(opts.steps);
|
|
40
32
|
const { nodes, order } = nodesFor(opts.steps);
|
package/dist/chain/gate.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { node } from "@spendgraph/graph";
|
|
2
|
-
/** The gate node's name for a stage, and the key its verdict is stored under. */
|
|
3
2
|
export function gateName(step) {
|
|
4
3
|
return `gate_${step}`;
|
|
5
4
|
}
|
|
@@ -13,13 +12,6 @@ export class GateRefused extends Error {
|
|
|
13
12
|
this.name = "GateRefused";
|
|
14
13
|
}
|
|
15
14
|
}
|
|
16
|
-
/**
|
|
17
|
-
* A gate as its own node, so a refusal is a step of its own.
|
|
18
|
-
*
|
|
19
|
-
* Folding the check into the stage would lose the output it refused — and the
|
|
20
|
-
* rejected output is the evidence for why the gate fired at all. This way the
|
|
21
|
-
* stage records what it produced and the gate records what was wrong with it.
|
|
22
|
-
*/
|
|
23
15
|
export function gateNode(step) {
|
|
24
16
|
return node({
|
|
25
17
|
name: gateName(step.name),
|
package/dist/chain/step.js
CHANGED
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
import { node } from "@spendgraph/graph";
|
|
2
|
-
/**
|
|
3
|
-
* A stage as a node.
|
|
4
|
-
*
|
|
5
|
-
* `previous` is read from the context rather than threaded through the input,
|
|
6
|
-
* because a gate may sit between two stages and its return is not what the next
|
|
7
|
-
* stage wants — the stage before it is.
|
|
8
|
-
*/
|
|
9
2
|
export function stepNode(step, previous) {
|
|
10
3
|
return node({
|
|
11
4
|
name: step.name,
|
package/dist/loop/act.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { answerOf, isFinished } from "@spendgraph/tools";
|
|
2
2
|
import { ask, blockedResult, tell } from "./hooks.js";
|
|
3
|
-
/** One tool call as a step, failures included. */
|
|
4
3
|
export function callStep(index, result) {
|
|
5
4
|
return {
|
|
6
5
|
index,
|
|
@@ -12,13 +11,6 @@ export function callStep(index, result) {
|
|
|
12
11
|
latencyMs: result.latencyMs,
|
|
13
12
|
};
|
|
14
13
|
}
|
|
15
|
-
/**
|
|
16
|
-
* Runs what the model asked for, in the order it asked.
|
|
17
|
-
*
|
|
18
|
-
* Never throws — `invoke` returns a failed result rather than raising, so a
|
|
19
|
-
* tool that fell over is something the model can read and try around, not
|
|
20
|
-
* something that ends the run.
|
|
21
|
-
*/
|
|
22
14
|
export async function act(tools, calls, hooks = {}) {
|
|
23
15
|
const results = [];
|
|
24
16
|
for (const call of calls) {
|
|
@@ -31,12 +23,6 @@ export async function act(tools, calls, hooks = {}) {
|
|
|
31
23
|
}
|
|
32
24
|
return results;
|
|
33
25
|
}
|
|
34
|
-
/**
|
|
35
|
-
* The answer, when the model said it was done.
|
|
36
|
-
*
|
|
37
|
-
* Reading `finish` here is what separates "it is done" from "it stopped asking",
|
|
38
|
-
* which otherwise look identical from outside and are opposite problems.
|
|
39
|
-
*/
|
|
40
26
|
export function finished(results) {
|
|
41
27
|
const done = results.find((result) => isFinished(result));
|
|
42
28
|
return done ? answerOf(done) : null;
|
package/dist/loop/compact.js
CHANGED
|
@@ -1,17 +1,7 @@
|
|
|
1
1
|
const DEFAULT_AT = 0.7;
|
|
2
2
|
const DEFAULT_KEEP = 4;
|
|
3
|
-
/**
|
|
4
|
-
* Whether the context is big enough to be worth summarising.
|
|
5
|
-
*
|
|
6
|
-
* Measured on the **last turn's input tokens**, not on what the run has spent.
|
|
7
|
-
* Spend only ever rises, so a threshold on it fires once and then every turn
|
|
8
|
-
* after; the input count is the size of what is actually being resent, and it
|
|
9
|
-
* falls the moment a compaction lands — which is what stops this firing twice
|
|
10
|
-
* for the same reason.
|
|
11
|
-
*/
|
|
12
3
|
export function shouldCompact(compact, last, results, maxTokens) {
|
|
13
4
|
const keep = compact.keep ?? DEFAULT_KEEP;
|
|
14
|
-
// Nothing to gain: everything left would be kept anyway.
|
|
15
5
|
if (results.length <= keep)
|
|
16
6
|
return false;
|
|
17
7
|
if (compact.after !== undefined && results.length > compact.after)
|
|
@@ -21,7 +11,6 @@ export function shouldCompact(compact, last, results, maxTokens) {
|
|
|
21
11
|
const context = last.inputTokens ?? 0;
|
|
22
12
|
return context >= (compact.at ?? DEFAULT_AT) * maxTokens;
|
|
23
13
|
}
|
|
24
|
-
/** The summary as a tool result, so the next turn reads it like any other. */
|
|
25
14
|
export function summaryResult(summary, replaced) {
|
|
26
15
|
return {
|
|
27
16
|
name: "compacted",
|
|
@@ -31,7 +20,6 @@ export function summaryResult(summary, replaced) {
|
|
|
31
20
|
latencyMs: 0,
|
|
32
21
|
};
|
|
33
22
|
}
|
|
34
|
-
/** The summarising call as a step, because deciding what to forget is not free. */
|
|
35
23
|
export function compactStep(index, summary, replaced) {
|
|
36
24
|
return {
|
|
37
25
|
index,
|
|
@@ -44,12 +32,6 @@ export function compactStep(index, summary, replaced) {
|
|
|
44
32
|
outputTokens: summary.outputTokens,
|
|
45
33
|
};
|
|
46
34
|
}
|
|
47
|
-
/**
|
|
48
|
-
* Replaces the older results with one summary, in place.
|
|
49
|
-
*
|
|
50
|
-
* The most recent `keep` survive verbatim: summarising what just happened is
|
|
51
|
-
* how a loop forgets what it was in the middle of doing.
|
|
52
|
-
*/
|
|
53
35
|
export function applyCompaction(results, summary, keep) {
|
|
54
36
|
const older = results.splice(0, results.length - keep);
|
|
55
37
|
const summarised = summaryResult(summary, older.length);
|
package/dist/loop/hooks.js
CHANGED
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Runs a hook that can refuse, treating its own failure as permission.
|
|
3
|
-
*
|
|
4
|
-
* A hook that throws is a bug in the guardrail, and taking the run down with it
|
|
5
|
-
* is worse than the thing it was guarding against — the call it was checking
|
|
6
|
-
* has not happened yet, and the next hook still gets its say.
|
|
7
|
-
*/
|
|
8
1
|
export async function ask(hook, ...args) {
|
|
9
2
|
if (!hook)
|
|
10
3
|
return null;
|
|
@@ -17,7 +10,6 @@ export async function ask(hook, ...args) {
|
|
|
17
10
|
return null;
|
|
18
11
|
}
|
|
19
12
|
}
|
|
20
|
-
/** Runs an observing hook. Its failure changes nothing; it was only watching. */
|
|
21
13
|
export async function tell(hook, arg) {
|
|
22
14
|
if (!hook)
|
|
23
15
|
return;
|
|
@@ -25,10 +17,8 @@ export async function tell(hook, arg) {
|
|
|
25
17
|
await hook(arg);
|
|
26
18
|
}
|
|
27
19
|
catch {
|
|
28
|
-
// Observation only.
|
|
29
20
|
}
|
|
30
21
|
}
|
|
31
|
-
/** A refused call, in the shape the model already knows how to read. */
|
|
32
22
|
export function blockedResult(call, reason) {
|
|
33
23
|
return {
|
|
34
24
|
name: call.name,
|
package/dist/loop/loop.js
CHANGED
|
@@ -3,17 +3,6 @@ import { applyCompaction, compactStep, DEFAULT_KEEP, shouldCompact } from "./com
|
|
|
3
3
|
import { ask, tell } from "./hooks.js";
|
|
4
4
|
import { tokensOf, turnStep } from "./turn.js";
|
|
5
5
|
const DEFAULT_MAX_TURNS = 8;
|
|
6
|
-
/**
|
|
7
|
-
* Model, tools, model again, until it stops asking.
|
|
8
|
-
*
|
|
9
|
-
* Not compiled to a graph: this is a cycle, and `@spendgraph/graph` is a DAG
|
|
10
|
-
* whose step ceiling exists to stop one. Same reasoning as `refine`.
|
|
11
|
-
*
|
|
12
|
-
* What this adds over a provider SDK's own tool loop is the part that is always
|
|
13
|
-
* hand-rolled and always wrong the same way: a ceiling that is checked before
|
|
14
|
-
* the spend rather than after, every turn and every tool call recorded, and
|
|
15
|
-
* `finished` told apart from `quiet`.
|
|
16
|
-
*/
|
|
17
6
|
export async function loop(opts) {
|
|
18
7
|
const startedAt = Date.now();
|
|
19
8
|
const maxTurns = Math.max(1, opts.maxTurns ?? DEFAULT_MAX_TURNS);
|
|
@@ -25,13 +14,6 @@ export async function loop(opts) {
|
|
|
25
14
|
let answer = null;
|
|
26
15
|
let stoppedBy = "turns";
|
|
27
16
|
const compactions = { count: 0, replaced: 0 };
|
|
28
|
-
/**
|
|
29
|
-
* Summarise the older results, if the context has grown enough to be worth it.
|
|
30
|
-
*
|
|
31
|
-
* Failing is survivable: the run carries on with the history it has rather
|
|
32
|
-
* than ending over a summary it could not write. Losing the compaction costs
|
|
33
|
-
* tokens; losing the run costs the work.
|
|
34
|
-
*/
|
|
35
17
|
const compactIfNeeded = async () => {
|
|
36
18
|
if (!opts.compact || !shouldCompact(opts.compact, last, results, opts.maxTokens))
|
|
37
19
|
return;
|
|
@@ -46,7 +28,6 @@ export async function loop(opts) {
|
|
|
46
28
|
compactions.replaced += replaced;
|
|
47
29
|
}
|
|
48
30
|
catch {
|
|
49
|
-
// Carry on uncompacted.
|
|
50
31
|
}
|
|
51
32
|
};
|
|
52
33
|
const finish = (status, error) => ({
|
|
@@ -110,8 +91,6 @@ export async function loop(opts) {
|
|
|
110
91
|
break;
|
|
111
92
|
}
|
|
112
93
|
if (calls.length === 0) {
|
|
113
|
-
// It said something and asked for nothing. Done, or lost — the caller
|
|
114
|
-
// cannot tell, which is why `finish` is worth offering it.
|
|
115
94
|
stoppedBy = "quiet";
|
|
116
95
|
break;
|
|
117
96
|
}
|
package/dist/loop/turn.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
/** What one turn consumed. Both halves, since the history is resent every time. */
|
|
2
1
|
export function tokensOf(turn) {
|
|
3
2
|
return (turn.inputTokens ?? 0) + (turn.outputTokens ?? 0);
|
|
4
3
|
}
|
|
5
|
-
/** One model call as a step. */
|
|
6
4
|
export function turnStep(index, at, turn) {
|
|
7
5
|
return {
|
|
8
6
|
index,
|
|
@@ -9,16 +9,6 @@ function tokens(steps) {
|
|
|
9
9
|
}
|
|
10
10
|
return { inputTokens, outputTokens };
|
|
11
11
|
}
|
|
12
|
-
/**
|
|
13
|
-
* A lead decomposes, workers do the pieces, the lead puts it back together.
|
|
14
|
-
*
|
|
15
|
-
* The workflow for work whose shape is not known until the input is read — the
|
|
16
|
-
* difference from `chain`, where the stages are named in advance.
|
|
17
|
-
*
|
|
18
|
-
* It is also the one that can spend the most: the lead decides how many calls
|
|
19
|
-
* to make, so `maxWorkers` is a cap on something a model chose rather than on
|
|
20
|
-
* something you did.
|
|
21
|
-
*/
|
|
22
12
|
export async function orchestrate(values, opts) {
|
|
23
13
|
const startedAt = Date.now();
|
|
24
14
|
const steps = [];
|
|
@@ -44,8 +34,6 @@ export async function orchestrate(values, opts) {
|
|
|
44
34
|
steps.push(planStep(steps.length, plan, dropped));
|
|
45
35
|
opts.onPlan?.(plan, dropped);
|
|
46
36
|
if (subtasks.length === 0) {
|
|
47
|
-
// Nothing to delegate. Handing back the lead's own words beats synthesising
|
|
48
|
-
// over an empty list, which invents an answer from nothing.
|
|
49
37
|
return finish("completed", "planned-nothing", plan.output ?? "");
|
|
50
38
|
}
|
|
51
39
|
const spentOnPlan = (plan.inputTokens ?? 0) + (plan.outputTokens ?? 0);
|
package/dist/orchestrate/plan.js
CHANGED
|
@@ -1,14 +1,4 @@
|
|
|
1
1
|
const DEFAULT_MAX_WORKERS = 5;
|
|
2
|
-
/**
|
|
3
|
-
* The subtasks that will run, and the ones the cap refused.
|
|
4
|
-
*
|
|
5
|
-
* Dropping the tail rather than the head: a lead asked for the most important
|
|
6
|
-
* piece first usually gives it first, and truncating from the front would throw
|
|
7
|
-
* away exactly what it thought mattered.
|
|
8
|
-
*
|
|
9
|
-
* Duplicated names are made unique rather than rejected — a lead naming two
|
|
10
|
-
* sections "summary" is a wording problem, not a reason to lose the run.
|
|
11
|
-
*/
|
|
12
2
|
export function capped(plan, max = DEFAULT_MAX_WORKERS) {
|
|
13
3
|
const limit = Math.max(1, max);
|
|
14
4
|
const seen = new Map();
|
|
@@ -19,7 +9,6 @@ export function capped(plan, max = DEFAULT_MAX_WORKERS) {
|
|
|
19
9
|
});
|
|
20
10
|
return { subtasks: named.slice(0, limit), dropped: named.slice(limit) };
|
|
21
11
|
}
|
|
22
|
-
/** The lead's own call as a step, because deciding is not free. */
|
|
23
12
|
export function planStep(index, plan, dropped) {
|
|
24
13
|
return {
|
|
25
14
|
index,
|
package/dist/orchestrate/work.js
CHANGED
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
import { parallel } from "../parallel/index.js";
|
|
2
|
-
/**
|
|
3
|
-
* Every worker at once.
|
|
4
|
-
*
|
|
5
|
-
* Delegated to `parallel` rather than fanned out again here: it already caps the
|
|
6
|
-
* concurrency, already turns a thrown worker into a result so the others land,
|
|
7
|
-
* and already numbers the steps by declaration so two runs can be compared.
|
|
8
|
-
*/
|
|
9
2
|
export function work(subtasks, values, opts) {
|
|
10
3
|
return parallel(values, {
|
|
11
4
|
tasks: subtasks.map((subtask) => ({
|
|
@@ -16,12 +9,9 @@ export function work(subtasks, values, opts) {
|
|
|
16
9
|
concurrency: opts.concurrency,
|
|
17
10
|
minSuccess: opts.minSuccess ?? subtasks.length,
|
|
18
11
|
onTask: opts.onWork,
|
|
19
|
-
// `orchestrate` synthesises with the lead rather than merging here, so this
|
|
20
|
-
// only has to hand the results back untouched.
|
|
21
12
|
merge: (results) => results,
|
|
22
13
|
});
|
|
23
14
|
}
|
|
24
|
-
/** The lead's second call as a step. */
|
|
25
15
|
export function synthesisStep(index, outcome) {
|
|
26
16
|
return {
|
|
27
17
|
index,
|
package/dist/parallel/merge.js
CHANGED
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The same task, N times over, for voting.
|
|
3
|
-
*
|
|
4
|
-
* An odd N on purpose — an even one ties, and a tie has no answer that is not
|
|
5
|
-
* arbitrary. Ask for an even number and you get the next one up.
|
|
6
|
-
*/
|
|
7
1
|
export function repeat(name, times, run) {
|
|
8
2
|
const odd = Math.max(1, times % 2 === 0 ? times + 1 : times);
|
|
9
3
|
return Array.from({ length: odd }, (_, at) => ({
|
|
@@ -11,7 +5,6 @@ export function repeat(name, times, run) {
|
|
|
11
5
|
run: (values) => run(values, at),
|
|
12
6
|
}));
|
|
13
7
|
}
|
|
14
|
-
/** How a value is compared when voting. Its own text, or the whole thing. */
|
|
15
8
|
function ballot(value) {
|
|
16
9
|
if (value &&
|
|
17
10
|
typeof value === "object" &&
|
|
@@ -20,16 +13,6 @@ function ballot(value) {
|
|
|
20
13
|
}
|
|
21
14
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
22
15
|
}
|
|
23
|
-
/**
|
|
24
|
-
* The answer most of them gave.
|
|
25
|
-
*
|
|
26
|
-
* Voting is worth the tokens where a model is right most of the time and wrong
|
|
27
|
-
* differently each time — the wrong answers scatter and the right one repeats.
|
|
28
|
-
* It buys nothing where the model is confidently wrong the same way every time,
|
|
29
|
-
* which is the failure it is easiest to mistake it for a fix for.
|
|
30
|
-
*
|
|
31
|
-
* Ties go to the first declared, so the result is the same on every run.
|
|
32
|
-
*/
|
|
33
16
|
export function majority(results) {
|
|
34
17
|
const done = results.filter((r) => r.status === "completed");
|
|
35
18
|
if (done.length === 0)
|
|
@@ -15,16 +15,6 @@ function stringify(value) {
|
|
|
15
15
|
return "";
|
|
16
16
|
return typeof value === "string" ? value : JSON.stringify(value);
|
|
17
17
|
}
|
|
18
|
-
/**
|
|
19
|
-
* Every task at once, then one answer out of what came back.
|
|
20
|
-
*
|
|
21
|
-
* Not compiled to a graph like `route` and `chain`: `@spendgraph/graph` walks one
|
|
22
|
-
* node at a time and picks a single edge, so a fan-out is not a shape it can
|
|
23
|
-
* express. The concurrency comes from `mapLimit`, which already exists.
|
|
24
|
-
*
|
|
25
|
-
* A task that throws is a result, not an exception — the point of running five
|
|
26
|
-
* is that the other four still land.
|
|
27
|
-
*/
|
|
28
18
|
export async function parallel(values, opts) {
|
|
29
19
|
if (opts.tasks.length === 0)
|
|
30
20
|
throw new Error("A parallel run needs at least one task.");
|
package/dist/parallel/task.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
function counted(value) {
|
|
2
2
|
return value && typeof value === "object" ? value : {};
|
|
3
3
|
}
|
|
4
|
-
/** Runs one task. Never throws — a failure is a result, so the others still land. */
|
|
5
4
|
export async function runTask(task, values, emit) {
|
|
6
5
|
try {
|
|
7
6
|
const value = await task.run(values, (text, stage) => emit?.(text, stage ?? task.name));
|
|
@@ -18,13 +17,6 @@ export async function runTask(task, values, emit) {
|
|
|
18
17
|
};
|
|
19
18
|
}
|
|
20
19
|
}
|
|
21
|
-
/**
|
|
22
|
-
* One task as a step.
|
|
23
|
-
*
|
|
24
|
-
* Indexed by where it was declared rather than when it landed: a fan-out that
|
|
25
|
-
* numbered its steps by completion order would record a different run every
|
|
26
|
-
* time for the same work, which makes two runs impossible to compare.
|
|
27
|
-
*/
|
|
28
20
|
export function taskStep(index, result) {
|
|
29
21
|
const { output, model } = counted(result.value);
|
|
30
22
|
return {
|
package/dist/refine/attempt.js
CHANGED
|
@@ -1,19 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* What one attempt consumed.
|
|
3
|
-
*
|
|
4
|
-
* Both halves, because a refine loop's cost is dominated by the input it resends
|
|
5
|
-
* every round — the draft, the critique and the instruction to try again.
|
|
6
|
-
*/
|
|
7
1
|
export function tokensOf(attempt) {
|
|
8
2
|
return (attempt.inputTokens ?? 0) + (attempt.outputTokens ?? 0);
|
|
9
3
|
}
|
|
10
|
-
/**
|
|
11
|
-
* One attempt as a step.
|
|
12
|
-
*
|
|
13
|
-
* Recorded whether it was accepted or not: the rejected drafts are the evidence
|
|
14
|
-
* for whether refining paid at all, and a loop that keeps only the winner cannot
|
|
15
|
-
* say what the answer cost.
|
|
16
|
-
*/
|
|
17
4
|
export function attemptStep(index, round, attempt) {
|
|
18
5
|
return {
|
|
19
6
|
index,
|
|
@@ -26,7 +13,6 @@ export function attemptStep(index, round, attempt) {
|
|
|
26
13
|
outputTokens: attempt.outputTokens,
|
|
27
14
|
};
|
|
28
15
|
}
|
|
29
|
-
/** Both halves of every attempt so far, for the result's totals. */
|
|
30
16
|
export function totalTokens(steps) {
|
|
31
17
|
let inputTokens = 0;
|
|
32
18
|
let outputTokens = 0;
|
package/dist/refine/judge.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
/** One verdict as a step, so a run says why each draft was turned down. */
|
|
2
1
|
export function judgeStep(index, round, verdict) {
|
|
3
2
|
return {
|
|
4
3
|
index,
|
|
@@ -7,16 +6,6 @@ export function judgeStep(index, round, verdict) {
|
|
|
7
6
|
status: "completed",
|
|
8
7
|
};
|
|
9
8
|
}
|
|
10
|
-
/**
|
|
11
|
-
* The attempt to hand back when nothing was accepted.
|
|
12
|
-
*
|
|
13
|
-
* Highest score wins, and the last attempt wins when nothing was scored — a loop
|
|
14
|
-
* that improves each round has its best work at the end, and returning the first
|
|
15
|
-
* draft after three revisions would throw the work away.
|
|
16
|
-
*
|
|
17
|
-
* Here rather than in the loop because the choice is the judge's: it is made on
|
|
18
|
-
* the verdicts, and a judge that never scores gets the fallback.
|
|
19
|
-
*/
|
|
20
9
|
export function best(history) {
|
|
21
10
|
const scored = history.filter((r) => typeof r.verdict.score === "number");
|
|
22
11
|
if (scored.length === 0)
|
package/dist/refine/refine.js
CHANGED
|
@@ -1,14 +1,6 @@
|
|
|
1
1
|
import { attemptStep, tokensOf, totalTokens } from "./attempt.js";
|
|
2
2
|
import { best, judgeStep } from "./judge.js";
|
|
3
3
|
const DEFAULT_ROUNDS = 3;
|
|
4
|
-
/**
|
|
5
|
-
* Draft, judge, revise, until it is good enough or the budget is out.
|
|
6
|
-
*
|
|
7
|
-
* The pattern worth naming when there is a clear criterion and a first draft
|
|
8
|
-
* rarely meets it. Every attempt and every judgement is recorded, the rejected
|
|
9
|
-
* ones included: a loop that keeps only the winner cannot say what the answer
|
|
10
|
-
* cost, and the rejected drafts are the evidence for whether refining paid.
|
|
11
|
-
*/
|
|
12
4
|
export async function refine(opts) {
|
|
13
5
|
const startedAt = Date.now();
|
|
14
6
|
const rounds = Math.max(1, opts.rounds ?? DEFAULT_ROUNDS);
|
package/dist/route/classify.js
CHANGED
|
@@ -1,25 +1,9 @@
|
|
|
1
1
|
import { node } from "@spendgraph/graph";
|
|
2
|
-
/** The classifier's node name, and the key its decision is stored under. */
|
|
3
2
|
export const CLASSIFY = "classify";
|
|
4
|
-
/**
|
|
5
|
-
* The branch that runs when no other one does.
|
|
6
|
-
*
|
|
7
|
-
* Declared here rather than beside the dispatch it names, because `decide` is
|
|
8
|
-
* what produces it — and the other way round leaves the two files importing
|
|
9
|
-
* each other, which is the thing a reader has to stop and untangle.
|
|
10
|
-
*/
|
|
11
3
|
export const FALLBACK = "fallback";
|
|
12
|
-
/** A bare string is shorthand for `{ route }`. */
|
|
13
4
|
export function normalise(value) {
|
|
14
5
|
return typeof value === "string" ? { route: value } : value;
|
|
15
6
|
}
|
|
16
|
-
/**
|
|
17
|
-
* Which branch runs.
|
|
18
|
-
*
|
|
19
|
-
* A class no branch handles falls back, and so does one the classifier is not
|
|
20
|
-
* sure enough about — a hedged guess sent to the wrong specialist is worse than
|
|
21
|
-
* the general path.
|
|
22
|
-
*/
|
|
23
7
|
export function decide(said, names, minConfidence) {
|
|
24
8
|
const known = names.includes(said.route);
|
|
25
9
|
const confident = minConfidence === undefined ||
|
|
@@ -32,13 +16,6 @@ export function decide(said, names, minConfidence) {
|
|
|
32
16
|
reason: said.reason,
|
|
33
17
|
};
|
|
34
18
|
}
|
|
35
|
-
/**
|
|
36
|
-
* The entry node.
|
|
37
|
-
*
|
|
38
|
-
* Its return is what the edges read, so the decision travels through the graph's
|
|
39
|
-
* own context rather than a variable closed over from outside — which means a
|
|
40
|
-
* finished run can be read apart from `outputs.classify` alone.
|
|
41
|
-
*/
|
|
42
19
|
export function classifier(opts, names) {
|
|
43
20
|
return node({
|
|
44
21
|
name: CLASSIFY,
|
|
@@ -46,7 +23,6 @@ export function classifier(opts, names) {
|
|
|
46
23
|
const said = normalise(await opts.classify(input, ctx));
|
|
47
24
|
return {
|
|
48
25
|
...decide(said, names, opts.minConfidence),
|
|
49
|
-
// The step records what the classifier said, not what we did about it.
|
|
50
26
|
output: said.route,
|
|
51
27
|
model: said.model,
|
|
52
28
|
inputTokens: said.inputTokens,
|
package/dist/route/dispatch.js
CHANGED
|
@@ -6,7 +6,6 @@ function handler(name, run) {
|
|
|
6
6
|
run: (input, ctx) => run(input, ctx),
|
|
7
7
|
});
|
|
8
8
|
}
|
|
9
|
-
/** One node per branch, plus the fallback. */
|
|
10
9
|
export function branches(opts, names) {
|
|
11
10
|
return [
|
|
12
11
|
...names.map((name) => handler(name, opts.routes[name])),
|
|
@@ -16,13 +15,6 @@ export function branches(opts, names) {
|
|
|
16
15
|
function taken(ctx) {
|
|
17
16
|
return ctx.outputs.classify?.taken ?? FALLBACK;
|
|
18
17
|
}
|
|
19
|
-
/**
|
|
20
|
-
* Classify, then exactly one branch, then stop.
|
|
21
|
-
*
|
|
22
|
-
* Conditional edges first and the fallback unconditional last, because `graph`
|
|
23
|
-
* refuses the other order — an unconditional edge ahead of a conditional one is
|
|
24
|
-
* a router whose default hides a branch, and it would never be reached.
|
|
25
|
-
*/
|
|
26
18
|
export function edgesTo(names) {
|
|
27
19
|
return [
|
|
28
20
|
...names.map((name) => edge("classify", name, (ctx) => taken(ctx) === name)),
|
package/dist/route/route.js
CHANGED
|
@@ -2,15 +2,6 @@ import { graph } from "@spendgraph/graph";
|
|
|
2
2
|
import { runGraph } from "../stream/graph.js";
|
|
3
3
|
import { CLASSIFY, classifier, FALLBACK } from "./classify.js";
|
|
4
4
|
import { branches, edgesTo } from "./dispatch.js";
|
|
5
|
-
/**
|
|
6
|
-
* Classify once, then hand the work to the branch built for it.
|
|
7
|
-
*
|
|
8
|
-
* Compiled to a graph rather than looped here: `@spendgraph/graph` already
|
|
9
|
-
* validates that every branch is reachable and that no default sits in front of
|
|
10
|
-
* one, already records a step per node with its tokens, and already returns the
|
|
11
|
-
* rollout shape. A second execution engine in this package would be a second
|
|
12
|
-
* thing to keep right.
|
|
13
|
-
*/
|
|
14
5
|
export async function route(values, opts) {
|
|
15
6
|
const names = Object.keys(opts.routes);
|
|
16
7
|
const flow = graph({
|
package/dist/stream/graph.js
CHANGED
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Runs a compiled graph, forwarding what its nodes emit.
|
|
3
|
-
*
|
|
4
|
-
* Streamed only when someone is listening: an unread stream is a queue nothing
|
|
5
|
-
* drains, and `execute` is the same run without one.
|
|
6
|
-
*/
|
|
7
1
|
export async function runGraph(flow, values, emit) {
|
|
8
2
|
if (!emit)
|
|
9
3
|
return flow.execute(values);
|
package/dist/stream/stream.js
CHANGED
|
@@ -28,23 +28,6 @@ class Events {
|
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
|
-
/**
|
|
32
|
-
* Any workflow, watched while it runs.
|
|
33
|
-
*
|
|
34
|
-
* One helper rather than a streaming variant of each of the seven: every
|
|
35
|
-
* workflow already takes an `emit`, so wrapping the call is all it takes, and a
|
|
36
|
-
* `stream()` per pattern would be seven places for the same queue to go wrong.
|
|
37
|
-
*
|
|
38
|
-
* const run = streamed((emit, note) =>
|
|
39
|
-
* cascade(values, { ...opts, emit, onTier: note })
|
|
40
|
-
* );
|
|
41
|
-
* for await (const event of run) { ... }
|
|
42
|
-
* const result = await run.result;
|
|
43
|
-
*
|
|
44
|
-
* The run starts on the call, not on the first read, so a caller who wants only
|
|
45
|
-
* the result never has to iterate, and a client that connects late still sees
|
|
46
|
-
* what already happened.
|
|
47
|
-
*/
|
|
48
31
|
export function streamed(run) {
|
|
49
32
|
const events = new Events();
|
|
50
33
|
const result = (async () => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spendgraph/harness",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "Workflows: the shapes an LLM app takes, built on prompts, tools and graphs.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -32,14 +32,14 @@
|
|
|
32
32
|
"README.md"
|
|
33
33
|
],
|
|
34
34
|
"scripts": {
|
|
35
|
-
"build": "tsc -p tsconfig.json",
|
|
35
|
+
"build": "tsc -p tsconfig.json --emitDeclarationOnly && tsc -p tsconfig.json --declaration false --removeComments",
|
|
36
36
|
"prebuild": "npm run build --workspace @spendgraph/graph --workspace @spendgraph/tools --workspace @spendgraph/prompt",
|
|
37
37
|
"test": "vitest run",
|
|
38
38
|
"pretest": "npm run build --workspace @spendgraph/graph --workspace @spendgraph/prompt --workspace @spendgraph/tools --workspace @spendgraph/llms",
|
|
39
39
|
"examples": "npm run build && node examples/run-all.mjs"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
|
-
"@spendgraph/llms": "^0.2.
|
|
42
|
+
"@spendgraph/llms": "^0.2.1",
|
|
43
43
|
"typescript": "^5"
|
|
44
44
|
},
|
|
45
45
|
"engines": {
|
|
@@ -49,9 +49,9 @@
|
|
|
49
49
|
"access": "public"
|
|
50
50
|
},
|
|
51
51
|
"dependencies": {
|
|
52
|
-
"@spendgraph/graph": "^0.2.
|
|
53
|
-
"@spendgraph/prompt": "^0.2.
|
|
54
|
-
"@spendgraph/sdk": "^0.2.
|
|
55
|
-
"@spendgraph/tools": "^0.2.
|
|
52
|
+
"@spendgraph/graph": "^0.2.1",
|
|
53
|
+
"@spendgraph/prompt": "^0.2.1",
|
|
54
|
+
"@spendgraph/sdk": "^0.2.1",
|
|
55
|
+
"@spendgraph/tools": "^0.2.1"
|
|
56
56
|
}
|
|
57
57
|
}
|