@spendgraph/harness 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -160
- package/dist/cascade/cascade.d.ts +13 -0
- package/dist/cascade/cascade.js +91 -0
- package/dist/cascade/index.d.ts +3 -0
- package/dist/cascade/index.js +2 -0
- package/dist/cascade/tier.d.ts +31 -0
- package/dist/cascade/tier.js +53 -0
- package/dist/cascade/types.d.ts +98 -0
- package/dist/chain/chain.d.ts +9 -0
- package/dist/chain/chain.js +58 -0
- package/dist/chain/gate.d.ts +16 -0
- package/dist/chain/gate.js +33 -0
- package/dist/chain/index.d.ts +3 -0
- package/dist/chain/index.js +2 -0
- package/dist/chain/step.d.ts +9 -0
- package/dist/chain/step.js +14 -0
- package/dist/chain/types.d.ts +50 -0
- package/dist/index.d.ts +16 -9
- package/dist/index.js +8 -7
- package/dist/loop/act.d.ts +21 -0
- package/dist/loop/act.js +43 -0
- package/dist/loop/compact.d.ts +49 -0
- package/dist/loop/compact.js +59 -0
- package/dist/loop/hooks.d.ts +43 -0
- package/dist/loop/hooks.js +41 -0
- package/dist/loop/index.d.ts +6 -0
- package/dist/loop/index.js +5 -0
- package/dist/loop/loop.d.ts +13 -0
- package/dist/loop/loop.js +120 -0
- package/dist/loop/turn.d.ts +6 -0
- package/dist/loop/turn.js +17 -0
- package/dist/loop/types.d.ts +140 -0
- package/dist/loop/types.js +1 -0
- package/dist/orchestrate/index.d.ts +4 -0
- package/dist/orchestrate/index.js +3 -0
- package/dist/orchestrate/orchestrate.d.ts +12 -0
- package/dist/orchestrate/orchestrate.js +68 -0
- package/dist/orchestrate/plan.d.ts +18 -0
- package/dist/orchestrate/plan.js +38 -0
- package/dist/orchestrate/types.d.ts +75 -0
- package/dist/orchestrate/types.js +1 -0
- package/dist/orchestrate/work.d.ts +13 -0
- package/dist/orchestrate/work.js +35 -0
- package/dist/parallel/index.d.ts +4 -0
- package/dist/parallel/index.js +3 -0
- package/dist/parallel/merge.d.ts +26 -0
- package/dist/parallel/merge.js +52 -0
- package/dist/parallel/parallel.d.ts +12 -0
- package/dist/parallel/parallel.js +66 -0
- package/dist/parallel/task.d.ts +13 -0
- package/dist/parallel/task.js +45 -0
- package/dist/parallel/types.d.ts +61 -0
- package/dist/parallel/types.js +1 -0
- package/dist/refine/attempt.d.ts +22 -0
- package/dist/refine/attempt.js +38 -0
- package/dist/refine/index.d.ts +4 -0
- package/dist/refine/index.js +3 -0
- package/dist/refine/judge.d.ts +15 -0
- package/dist/refine/judge.js +25 -0
- package/dist/refine/refine.d.ts +10 -0
- package/dist/refine/refine.js +67 -0
- package/dist/refine/types.d.ts +90 -0
- package/dist/refine/types.js +1 -0
- package/dist/route/classify.d.ts +52 -0
- package/dist/route/classify.js +57 -0
- package/dist/route/dispatch.d.ts +12 -0
- package/dist/route/dispatch.js +33 -0
- package/dist/route/index.d.ts +3 -0
- package/dist/route/index.js +2 -0
- package/dist/route/route.d.ts +11 -0
- package/dist/route/route.js +31 -0
- package/dist/route/types.d.ts +53 -0
- package/dist/route/types.js +1 -0
- package/dist/stream/graph.d.ts +9 -0
- package/dist/stream/graph.js +16 -0
- package/dist/stream/index.d.ts +2 -0
- package/dist/stream/index.js +1 -0
- package/dist/stream/stream.d.ts +19 -0
- package/dist/stream/stream.js +65 -0
- package/dist/stream/types.d.ts +31 -0
- package/dist/stream/types.js +1 -0
- package/docs/README.md +41 -0
- package/docs/cascade.md +165 -0
- package/docs/chain.md +134 -0
- package/docs/loop.md +318 -0
- package/docs/orchestrate.md +128 -0
- package/docs/parallel.md +139 -0
- package/docs/refine.md +206 -0
- package/docs/route.md +212 -0
- package/package.json +18 -14
- package/bin/harness.mjs +0 -76
- package/dist/core/budget.d.ts +0 -46
- package/dist/core/budget.js +0 -72
- package/dist/core/cache.d.ts +0 -119
- package/dist/core/cache.js +0 -206
- package/dist/core/client.d.ts +0 -33
- package/dist/core/client.js +0 -98
- package/dist/core/errors.d.ts +0 -22
- package/dist/core/errors.js +0 -28
- package/dist/core/index.d.ts +0 -5
- package/dist/core/index.js +0 -5
- package/dist/core/pull.d.ts +0 -17
- package/dist/core/pull.js +0 -38
- package/dist/core/schema/serialize.d.ts +0 -12
- package/dist/core/schema/serialize.js +0 -42
- package/dist/core/schema/types.d.ts +0 -58
- package/dist/core/schema/validate.d.ts +0 -28
- package/dist/core/schema/validate.js +0 -98
- package/dist/harness.d.ts +0 -128
- package/dist/harness.js +0 -187
- package/dist/prompt/codegen.d.ts +0 -37
- package/dist/prompt/codegen.js +0 -97
- package/dist/prompt/render.d.ts +0 -35
- package/dist/prompt/render.js +0 -66
- package/dist/prompt/run.d.ts +0 -38
- package/dist/prompt/run.js +0 -96
- package/dist/prompt/types.d.ts +0 -156
- /package/dist/{core/schema → cascade}/types.js +0 -0
- /package/dist/{prompt → chain}/types.js +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { node } from "@spendgraph/graph";
|
|
2
|
+
/** The gate node's name for a stage, and the key its verdict is stored under. */
|
|
3
|
+
export function gateName(step) {
|
|
4
|
+
return `gate_${step}`;
|
|
5
|
+
}
|
|
6
|
+
export class GateRefused extends Error {
|
|
7
|
+
step;
|
|
8
|
+
reason;
|
|
9
|
+
constructor(step, reason) {
|
|
10
|
+
super(`"${step}" did not pass its gate: ${reason}`);
|
|
11
|
+
this.step = step;
|
|
12
|
+
this.reason = reason;
|
|
13
|
+
this.name = "GateRefused";
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* A gate as its own node, so a refusal is a step of its own.
|
|
18
|
+
*
|
|
19
|
+
* Folding the check into the stage would lose the output it refused — and the
|
|
20
|
+
* rejected output is the evidence for why the gate fired at all. This way the
|
|
21
|
+
* stage records what it produced and the gate records what was wrong with it.
|
|
22
|
+
*/
|
|
23
|
+
export function gateNode(step) {
|
|
24
|
+
return node({
|
|
25
|
+
name: gateName(step.name),
|
|
26
|
+
run: async (_values, ctx) => {
|
|
27
|
+
const verdict = await step.gate?.(ctx.outputs[step.name], ctx);
|
|
28
|
+
if (!verdict || verdict.ok)
|
|
29
|
+
return verdict ?? { ok: true };
|
|
30
|
+
throw new GateRefused(step.name, verdict.reason ?? "no reason given");
|
|
31
|
+
},
|
|
32
|
+
});
|
|
33
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Step } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* A stage as a node.
|
|
4
|
+
*
|
|
5
|
+
* `previous` is read from the context rather than threaded through the input,
|
|
6
|
+
* because a gate may sit between two stages and its return is not what the next
|
|
7
|
+
* stage wants — the stage before it is.
|
|
8
|
+
*/
|
|
9
|
+
export declare function stepNode(step: Step, previous: string | null): import("@spendgraph/graph").Node<import("@spendgraph/graph").ArgsOf<readonly import("@spendgraph/graph").ArgSpec[]>>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { node } from "@spendgraph/graph";
|
|
2
|
+
/**
|
|
3
|
+
* A stage as a node.
|
|
4
|
+
*
|
|
5
|
+
* `previous` is read from the context rather than threaded through the input,
|
|
6
|
+
* because a gate may sit between two stages and its return is not what the next
|
|
7
|
+
* stage wants — the stage before it is.
|
|
8
|
+
*/
|
|
9
|
+
export function stepNode(step, previous) {
|
|
10
|
+
return node({
|
|
11
|
+
name: step.name,
|
|
12
|
+
run: (values, ctx) => step.run(previous ? ctx.outputs[previous] : values, ctx),
|
|
13
|
+
});
|
|
14
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { GraphContext, GraphResult } from "@spendgraph/graph";
|
|
2
|
+
import type { Emit } from "../stream/index.js";
|
|
3
|
+
/** Whether the chain may continue, and why not when it may not. */
|
|
4
|
+
export interface Gate {
|
|
5
|
+
ok: boolean;
|
|
6
|
+
/** Said in the record and in the error. A failed gate with no reason is a mystery. */
|
|
7
|
+
reason?: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* One stage.
|
|
11
|
+
*
|
|
12
|
+
* `run` is handed whatever the previous stage returned — the values the chain
|
|
13
|
+
* was called with, for the first one.
|
|
14
|
+
*/
|
|
15
|
+
export interface Step {
|
|
16
|
+
/** Unique in the chain, and the key its output is stored under. */
|
|
17
|
+
name: string;
|
|
18
|
+
run(previous: unknown, ctx: GraphContext): Promise<unknown> | unknown;
|
|
19
|
+
/**
|
|
20
|
+
* Whether that will do to continue on.
|
|
21
|
+
*
|
|
22
|
+
* The reason this pattern is worth naming: without a gate, a stage that
|
|
23
|
+
* produced nothing useful is passed to the next one, which confidently works
|
|
24
|
+
* on it, and you get a polished answer three calls later that is about
|
|
25
|
+
* nothing. A gate stops at the stage that went wrong and says which.
|
|
26
|
+
*/
|
|
27
|
+
gate?(output: unknown, ctx: GraphContext): Promise<Gate> | Gate;
|
|
28
|
+
}
|
|
29
|
+
export interface ChainOptions {
|
|
30
|
+
steps: Step[];
|
|
31
|
+
/**
|
|
32
|
+
* Where the tokens go while this runs.
|
|
33
|
+
*
|
|
34
|
+
* Passed down to whatever makes the model call; omit it and nothing is
|
|
35
|
+
* emitted. `streamed()` supplies one.
|
|
36
|
+
*/
|
|
37
|
+
emit?: Emit;
|
|
38
|
+
/** Ceiling on total steps, nested ones included. Default 25. */
|
|
39
|
+
maxSteps?: number;
|
|
40
|
+
/** Injectable clock, so tests do not measure real time. */
|
|
41
|
+
now?: () => number;
|
|
42
|
+
}
|
|
43
|
+
export interface ChainResult extends GraphResult {
|
|
44
|
+
/** The stages that ran, in order. */
|
|
45
|
+
ran: string[];
|
|
46
|
+
/** The stage whose gate refused, or which threw. Null when the chain finished. */
|
|
47
|
+
stoppedAt: string | null;
|
|
48
|
+
/** What the gate said. */
|
|
49
|
+
reason?: string;
|
|
50
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export {
|
|
3
|
-
export type {
|
|
4
|
-
export {
|
|
5
|
-
export {
|
|
6
|
-
export {
|
|
7
|
-
export {
|
|
8
|
-
export {
|
|
9
|
-
export type {
|
|
1
|
+
export type { Answer, CascadeOptions, CascadeResult, Escalation, Tier, TierRecord, } from "./cascade/index.js";
|
|
2
|
+
export { cascade } from "./cascade/index.js";
|
|
3
|
+
export type { ChainOptions, ChainResult, Gate, Step } from "./chain/index.js";
|
|
4
|
+
export { chain, GateRefused } from "./chain/index.js";
|
|
5
|
+
export type { Call, Compaction, Hooks, Invoker, LoopEnd, LoopOptions, LoopResult, Refusal, Remaining, Summary, Turn, TurnRecord, TurnState, } from "./loop/index.js";
|
|
6
|
+
export { loop } from "./loop/index.js";
|
|
7
|
+
export type { OrchestrateEnd, OrchestrateOptions, OrchestrateResult, Outcome, Plan, Subtask, } from "./orchestrate/index.js";
|
|
8
|
+
export { orchestrate } from "./orchestrate/index.js";
|
|
9
|
+
export type { ParallelOptions, ParallelResult, Task, TaskResult } from "./parallel/index.js";
|
|
10
|
+
export { majority, parallel, repeat, type Vote } from "./parallel/index.js";
|
|
11
|
+
export type { Attempt, RefineOptions, RefineResult, RoundRecord, Verdict } from "./refine/index.js";
|
|
12
|
+
export { refine } from "./refine/index.js";
|
|
13
|
+
export type { Classification, Handler, RouteOptions, RouteResult } from "./route/index.js";
|
|
14
|
+
export { route } from "./route/index.js";
|
|
15
|
+
export type { Emit, HarnessEvent, HarnessStream } from "./stream/index.js";
|
|
16
|
+
export { streamed } from "./stream/index.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export {
|
|
3
|
-
export {
|
|
4
|
-
export {
|
|
5
|
-
export {
|
|
6
|
-
export {
|
|
7
|
-
export {
|
|
1
|
+
export { cascade } from "./cascade/index.js";
|
|
2
|
+
export { chain, GateRefused } from "./chain/index.js";
|
|
3
|
+
export { loop } from "./loop/index.js";
|
|
4
|
+
export { orchestrate } from "./orchestrate/index.js";
|
|
5
|
+
export { majority, parallel, repeat } from "./parallel/index.js";
|
|
6
|
+
export { refine } from "./refine/index.js";
|
|
7
|
+
export { route } from "./route/index.js";
|
|
8
|
+
export { streamed } from "./stream/index.js";
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { RolloutStep } from "@spendgraph/sdk";
|
|
2
|
+
import { type ToolResult } from "@spendgraph/tools";
|
|
3
|
+
import { type Hooks } from "./hooks.js";
|
|
4
|
+
import type { Call, Invoker } from "./types.js";
|
|
5
|
+
/** One tool call as a step, failures included. */
|
|
6
|
+
export declare function callStep(index: number, result: ToolResult): RolloutStep;
|
|
7
|
+
/**
|
|
8
|
+
* Runs what the model asked for, in the order it asked.
|
|
9
|
+
*
|
|
10
|
+
* Never throws — `invoke` returns a failed result rather than raising, so a
|
|
11
|
+
* tool that fell over is something the model can read and try around, not
|
|
12
|
+
* something that ends the run.
|
|
13
|
+
*/
|
|
14
|
+
export declare function act(tools: Invoker, calls: Call[], hooks?: Hooks): Promise<ToolResult[]>;
|
|
15
|
+
/**
|
|
16
|
+
* The answer, when the model said it was done.
|
|
17
|
+
*
|
|
18
|
+
* Reading `finish` here is what separates "it is done" from "it stopped asking",
|
|
19
|
+
* which otherwise look identical from outside and are opposite problems.
|
|
20
|
+
*/
|
|
21
|
+
export declare function finished(results: ToolResult[]): string | null;
|
package/dist/loop/act.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { answerOf, isFinished } from "@spendgraph/tools";
|
|
2
|
+
import { ask, blockedResult, tell } from "./hooks.js";
|
|
3
|
+
/** One tool call as a step, failures included. */
|
|
4
|
+
export function callStep(index, result) {
|
|
5
|
+
return {
|
|
6
|
+
index,
|
|
7
|
+
source: result.name,
|
|
8
|
+
input: result.args,
|
|
9
|
+
output: result.output,
|
|
10
|
+
status: result.status,
|
|
11
|
+
error: result.error,
|
|
12
|
+
latencyMs: result.latencyMs,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Runs what the model asked for, in the order it asked.
|
|
17
|
+
*
|
|
18
|
+
* Never throws — `invoke` returns a failed result rather than raising, so a
|
|
19
|
+
* tool that fell over is something the model can read and try around, not
|
|
20
|
+
* something that ends the run.
|
|
21
|
+
*/
|
|
22
|
+
export async function act(tools, calls, hooks = {}) {
|
|
23
|
+
const results = [];
|
|
24
|
+
for (const call of calls) {
|
|
25
|
+
const refused = await ask(hooks.beforeTool, call, tools.effectOf?.(call.name));
|
|
26
|
+
const result = refused
|
|
27
|
+
? blockedResult(call, refused)
|
|
28
|
+
: await tools.invoke(call.name, call.args);
|
|
29
|
+
await tell(hooks.afterTool, result);
|
|
30
|
+
results.push(result);
|
|
31
|
+
}
|
|
32
|
+
return results;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The answer, when the model said it was done.
|
|
36
|
+
*
|
|
37
|
+
* Reading `finish` here is what separates "it is done" from "it stopped asking",
|
|
38
|
+
* which otherwise look identical from outside and are opposite problems.
|
|
39
|
+
*/
|
|
40
|
+
export function finished(results) {
|
|
41
|
+
const done = results.find((result) => isFinished(result));
|
|
42
|
+
return done ? answerOf(done) : null;
|
|
43
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { RolloutStep } from "@spendgraph/sdk";
|
|
2
|
+
import type { ToolResult } from "@spendgraph/tools";
|
|
3
|
+
import type { Turn } from "./types.js";
|
|
4
|
+
/** What the summariser produced, and what producing it cost. */
|
|
5
|
+
export interface Summary {
|
|
6
|
+
output: string;
|
|
7
|
+
model?: string;
|
|
8
|
+
inputTokens?: number;
|
|
9
|
+
outputTokens?: number;
|
|
10
|
+
}
|
|
11
|
+
export interface Compaction {
|
|
12
|
+
/**
|
|
13
|
+
* Fraction of `maxTokens` the resent context may reach before compacting.
|
|
14
|
+
* Default 0.7.
|
|
15
|
+
*/
|
|
16
|
+
at?: number;
|
|
17
|
+
/** Compact once there are more results than this, whatever the tokens say. */
|
|
18
|
+
after?: number;
|
|
19
|
+
/** Most recent results kept verbatim. Default 4. */
|
|
20
|
+
keep?: number;
|
|
21
|
+
/** Turns the older results into one. */
|
|
22
|
+
summarise(older: ToolResult[]): Promise<Summary> | Summary;
|
|
23
|
+
}
|
|
24
|
+
declare const DEFAULT_KEEP = 4;
|
|
25
|
+
/**
|
|
26
|
+
* Whether the context is big enough to be worth summarising.
|
|
27
|
+
*
|
|
28
|
+
* Measured on the **last turn's input tokens**, not on what the run has spent.
|
|
29
|
+
* Spend only ever rises, so a threshold on it fires once and then every turn
|
|
30
|
+
* after; the input count is the size of what is actually being resent, and it
|
|
31
|
+
* falls the moment a compaction lands — which is what stops this firing twice
|
|
32
|
+
* for the same reason.
|
|
33
|
+
*/
|
|
34
|
+
export declare function shouldCompact(compact: Compaction, last: Turn | null, results: ToolResult[], maxTokens?: number): boolean;
|
|
35
|
+
/** The summary as a tool result, so the next turn reads it like any other. */
|
|
36
|
+
export declare function summaryResult(summary: Summary, replaced: number): ToolResult;
|
|
37
|
+
/** The summarising call as a step, because deciding what to forget is not free. */
|
|
38
|
+
export declare function compactStep(index: number, summary: Summary, replaced: number): RolloutStep;
|
|
39
|
+
/**
|
|
40
|
+
* Replaces the older results with one summary, in place.
|
|
41
|
+
*
|
|
42
|
+
* The most recent `keep` survive verbatim: summarising what just happened is
|
|
43
|
+
* how a loop forgets what it was in the middle of doing.
|
|
44
|
+
*/
|
|
45
|
+
export declare function applyCompaction(results: ToolResult[], summary: Summary, keep: number): {
|
|
46
|
+
replaced: number;
|
|
47
|
+
summarised: ToolResult;
|
|
48
|
+
};
|
|
49
|
+
export { DEFAULT_KEEP };
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
const DEFAULT_AT = 0.7;
|
|
2
|
+
const DEFAULT_KEEP = 4;
|
|
3
|
+
/**
|
|
4
|
+
* Whether the context is big enough to be worth summarising.
|
|
5
|
+
*
|
|
6
|
+
* Measured on the **last turn's input tokens**, not on what the run has spent.
|
|
7
|
+
* Spend only ever rises, so a threshold on it fires once and then every turn
|
|
8
|
+
* after; the input count is the size of what is actually being resent, and it
|
|
9
|
+
* falls the moment a compaction lands — which is what stops this firing twice
|
|
10
|
+
* for the same reason.
|
|
11
|
+
*/
|
|
12
|
+
export function shouldCompact(compact, last, results, maxTokens) {
|
|
13
|
+
const keep = compact.keep ?? DEFAULT_KEEP;
|
|
14
|
+
// Nothing to gain: everything left would be kept anyway.
|
|
15
|
+
if (results.length <= keep)
|
|
16
|
+
return false;
|
|
17
|
+
if (compact.after !== undefined && results.length > compact.after)
|
|
18
|
+
return true;
|
|
19
|
+
if (maxTokens === undefined || last === null)
|
|
20
|
+
return false;
|
|
21
|
+
const context = last.inputTokens ?? 0;
|
|
22
|
+
return context >= (compact.at ?? DEFAULT_AT) * maxTokens;
|
|
23
|
+
}
|
|
24
|
+
/** The summary as a tool result, so the next turn reads it like any other. */
|
|
25
|
+
export function summaryResult(summary, replaced) {
|
|
26
|
+
return {
|
|
27
|
+
name: "compacted",
|
|
28
|
+
args: { replaced },
|
|
29
|
+
status: "completed",
|
|
30
|
+
output: summary.output,
|
|
31
|
+
latencyMs: 0,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/** The summarising call as a step, because deciding what to forget is not free. */
|
|
35
|
+
export function compactStep(index, summary, replaced) {
|
|
36
|
+
return {
|
|
37
|
+
index,
|
|
38
|
+
source: "compact",
|
|
39
|
+
input: { replaced },
|
|
40
|
+
model: summary.model,
|
|
41
|
+
output: summary.output,
|
|
42
|
+
status: "completed",
|
|
43
|
+
inputTokens: summary.inputTokens,
|
|
44
|
+
outputTokens: summary.outputTokens,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Replaces the older results with one summary, in place.
|
|
49
|
+
*
|
|
50
|
+
* The most recent `keep` survive verbatim: summarising what just happened is
|
|
51
|
+
* how a loop forgets what it was in the middle of doing.
|
|
52
|
+
*/
|
|
53
|
+
export function applyCompaction(results, summary, keep) {
|
|
54
|
+
const older = results.splice(0, results.length - keep);
|
|
55
|
+
const summarised = summaryResult(summary, older.length);
|
|
56
|
+
results.unshift(summarised);
|
|
57
|
+
return { replaced: older.length, summarised };
|
|
58
|
+
}
|
|
59
|
+
export { DEFAULT_KEEP };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Effect, ToolResult } from "@spendgraph/tools";
|
|
2
|
+
import type { Call, Turn, TurnState } from "./types.js";
|
|
3
|
+
/** Return a reason to refuse; return nothing to allow. */
|
|
4
|
+
export type Refusal = {
|
|
5
|
+
block: string;
|
|
6
|
+
} | undefined | void;
|
|
7
|
+
/**
|
|
8
|
+
* Deterministic code at fixed points in the loop.
|
|
9
|
+
*
|
|
10
|
+
* A guardrail written into the prompt is a request. This is enforcement: it
|
|
11
|
+
* runs whether or not the model read the instruction, and it cannot be argued
|
|
12
|
+
* out of by the next turn's wording.
|
|
13
|
+
*/
|
|
14
|
+
export interface Hooks {
|
|
15
|
+
/** Before a model call. Blocking here ends the run — a turn has no way around it. */
|
|
16
|
+
beforeTurn?(state: TurnState): Promise<Refusal> | Refusal;
|
|
17
|
+
/**
|
|
18
|
+
* Before a tool runs. Blocking here is a failed result the model can work
|
|
19
|
+
* around.
|
|
20
|
+
*
|
|
21
|
+
* `effect` is what the tool declared, and `undefined` when it declared
|
|
22
|
+
* nothing. Gate on it rather than on `call.name`: a rule written against a
|
|
23
|
+
* name stops firing the day the tool is renamed, silently and in the
|
|
24
|
+
* permissive direction.
|
|
25
|
+
*/
|
|
26
|
+
beforeTool?(call: Call, effect: Effect | undefined): Promise<Refusal> | Refusal;
|
|
27
|
+
/** After a tool ran. Observation only — the audit trail's natural home. */
|
|
28
|
+
afterTool?(result: ToolResult): Promise<void> | void;
|
|
29
|
+
/** After a model call. Observation only. */
|
|
30
|
+
afterTurn?(turn: Turn): Promise<void> | void;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Runs a hook that can refuse, treating its own failure as permission.
|
|
34
|
+
*
|
|
35
|
+
* A hook that throws is a bug in the guardrail, and taking the run down with it
|
|
36
|
+
* is worse than the thing it was guarding against — the call it was checking
|
|
37
|
+
* has not happened yet, and the next hook still gets its say.
|
|
38
|
+
*/
|
|
39
|
+
export declare function ask(hook: ((...args: never[]) => Promise<Refusal> | Refusal) | undefined, ...args: unknown[]): Promise<string | null>;
|
|
40
|
+
/** Runs an observing hook. Its failure changes nothing; it was only watching. */
|
|
41
|
+
export declare function tell(hook: ((arg: never) => Promise<void> | void) | undefined, arg: unknown): Promise<void>;
|
|
42
|
+
/** A refused call, in the shape the model already knows how to read. */
|
|
43
|
+
export declare function blockedResult(call: Call, reason: string): ToolResult;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs a hook that can refuse, treating its own failure as permission.
|
|
3
|
+
*
|
|
4
|
+
* A hook that throws is a bug in the guardrail, and taking the run down with it
|
|
5
|
+
* is worse than the thing it was guarding against — the call it was checking
|
|
6
|
+
* has not happened yet, and the next hook still gets its say.
|
|
7
|
+
*/
|
|
8
|
+
export async function ask(hook, ...args) {
|
|
9
|
+
if (!hook)
|
|
10
|
+
return null;
|
|
11
|
+
try {
|
|
12
|
+
const call = hook;
|
|
13
|
+
const verdict = await call(...args);
|
|
14
|
+
return verdict?.block ?? null;
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** Runs an observing hook. Its failure changes nothing; it was only watching. */
|
|
21
|
+
export async function tell(hook, arg) {
|
|
22
|
+
if (!hook)
|
|
23
|
+
return;
|
|
24
|
+
try {
|
|
25
|
+
await hook(arg);
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
// Observation only.
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** A refused call, in the shape the model already knows how to read. */
|
|
32
|
+
export function blockedResult(call, reason) {
|
|
33
|
+
return {
|
|
34
|
+
name: call.name,
|
|
35
|
+
args: call.args,
|
|
36
|
+
status: "failed",
|
|
37
|
+
output: "",
|
|
38
|
+
error: `blocked: ${reason}`,
|
|
39
|
+
latencyMs: 0,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { act, callStep, finished } from "./act.js";
|
|
2
|
+
export { applyCompaction, type Compaction, type Summary, shouldCompact } from "./compact.js";
|
|
3
|
+
export { blockedResult, type Hooks, type Refusal } from "./hooks.js";
|
|
4
|
+
export { loop } from "./loop.js";
|
|
5
|
+
export { tokensOf, turnStep } from "./turn.js";
|
|
6
|
+
export type { Call, Invoker, LoopEnd, LoopOptions, LoopResult, Remaining, Turn, TurnRecord, TurnState, } from "./types.js";
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { LoopOptions, LoopResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Model, tools, model again, until it stops asking.
|
|
4
|
+
*
|
|
5
|
+
* Not compiled to a graph: this is a cycle, and `@spendgraph/graph` is a DAG
|
|
6
|
+
* whose step ceiling exists to stop one. Same reasoning as `refine`.
|
|
7
|
+
*
|
|
8
|
+
* What this adds over a provider SDK's own tool loop is the part that is always
|
|
9
|
+
* hand-rolled and always wrong the same way: a ceiling that is checked before
|
|
10
|
+
* the spend rather than after, every turn and every tool call recorded, and
|
|
11
|
+
* `finished` told apart from `quiet`.
|
|
12
|
+
*/
|
|
13
|
+
export declare function loop(opts: LoopOptions): Promise<LoopResult>;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { act, callStep, finished } from "./act.js";
|
|
2
|
+
import { applyCompaction, compactStep, DEFAULT_KEEP, shouldCompact } from "./compact.js";
|
|
3
|
+
import { ask, tell } from "./hooks.js";
|
|
4
|
+
import { tokensOf, turnStep } from "./turn.js";
|
|
5
|
+
const DEFAULT_MAX_TURNS = 8;
|
|
6
|
+
/**
|
|
7
|
+
* Model, tools, model again, until it stops asking.
|
|
8
|
+
*
|
|
9
|
+
* Not compiled to a graph: this is a cycle, and `@spendgraph/graph` is a DAG
|
|
10
|
+
* whose step ceiling exists to stop one. Same reasoning as `refine`.
|
|
11
|
+
*
|
|
12
|
+
* What this adds over a provider SDK's own tool loop is the part that is always
|
|
13
|
+
* hand-rolled and always wrong the same way: a ceiling that is checked before
|
|
14
|
+
* the spend rather than after, every turn and every tool call recorded, and
|
|
15
|
+
* `finished` told apart from `quiet`.
|
|
16
|
+
*/
|
|
17
|
+
export async function loop(opts) {
|
|
18
|
+
const startedAt = Date.now();
|
|
19
|
+
const maxTurns = Math.max(1, opts.maxTurns ?? DEFAULT_MAX_TURNS);
|
|
20
|
+
const steps = [];
|
|
21
|
+
const history = [];
|
|
22
|
+
const results = [];
|
|
23
|
+
let last = null;
|
|
24
|
+
let spent = 0;
|
|
25
|
+
let answer = null;
|
|
26
|
+
let stoppedBy = "turns";
|
|
27
|
+
const compactions = { count: 0, replaced: 0 };
|
|
28
|
+
/**
|
|
29
|
+
* Summarise the older results, if the context has grown enough to be worth it.
|
|
30
|
+
*
|
|
31
|
+
* Failing is survivable: the run carries on with the history it has rather
|
|
32
|
+
* than ending over a summary it could not write. Losing the compaction costs
|
|
33
|
+
* tokens; losing the run costs the work.
|
|
34
|
+
*/
|
|
35
|
+
const compactIfNeeded = async () => {
|
|
36
|
+
if (!opts.compact || !shouldCompact(opts.compact, last, results, opts.maxTokens))
|
|
37
|
+
return;
|
|
38
|
+
try {
|
|
39
|
+
const keep = opts.compact.keep ?? DEFAULT_KEEP;
|
|
40
|
+
const older = results.slice(0, results.length - keep);
|
|
41
|
+
const summary = await opts.compact.summarise(older);
|
|
42
|
+
const { replaced } = applyCompaction(results, summary, keep);
|
|
43
|
+
steps.push(compactStep(steps.length, summary, replaced));
|
|
44
|
+
spent += (summary.inputTokens ?? 0) + (summary.outputTokens ?? 0);
|
|
45
|
+
compactions.count += 1;
|
|
46
|
+
compactions.replaced += replaced;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
// Carry on uncompacted.
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
const finish = (status, error) => ({
|
|
53
|
+
status,
|
|
54
|
+
output: status === "completed" ? (answer ?? last?.output ?? "") : "",
|
|
55
|
+
error,
|
|
56
|
+
steps,
|
|
57
|
+
outputs: { history },
|
|
58
|
+
latencyMs: Date.now() - startedAt,
|
|
59
|
+
inputTokens: history.reduce((n, r) => n + (r.outcome.inputTokens ?? 0), 0),
|
|
60
|
+
outputTokens: history.reduce((n, r) => n + (r.outcome.outputTokens ?? 0), 0),
|
|
61
|
+
turns: history.length,
|
|
62
|
+
stoppedBy,
|
|
63
|
+
compactions,
|
|
64
|
+
...(answer === null ? {} : { answer }),
|
|
65
|
+
history,
|
|
66
|
+
});
|
|
67
|
+
for (let at = 0; at < maxTurns; at++) {
|
|
68
|
+
if (opts.maxTokens !== undefined && spent >= opts.maxTokens) {
|
|
69
|
+
stoppedBy = "tokens";
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
await compactIfNeeded();
|
|
73
|
+
const remaining = {
|
|
74
|
+
turns: maxTurns - at,
|
|
75
|
+
tokens: opts.maxTokens === undefined ? Infinity : Math.max(0, opts.maxTokens - spent),
|
|
76
|
+
};
|
|
77
|
+
const state = {
|
|
78
|
+
turn: at,
|
|
79
|
+
results: [...results],
|
|
80
|
+
last,
|
|
81
|
+
remaining,
|
|
82
|
+
emit: (text, stage) => opts.emit?.(text, stage ?? `turn ${at}`),
|
|
83
|
+
};
|
|
84
|
+
const refused = await ask(opts.hooks?.beforeTurn, state);
|
|
85
|
+
if (refused) {
|
|
86
|
+
stoppedBy = "blocked";
|
|
87
|
+
return finish("failed", `A hook stopped the run: ${refused}`);
|
|
88
|
+
}
|
|
89
|
+
const outcome = await opts.turn(state);
|
|
90
|
+
steps.push(turnStep(steps.length, at, outcome));
|
|
91
|
+
spent += tokensOf(outcome);
|
|
92
|
+
last = outcome;
|
|
93
|
+
if (outcome.status === "failed") {
|
|
94
|
+
stoppedBy = "failed";
|
|
95
|
+
return finish("failed", outcome.error ?? "The turn failed.");
|
|
96
|
+
}
|
|
97
|
+
await tell(opts.hooks?.afterTurn, outcome);
|
|
98
|
+
const calls = outcome.toolCalls ?? [];
|
|
99
|
+
const ran = calls.length > 0 ? await act(opts.tools, calls, opts.hooks ?? {}) : [];
|
|
100
|
+
for (const result of ran) {
|
|
101
|
+
steps.push(callStep(steps.length, result));
|
|
102
|
+
results.push(result);
|
|
103
|
+
}
|
|
104
|
+
const record = { turn: at, outcome, results: ran, tokens: tokensOf(outcome) };
|
|
105
|
+
history.push(record);
|
|
106
|
+
opts.onTurn?.(record);
|
|
107
|
+
answer = finished(ran);
|
|
108
|
+
if (answer !== null) {
|
|
109
|
+
stoppedBy = "finished";
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
if (calls.length === 0) {
|
|
113
|
+
// It said something and asked for nothing. Done, or lost — the caller
|
|
114
|
+
// cannot tell, which is why `finish` is worth offering it.
|
|
115
|
+
stoppedBy = "quiet";
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return finish("completed");
|
|
120
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { RolloutStep } from "@spendgraph/sdk";
|
|
2
|
+
import type { Turn } from "./types.js";
|
|
3
|
+
/** What one turn consumed. Both halves, since the history is resent every time. */
|
|
4
|
+
export declare function tokensOf(turn: Turn): number;
|
|
5
|
+
/** One model call as a step. */
|
|
6
|
+
export declare function turnStep(index: number, at: number, turn: Turn): RolloutStep;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** What one turn consumed. Both halves, since the history is resent every time. */
|
|
2
|
+
export function tokensOf(turn) {
|
|
3
|
+
return (turn.inputTokens ?? 0) + (turn.outputTokens ?? 0);
|
|
4
|
+
}
|
|
5
|
+
/** One model call as a step. */
|
|
6
|
+
export function turnStep(index, at, turn) {
|
|
7
|
+
return {
|
|
8
|
+
index,
|
|
9
|
+
source: `turn_${at}`,
|
|
10
|
+
model: turn.model,
|
|
11
|
+
output: turn.output,
|
|
12
|
+
status: turn.status ?? "completed",
|
|
13
|
+
error: turn.error,
|
|
14
|
+
inputTokens: turn.inputTokens,
|
|
15
|
+
outputTokens: turn.outputTokens,
|
|
16
|
+
};
|
|
17
|
+
}
|