@spendgraph/workflows 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/noesis/attribution.js +0 -20
- package/dist/noesis/context.js +0 -8
- package/dist/noesis/draft/draft.js +0 -13
- package/dist/noesis/events.js +0 -10
- package/dist/noesis/execute/execute.js +0 -17
- package/dist/noesis/execute/order.js +0 -8
- package/dist/noesis/gates/closure-checklist/invoke.js +0 -1
- package/dist/noesis/gates/closure-checklist/prompt.js +0 -2
- package/dist/noesis/gates/cqot-gate/invoke.js +0 -1
- package/dist/noesis/gates/cqot-gate/prompt.js +0 -2
- package/dist/noesis/gates/premortem/invoke.js +0 -1
- package/dist/noesis/gates/premortem/prompt.js +0 -2
- package/dist/noesis/gates/red-team/invoke.js +0 -1
- package/dist/noesis/gates/red-team/prompt.js +0 -2
- package/dist/noesis/gates/standards-review/invoke.js +0 -1
- package/dist/noesis/gates/standards-review/prompt.js +0 -2
- package/dist/noesis/gates/think-checkpoint/invoke.js +0 -1
- package/dist/noesis/gates/think-checkpoint/prompt.js +0 -2
- package/dist/noesis/producers/cove-answers/invoke.js +0 -1
- package/dist/noesis/producers/cove-answers/prompt.js +0 -2
- package/dist/noesis/producers/cove-questions/invoke.js +0 -1
- package/dist/noesis/producers/cove-questions/prompt.js +0 -2
- package/dist/noesis/producers/decompose/invoke.js +0 -1
- package/dist/noesis/producers/decompose/prompt.js +0 -18
- package/dist/noesis/producers/discriminating-test/invoke.js +0 -1
- package/dist/noesis/producers/discriminating-test/prompt.js +0 -2
- package/dist/noesis/producers/plan-candidates/invoke.js +0 -1
- package/dist/noesis/producers/plan-candidates/prompt.js +0 -2
- package/dist/noesis/producers/qn-brief/invoke.js +0 -1
- package/dist/noesis/producers/qn-brief/prompt.js +0 -9
- package/dist/noesis/producers/toulmin/answer.js +0 -21
- package/dist/noesis/producers/toulmin/invoke.js +0 -10
- package/dist/noesis/producers/toulmin/prompt.js +0 -9
- package/dist/noesis/producers/toulmin/render.js +0 -24
- package/dist/noesis/question.js +0 -19
- package/dist/noesis/repair/reflection/invoke.js +0 -1
- package/dist/noesis/repair/reflection/prompt.js +0 -2
- package/dist/noesis/retry.js +0 -34
- package/dist/noesis/route.js +0 -27
- package/dist/noesis/router/triage/invoke.js +0 -23
- package/dist/noesis/router/triage/prompt.js +0 -8
- package/dist/noesis/run/artifacts.js +0 -17
- package/dist/noesis/run/errors.js +0 -18
- package/dist/noesis/run/ledger.js +0 -13
- package/dist/noesis/run/loop.js +0 -19
- package/dist/noesis/run/plan.js +0 -13
- package/dist/noesis/run/produce.js +0 -34
- package/dist/noesis/run/redo.js +0 -20
- package/dist/noesis/run/result.js +0 -8
- package/dist/noesis/run/run.js +0 -21
- package/dist/noesis/run/stages/classify.js +0 -1
- package/dist/noesis/run/stages/direct.js +0 -1
- package/dist/noesis/run/stages/full.js +0 -9
- package/dist/noesis/run/stages/retrieve.js +0 -8
- package/dist/noesis/run/stages/standard.js +0 -7
- package/dist/noesis/run/state.js +0 -4
- package/dist/noesis/run/stream.js +0 -13
- package/dist/noesis/run/verify.js +0 -18
- package/dist/noesis/slugs.js +0 -14
- package/dist/noesis/stage.js +0 -10
- package/dist/noesis/started.js +0 -15
- package/dist/noesis/usage.js +0 -16
- package/dist/noesis/verify/verify.js +0 -29
- package/package.json +7 -7
|
@@ -7,21 +7,10 @@ function reTriage(currentTask) {
|
|
|
7
7
|
const task = currentTask?.trim();
|
|
8
8
|
return task ? [{ role: "user", content: `<CURRENT_TASK>\n${task}\n</CURRENT_TASK>` }] : [];
|
|
9
9
|
}
|
|
10
|
-
/** Stands in for an attachment nobody sent. */
|
|
11
10
|
const NOTHING_ATTACHED = "(none)";
|
|
12
|
-
/**
|
|
13
|
-
* An attachment slot, filled or explicitly empty.
|
|
14
|
-
*
|
|
15
|
-
* The prompt's question box wraps every request in an `<attachments>` block, so
|
|
16
|
-
* the tags are sent whether or not anything was attached. Left empty they read
|
|
17
|
-
* as an attachment the router cannot open, which it prices as ambiguity and
|
|
18
|
-
* tool need — enough on its own to push a plain question from `standard` to
|
|
19
|
-
* `full` and pay for the long route on every call that attaches nothing.
|
|
20
|
-
*/
|
|
21
11
|
export function attached(content) {
|
|
22
12
|
return content?.trim() ? content : NOTHING_ATTACHED;
|
|
23
13
|
}
|
|
24
|
-
/** The reply came back, but not as the object the router is supposed to return. */
|
|
25
14
|
export class TriageReplyError extends Error {
|
|
26
15
|
raw;
|
|
27
16
|
constructor(raw) {
|
|
@@ -30,18 +19,6 @@ export class TriageReplyError extends Error {
|
|
|
30
19
|
this.name = "TriageReplyError";
|
|
31
20
|
}
|
|
32
21
|
}
|
|
33
|
-
/**
|
|
34
|
-
* Stage 0. Pulls the router, calls it, and hands back the parsed signals.
|
|
35
|
-
*
|
|
36
|
-
* The call is recorded once, as the prompt's rollout — the model client is told
|
|
37
|
-
* not to trace, or the same tokens land again as usage and every run prices
|
|
38
|
-
* double.
|
|
39
|
-
*
|
|
40
|
-
* An open task is sent as `<CURRENT_TASK>…</CURRENT_TASK>`. The prompt names
|
|
41
|
-
* the block but never shows its markup, so this is the form it is given — keep
|
|
42
|
-
* the two matching, or the router is pattern-matching on something it was never
|
|
43
|
-
* shown.
|
|
44
|
-
*/
|
|
45
22
|
export async function invoke(client, llm, input, opts = {}) {
|
|
46
23
|
announce(opts.emit, TRIAGE_SLUG, opts.model);
|
|
47
24
|
return attempting(async () => {
|
|
@@ -1,13 +1,5 @@
|
|
|
1
|
-
/** The stored router prompt. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const TRIAGE_SLUG = "triage-router-v1-81c2cc";
|
|
3
2
|
const UNIT = { type: "number", minimum: 0, maximum: 1 };
|
|
4
|
-
/**
|
|
5
|
-
* What the reply must be, as the prompt itself specifies it.
|
|
6
|
-
*
|
|
7
|
-
* `route`, `next_stage` and the rest are deliberately absent: the prompt forbids
|
|
8
|
-
* them and code derives them, so a model that returns one fails here rather
|
|
9
|
-
* than having it read downstream.
|
|
10
|
-
*/
|
|
11
3
|
export const TRIAGE_SCHEMA = {
|
|
12
4
|
type: "object",
|
|
13
5
|
properties: {
|
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* How much of any one string an artifact may carry into a prompt.
|
|
3
|
-
*
|
|
4
|
-
* A tool answer is the one artifact with no bound on its size: everything else
|
|
5
|
-
* is a model reply against a schema, and a wide `deep_recall` can be larger
|
|
6
|
-
* than every other stage put together.
|
|
7
|
-
*/
|
|
8
1
|
const DEFAULT_LIMIT = 2000;
|
|
9
2
|
const cut = (text, limit) => `${text.slice(0, limit)}… (${text.length - limit} more characters)`;
|
|
10
3
|
function shorten(value, limit) {
|
|
@@ -17,15 +10,5 @@ function shorten(value, limit) {
|
|
|
17
10
|
}
|
|
18
11
|
return value;
|
|
19
12
|
}
|
|
20
|
-
/**
|
|
21
|
-
* The artifacts as a stage that reasons about them reads them, long strings cut.
|
|
22
|
-
*
|
|
23
|
-
* Every stage handed the whole ledger — 3A, 3B, 3C, 4A and R — is reading it
|
|
24
|
-
* for the decisions it records, and a decision is legible from the head of a
|
|
25
|
-
* tool answer. 6C is the exception and takes the artifacts whole: it cites
|
|
26
|
-
* evidence rather than reviewing choices, and a citation cut in half is a
|
|
27
|
-
* citation of nothing.
|
|
28
|
-
*/
|
|
29
13
|
export const digest = (artifacts, limit = DEFAULT_LIMIT) => shorten(artifacts, limit);
|
|
30
|
-
/** One artifact, cut the same way — what an event carries rather than a prompt. */
|
|
31
14
|
export const digestOne = (artifact, limit = DEFAULT_LIMIT) => shorten(artifact, limit);
|
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* A route the table names with nothing behind it.
|
|
3
|
-
*
|
|
4
|
-
* Thrown only where the fallback ran — `RouteResult.route` is null — which is
|
|
5
|
-
* the one case where the branch really does not exist. A branch that ran and
|
|
6
|
-
* threw is a `RouteFailedError`, and the two are separate classes because the
|
|
7
|
-
* answer to them is: write the branch, versus try again.
|
|
8
|
-
*/
|
|
9
1
|
export class RouteNotBuiltError extends Error {
|
|
10
2
|
route;
|
|
11
3
|
estimates;
|
|
@@ -20,14 +12,6 @@ export class RouteNotBuiltError extends Error {
|
|
|
20
12
|
this.name = "RouteNotBuiltError";
|
|
21
13
|
}
|
|
22
14
|
}
|
|
23
|
-
/**
|
|
24
|
-
* A branch that ran and threw, with the estimates that chose it.
|
|
25
|
-
*
|
|
26
|
-
* `detail` is the branch's own message, carried up because the graph underneath
|
|
27
|
-
* catches whatever a branch throws and reports only a failed run. A caller
|
|
28
|
-
* catching this should decide whether the cause is worth retrying; catching
|
|
29
|
-
* `RouteNotBuiltError` never is.
|
|
30
|
-
*/
|
|
31
15
|
export class RouteFailedError extends Error {
|
|
32
16
|
route;
|
|
33
17
|
estimates;
|
|
@@ -42,11 +26,9 @@ export class RouteFailedError extends Error {
|
|
|
42
26
|
this.name = "RouteFailedError";
|
|
43
27
|
}
|
|
44
28
|
}
|
|
45
|
-
/** A branch named in the routing table with nothing behind it yet. */
|
|
46
29
|
export const notBuilt = (route) => () => {
|
|
47
30
|
throw new RouteNotBuiltError(route);
|
|
48
31
|
};
|
|
49
|
-
/** A route that has to look something up, with nothing to look it up with. */
|
|
50
32
|
export class NoToolsError extends Error {
|
|
51
33
|
constructor() {
|
|
52
34
|
super("This route looks something up, so it needs a ToolBus in opts.tools.");
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { addUsage, NO_USAGE, usageOf } from "../usage.js";
|
|
2
2
|
import { digestOne } from "./artifacts.js";
|
|
3
|
-
/** Every stage the loop can run, in the order it runs them. */
|
|
4
3
|
export const STAGES = [
|
|
5
4
|
"0",
|
|
6
5
|
"1",
|
|
@@ -20,7 +19,6 @@ export const STAGES = [
|
|
|
20
19
|
"6C",
|
|
21
20
|
"R",
|
|
22
21
|
];
|
|
23
|
-
/** What a stage is called where a person or a model has to read it. */
|
|
24
22
|
export const STAGE_NAMES = {
|
|
25
23
|
"0": "triage",
|
|
26
24
|
"1": "question brief",
|
|
@@ -42,17 +40,6 @@ export const STAGE_NAMES = {
|
|
|
42
40
|
};
|
|
43
41
|
const DEFAULT_MAX_RETRIES = 5;
|
|
44
42
|
const rolloutsOf = (result) => "rolloutIds" in result ? result.rolloutIds : [result.rolloutId];
|
|
45
|
-
/**
|
|
46
|
-
* What a run leaves behind, as one thing.
|
|
47
|
-
*
|
|
48
|
-
* A branch that keeps its own artifacts hands 6C a different shape from the
|
|
49
|
-
* next branch, and nothing can invalidate a stale one on a retry. Cost is
|
|
50
|
-
* counted here too, because the graph underneath reads a node's tokens off its
|
|
51
|
-
* return value — and a branch that returns text reports none.
|
|
52
|
-
*
|
|
53
|
-
* `maxRetries` is a total across every gate: one gate flapping and five gates
|
|
54
|
-
* failing once each are the same amount of a run going nowhere.
|
|
55
|
-
*/
|
|
56
43
|
export function newLedger(maxRetries = DEFAULT_MAX_RETRIES, emit) {
|
|
57
44
|
const filed = [];
|
|
58
45
|
const spentOn = [];
|
package/dist/noesis/run/loop.js
CHANGED
|
@@ -4,17 +4,6 @@ import { invoke as standardsReview } from "../gates/standards-review/index.js";
|
|
|
4
4
|
import { STAGE_NAMES } from "./ledger.js";
|
|
5
5
|
import { dropPoint, resumeAt } from "./redo.js";
|
|
6
6
|
import { checks, reflect } from "./verify.js";
|
|
7
|
-
/**
|
|
8
|
-
* One return trip: spend a retry, find the root cause, cut the ledger back.
|
|
9
|
-
*
|
|
10
|
-
* The budget is checked before R runs, because a run with nothing left to spend
|
|
11
|
-
* has no use for knowing where it went wrong.
|
|
12
|
-
*
|
|
13
|
-
* The ledger is cut where the loop actually restarts, not where R pointed. A
|
|
14
|
-
* gate is the same gate on the same input, so anything it names redoes the
|
|
15
|
-
* checks — and cutting only from the gate would leave the stale checks filed
|
|
16
|
-
* beside the fresh ones.
|
|
17
|
-
*/
|
|
18
7
|
async function stepBack(client, llm, question, opts, state, gates, gate, reason, hint) {
|
|
19
8
|
if (!state.ledger.retry(gate)) {
|
|
20
9
|
state.ledger.escalate(gate, `Out of retries. ${gate} last said: ${reason}`);
|
|
@@ -25,13 +14,6 @@ async function stepBack(client, llm, question, opts, state, gates, gate, reason,
|
|
|
25
14
|
state.ledger.dropFrom(again === "produce" ? dropPoint(redo) : "5A");
|
|
26
15
|
return { stopped: false, draft: again === "produce" ? await gates.reproduce() : undefined };
|
|
27
16
|
}
|
|
28
|
-
/**
|
|
29
|
-
* The gates, in the order they earn their place.
|
|
30
|
-
*
|
|
31
|
-
* 5C reads the answer against its standards, 6A asks the last questions before
|
|
32
|
-
* closing, and 6B attacks what survived both. The first refusal returns, so a
|
|
33
|
-
* gate that fires early saves the later ones the work.
|
|
34
|
-
*/
|
|
35
17
|
async function judge(client, llm, opts, state, checked, gates) {
|
|
36
18
|
if (gates.standards) {
|
|
37
19
|
const review = state.ledger.record("5C", await standardsReview(client, llm, { question: checked, ...opts.context }, opts.stage));
|
|
@@ -62,7 +44,6 @@ async function judge(client, llm, opts, state, checked, gates) {
|
|
|
62
44
|
}
|
|
63
45
|
return undefined;
|
|
64
46
|
}
|
|
65
|
-
/** Check the draft, and keep going back until every gate lets it close. */
|
|
66
47
|
export async function settle(client, llm, question, opts, state, first, gates) {
|
|
67
48
|
if (state.ledger.escalated())
|
|
68
49
|
return;
|
package/dist/noesis/run/plan.js
CHANGED
|
@@ -2,19 +2,6 @@ import { invoke as cqotGate } from "../gates/cqot-gate/index.js";
|
|
|
2
2
|
import { invoke as premortem } from "../gates/premortem/index.js";
|
|
3
3
|
import { invoke as planCandidates } from "../producers/plan-candidates/index.js";
|
|
4
4
|
import { soFar } from "./produce.js";
|
|
5
|
-
/**
|
|
6
|
-
* Stage 3A, and the audits only `full` pays for.
|
|
7
|
-
*
|
|
8
|
-
* Both long routes plan, because 4A judges each step against the `tolerance` 3A
|
|
9
|
-
* wrote for it and a run with no plan has nothing to check against. What `full`
|
|
10
|
-
* adds is 3B and 3C: the plan assumed dead, then gated.
|
|
11
|
-
*
|
|
12
|
-
* Chained rather than forked, because 3C points at the step at fault and 3B may
|
|
13
|
-
* have patched that step. Run side by side, 3C would gate a plan 3B had already
|
|
14
|
-
* said was wrong. A patch is prose, so `patch_and_proceed` proceeds and the
|
|
15
|
-
* premortem is filed — 3C reads the plan with the patches beside it rather than
|
|
16
|
-
* a plan nobody rewrote.
|
|
17
|
-
*/
|
|
18
5
|
export async function plan(client, llm, question, opts, state, audits) {
|
|
19
6
|
for (;;) {
|
|
20
7
|
state.ledger.dropFrom("3A");
|
|
@@ -6,23 +6,7 @@ import { invoke as brief } from "../producers/qn-brief/index.js";
|
|
|
6
6
|
import { digest } from "./artifacts.js";
|
|
7
7
|
import { NoToolsError } from "./errors.js";
|
|
8
8
|
import { plan } from "./plan.js";
|
|
9
|
-
/**
|
|
10
|
-
* Everything the ledger holds so far, as a stage reads it.
|
|
11
|
-
*
|
|
12
|
-
* Cut through `digest`, because every caller here is reviewing decisions rather
|
|
13
|
-
* than citing evidence and one wide tool answer would otherwise be most of the
|
|
14
|
-
* prompt. 6C reads `ledger.artifacts()` itself, uncut.
|
|
15
|
-
*/
|
|
16
9
|
export const soFar = (state, question) => JSON.stringify({ question, ...digest(state.ledger.artifacts()) }, null, 2);
|
|
17
|
-
/**
|
|
18
|
-
* Stages 1 and 2: interrogate the question, then split it.
|
|
19
|
-
*
|
|
20
|
-
* Stage 1 is the one place a person is worth asking mid-run. It answers
|
|
21
|
-
* `is_right_question: false` when the asker aimed at the wrong thing, and
|
|
22
|
-
* splitting the wrong question is the most expensive way to find that out. With
|
|
23
|
-
* nobody to ask it proceeds on the question as put, and the brief says so in
|
|
24
|
-
* the ledger either way.
|
|
25
|
-
*/
|
|
26
10
|
export async function frame(client, llm, question, opts, state) {
|
|
27
11
|
const brief1 = state.ledger.record("1", await brief(client, llm, { question, ...opts.context }, opts.stage));
|
|
28
12
|
let asked = question;
|
|
@@ -44,13 +28,6 @@ export async function frame(client, llm, question, opts, state) {
|
|
|
44
28
|
const split = state.ledger.record("2", await decompose(client, llm, { question: asked, ...opts.context }, opts.stage));
|
|
45
29
|
return { asked, split };
|
|
46
30
|
}
|
|
47
|
-
/**
|
|
48
|
-
* Stages 4 and 4A: answer the parts, then judge what came back against the plan.
|
|
49
|
-
*
|
|
50
|
-
* The draft is assembled either way. 4A decides whether it is worth reading:
|
|
51
|
-
* `continue` means the work is done, and the two that are not are the run's to
|
|
52
|
-
* act on.
|
|
53
|
-
*/
|
|
54
31
|
export async function work(client, llm, question, opts, state, split) {
|
|
55
32
|
if (!opts.tools)
|
|
56
33
|
throw new NoToolsError();
|
|
@@ -76,17 +53,6 @@ export async function work(client, llm, question, opts, state, split) {
|
|
|
76
53
|
return { draft, action: "escalate" };
|
|
77
54
|
return { draft, action: "go" };
|
|
78
55
|
}
|
|
79
|
-
/**
|
|
80
|
-
* Everything up to the draft, and the checkpoint's word that it is worth having.
|
|
81
|
-
*
|
|
82
|
-
* The framing happens once. `replan` returns to 3A because a result outside the
|
|
83
|
-
* tolerance a step predicted is the plan being wrong, not the question — and
|
|
84
|
-
* re-framing would ask the person the same thing twice.
|
|
85
|
-
*
|
|
86
|
-
* A plan that never cleared its own audits stops here rather than executing:
|
|
87
|
-
* the run already knows it needs somebody, and the draft it hands back says
|
|
88
|
-
* nothing was answered rather than pretending otherwise.
|
|
89
|
-
*/
|
|
90
56
|
export async function produce(client, llm, question, opts, state, audits) {
|
|
91
57
|
const { asked, split } = await frame(client, llm, question, opts, state);
|
|
92
58
|
for (;;) {
|
package/dist/noesis/run/redo.js
CHANGED
|
@@ -1,18 +1,6 @@
|
|
|
1
1
|
import { STAGE_NAMES, STAGES } from "./ledger.js";
|
|
2
2
|
const LONGEST_FIRST = [...STAGES].sort((a, b) => b.length - a.length);
|
|
3
3
|
const boundary = (id) => new RegExp(`(^|[^a-z0-9])${id.toLowerCase()}([^a-z0-9]|$)`);
|
|
4
|
-
/**
|
|
5
|
-
* A stage id out of whatever a stage called it.
|
|
6
|
-
*
|
|
7
|
-
* `redo_from_stage` and `loop_back_to` are free strings, so they arrive as
|
|
8
|
-
* `"2"`, `"stage 3A"` or `"the decomposition"`. Longest first, or `"5A"` is
|
|
9
|
-
* read as `"5"`.
|
|
10
|
-
*
|
|
11
|
-
* Absent counts as unnameable. Both fields are `required` in their schemas and
|
|
12
|
-
* neither is checked on the way back, so a stage that omitted one hands this
|
|
13
|
-
* `undefined` — and a stage nobody can name redoes everything, which is the
|
|
14
|
-
* answer this already had for a name it could not parse.
|
|
15
|
-
*/
|
|
16
4
|
export function parseStage(said) {
|
|
17
5
|
const text = said?.trim().toLowerCase();
|
|
18
6
|
if (!text)
|
|
@@ -25,17 +13,9 @@ export function parseStage(said) {
|
|
|
25
13
|
return mentioned;
|
|
26
14
|
return STAGES.find((id) => text.includes(STAGE_NAMES[id]));
|
|
27
15
|
}
|
|
28
|
-
/**
|
|
29
|
-
* Which half of the route has to run again.
|
|
30
|
-
*
|
|
31
|
-
* The draft is the seam: anything up to 4B was produced, and anything after it
|
|
32
|
-
* only read what the draft already held. A stage nobody could name redoes the
|
|
33
|
-
* lot, because the alternative is compiling something stale.
|
|
34
|
-
*/
|
|
35
16
|
export function resumeAt(stage) {
|
|
36
17
|
if (!stage)
|
|
37
18
|
return "produce";
|
|
38
19
|
return STAGES.indexOf(stage) <= STAGES.indexOf("4B") ? "produce" : "verify";
|
|
39
20
|
}
|
|
40
|
-
/** Where the ledger is cut back to. The earliest thing a retry can redo is 1. */
|
|
41
21
|
export const dropPoint = (stage) => !stage || stage === "0" ? "1" : stage;
|
|
@@ -1,17 +1,10 @@
|
|
|
1
1
|
import { RouteFailedError, RouteNotBuiltError } from "./errors.js";
|
|
2
|
-
/**
|
|
3
|
-
* What the run is worth acting on.
|
|
4
|
-
*
|
|
5
|
-
* An escalation outranks an abstention: a gate that asked for a person said so
|
|
6
|
-
* about the whole run, and 6C abstaining afterwards is a consequence of it.
|
|
7
|
-
*/
|
|
8
2
|
function statusOf(state) {
|
|
9
3
|
if (state.ledger.escalated())
|
|
10
4
|
return "escalated";
|
|
11
5
|
const answered = state.answered;
|
|
12
6
|
return answered && (answered.answer.abstained || !answered.accepted) ? "abstained" : "answered";
|
|
13
7
|
}
|
|
14
|
-
/** Everything the run was billed for, whole and split two ways. */
|
|
15
8
|
function usageOfRun(state) {
|
|
16
9
|
const byStage = state.ledger.byStage();
|
|
17
10
|
return {
|
|
@@ -21,7 +14,6 @@ function usageOfRun(state) {
|
|
|
21
14
|
calls: byStage.reduce((total, stage) => total + stage.calls, 0),
|
|
22
15
|
};
|
|
23
16
|
}
|
|
24
|
-
/** The state, once every branch has run, as a result. */
|
|
25
17
|
export function finish(decided, state, latencyMs) {
|
|
26
18
|
const { estimates, routing, answered, text, ledger } = state;
|
|
27
19
|
if (decided.status === "failed" || !estimates || !routing || !answered || !text) {
|
package/dist/noesis/run/run.js
CHANGED
|
@@ -4,27 +4,7 @@ import { notBuilt } from "./errors.js";
|
|
|
4
4
|
import { finish } from "./result.js";
|
|
5
5
|
import { classifyStage, directStage, fullStage, retrieveStage, standardStage, } from "./stages/index.js";
|
|
6
6
|
import { newRunState } from "./state.js";
|
|
7
|
-
/**
|
|
8
|
-
* Question in, answer out.
|
|
9
|
-
*
|
|
10
|
-
* `onDeliver` runs once, after the loop, and only for an answer worth acting
|
|
11
|
-
* on. Anything that renders a deliverable belongs there rather than inside a
|
|
12
|
-
* branch: nothing downstream reads it, no gate can check it, every back edge
|
|
13
|
-
* would make it again, and generating a finished-looking document out of an
|
|
14
|
-
* abstention is the worst thing this loop could produce.
|
|
15
|
-
*
|
|
16
|
-
* Pass the previous run's `currentTask` back in through `opts.triage` to keep a
|
|
17
|
-
* session going: it is what lets triage answer `is_new_task` at all, and the
|
|
18
|
-
* result carries the task forward for the turn after this one.
|
|
19
|
-
*/
|
|
20
7
|
const DEFAULT_MAX_RETRIES = 5;
|
|
21
|
-
/**
|
|
22
|
-
* The options every stage is handed, with the emitter already on them.
|
|
23
|
-
*
|
|
24
|
-
* Threading it here rather than at each stage call is what let fifteen stages
|
|
25
|
-
* start announcing themselves without one of them being edited: they all
|
|
26
|
-
* already forward `opts.stage`, `opts.triage` and `opts.answer` verbatim.
|
|
27
|
-
*/
|
|
28
8
|
const watched = (opts, emit) => ({
|
|
29
9
|
...opts,
|
|
30
10
|
triage: { ...opts.triage, emit },
|
|
@@ -53,7 +33,6 @@ export async function run(client, llm, question, options = {}) {
|
|
|
53
33
|
},
|
|
54
34
|
fallback: notBuilt("unknown"),
|
|
55
35
|
});
|
|
56
|
-
// Every price the stages left in flight, collected before the result is built.
|
|
57
36
|
await state.ledger.settle();
|
|
58
37
|
const result = finish(decided, state, Date.now() - startedAt);
|
|
59
38
|
emit?.({
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { routeFor } from "../../route.js";
|
|
2
2
|
import { invoke as triage } from "../../router/triage/index.js";
|
|
3
3
|
import { nextTask } from "../state.js";
|
|
4
|
-
/** Stage 0, and the route its estimates earn. */
|
|
5
4
|
export const classifyStage = (client, llm, question, opts, state) => async () => {
|
|
6
5
|
const stage = await triage(client, llm, { ...opts.triage, currentTask: state.currentTask, question }, { ...opts.triage });
|
|
7
6
|
state.estimates = state.ledger.record("0", { ...stage, data: stage.estimates });
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { answer } from "../../producers/toulmin/index.js";
|
|
2
|
-
/** The direct branch: the question and what triage made of it, straight to 6C. */
|
|
3
2
|
export const directStage = (client, llm, question, opts, state) => async () => {
|
|
4
3
|
const answered = await answer(client, llm, { question, ...state.ledger.artifacts() }, opts.answer ?? {});
|
|
5
4
|
state.answered = answered;
|
|
@@ -1,15 +1,6 @@
|
|
|
1
1
|
import { settle } from "../loop.js";
|
|
2
2
|
import { produce } from "../produce.js";
|
|
3
3
|
import { compile } from "../verify.js";
|
|
4
|
-
/**
|
|
5
|
-
* The full branch: everything standard does, plus the audits.
|
|
6
|
-
*
|
|
7
|
-
* 3B and 3C interrogate the plan before it runs; 5C and 6B interrogate the
|
|
8
|
-
* answer before it ships. Standard already plans, so what full adds is four
|
|
9
|
-
* audits and nothing else. 6B fires on `routing.redTeam` rather than on the
|
|
10
|
-
* route, because stakes are what earn it — a run sent here by difficulty or
|
|
11
|
-
* ambiguity has no attacker to pay for.
|
|
12
|
-
*/
|
|
13
4
|
export const fullStage = (client, llm, question, opts, state) => async () => {
|
|
14
5
|
const reproduce = () => produce(client, llm, question, opts, state, true);
|
|
15
6
|
await settle(client, llm, question, opts, state, await reproduce(), {
|
|
@@ -11,14 +11,6 @@ const lookup = (question, tool) => ({
|
|
|
11
11
|
kind: "read_only",
|
|
12
12
|
tool,
|
|
13
13
|
});
|
|
14
|
-
/**
|
|
15
|
-
* The retrieve branch: one lookup, then 6C compiles what came back.
|
|
16
|
-
*
|
|
17
|
-
* No brief and no decomposition, because there is nothing to interrogate or
|
|
18
|
-
* split — the question is clear and the only reason the model cannot answer it
|
|
19
|
-
* is that the answer is somewhere else. A tool that finds nothing leaves 6C
|
|
20
|
-
* with no grounds, which is what abstention is for.
|
|
21
|
-
*/
|
|
22
14
|
export const retrieveStage = (client, llm, question, opts, state) => async () => {
|
|
23
15
|
if (!opts.tools)
|
|
24
16
|
throw new NoToolsError();
|
|
@@ -1,13 +1,6 @@
|
|
|
1
1
|
import { settle } from "../loop.js";
|
|
2
2
|
import { produce } from "../produce.js";
|
|
3
3
|
import { compile } from "../verify.js";
|
|
4
|
-
/**
|
|
5
|
-
* The standard branch: frame it, split it, answer the parts, then check them.
|
|
6
|
-
*
|
|
7
|
-
* It plans, because 4A judges each step against the tolerance 3A wrote for it.
|
|
8
|
-
* What it does not do is audit: no premortem or gate on the plan, and no
|
|
9
|
-
* standards review or red team on the answer. That is what `full` buys.
|
|
10
|
-
*/
|
|
11
4
|
export const standardStage = (client, llm, question, opts, state) => async () => {
|
|
12
5
|
const reproduce = () => produce(client, llm, question, opts, state, false);
|
|
13
6
|
await settle(client, llm, question, opts, state, await reproduce(), {
|
package/dist/noesis/run/state.js
CHANGED
|
@@ -4,10 +4,6 @@ export const newRunState = (currentTask, emit, maxRetries) => ({
|
|
|
4
4
|
emit,
|
|
5
5
|
currentTask,
|
|
6
6
|
});
|
|
7
|
-
/**
|
|
8
|
-
* The open task after triage has read the request: the router's summary when it
|
|
9
|
-
* calls the request new, otherwise the task carried in.
|
|
10
|
-
*/
|
|
11
7
|
export function nextTask(estimates, current) {
|
|
12
8
|
return estimates.is_new_task ? estimates.summary : (current ?? estimates.summary);
|
|
13
9
|
}
|
|
@@ -1,18 +1,5 @@
|
|
|
1
1
|
import { streamed } from "@spendgraph/harness";
|
|
2
2
|
import { run } from "./run.js";
|
|
3
|
-
/**
|
|
4
|
-
* `run()`, watched.
|
|
5
|
-
*
|
|
6
|
-
* The run starts on the call rather than on the first read, so a client that
|
|
7
|
-
* connects late still receives everything that already happened, and a caller
|
|
8
|
-
* who only wants the answer never has to iterate.
|
|
9
|
-
*
|
|
10
|
-
* Events arrive unwrapped — a consumer switches on `event.type` directly rather
|
|
11
|
-
* than reaching through the harness envelope the queue underneath uses.
|
|
12
|
-
*
|
|
13
|
-
* `opts.onEvent` still works and is called first, so a caller can log and
|
|
14
|
-
* stream from the same run.
|
|
15
|
-
*/
|
|
16
3
|
export function runStream(client, llm, question, opts = {}) {
|
|
17
4
|
const inner = streamed((_emit, note) => run(client, llm, question, {
|
|
18
5
|
...opts,
|
|
@@ -7,16 +7,6 @@ import { invoke as reflection } from "../repair/reflection/index.js";
|
|
|
7
7
|
import { blind, compare, render as renderChecks } from "../verify/index.js";
|
|
8
8
|
import { soFar } from "./produce.js";
|
|
9
9
|
import { parseStage, resumeAt } from "./redo.js";
|
|
10
|
-
/**
|
|
11
|
-
* Stages 5A, 5B and the pairing between them.
|
|
12
|
-
*
|
|
13
|
-
* 5B is given `blind()` rather than the questions themselves, so it answers
|
|
14
|
-
* without sight of the claims it settles. Putting the two back together is the
|
|
15
|
-
* step in between, and it stays in code for the same reason.
|
|
16
|
-
*
|
|
17
|
-
* A 5A that found nothing to check skips 5B rather than sending it an empty
|
|
18
|
-
* turn, which every provider rejects.
|
|
19
|
-
*/
|
|
20
10
|
export async function checks(client, llm, opts, state, draft) {
|
|
21
11
|
const drafted = renderDraft(draft);
|
|
22
12
|
const questions = state.ledger.record("5A", await coveQuestions(client, llm, { question: drafted, ...opts.context }, opts.stage));
|
|
@@ -43,13 +33,6 @@ export async function checks(client, llm, opts, state, draft) {
|
|
|
43
33
|
});
|
|
44
34
|
return `${drafted}\n\n${renderChecks(verification)}`;
|
|
45
35
|
}
|
|
46
|
-
/**
|
|
47
|
-
* Stage R. The earliest wrong decision, and the stage to redo from.
|
|
48
|
-
*
|
|
49
|
-
* The gate says where the problem surfaced; this says where it started, which
|
|
50
|
-
* is rarely the same place and is the only one worth returning to. The gate's
|
|
51
|
-
* own suggestion is the fallback, not the answer.
|
|
52
|
-
*/
|
|
53
36
|
export async function reflect(client, llm, question, opts, state, gate, reason, hint) {
|
|
54
37
|
const looked = state.ledger.record("R", await reflection(client, llm, { question: `${soFar(state, question)}\n\nThe gate said: ${reason}`, ...opts.context }, opts.stage));
|
|
55
38
|
const redoFrom = parseStage(looked.redo_from_stage) ?? parseStage(hint ?? "");
|
|
@@ -62,7 +45,6 @@ export async function reflect(client, llm, question, opts, state, gate, reason,
|
|
|
62
45
|
});
|
|
63
46
|
return redoFrom;
|
|
64
47
|
}
|
|
65
|
-
/** Stage 6C. Everything the ledger holds, compiled into a claim. */
|
|
66
48
|
export async function compile(client, llm, question, opts, state) {
|
|
67
49
|
const answered = await answer(client, llm, { question, ...state.ledger.artifacts() }, opts.answer ?? {});
|
|
68
50
|
state.answered = answered;
|
package/dist/noesis/slugs.js
CHANGED
|
@@ -13,20 +13,6 @@ import { BRIEF_SLUG } from "./producers/qn-brief/prompt.js";
|
|
|
13
13
|
import { TOULMIN_SLUG } from "./producers/toulmin/prompt.js";
|
|
14
14
|
import { REFLECTION_SLUG } from "./repair/reflection/prompt.js";
|
|
15
15
|
import { TRIAGE_SLUG } from "./router/triage/prompt.js";
|
|
16
|
-
/**
|
|
17
|
-
* Which stage a slug belongs to.
|
|
18
|
-
*
|
|
19
|
-
* The inverse of what the fifteen modules already declare, and the reason
|
|
20
|
-
* `runStage` can announce a stage it was only ever told the slug of. A slug
|
|
21
|
-
* nobody claims is not an error — it means a caller reached `runStage` with a
|
|
22
|
-
* prompt of their own, and an event for it would be a lie.
|
|
23
|
-
*
|
|
24
|
-
* Imported from each `prompt.ts` rather than from the module's `index.ts`,
|
|
25
|
-
* which would be a cycle: an index re-exports `invoke`, `invoke` reaches
|
|
26
|
-
* `announce`, and `announce` reads this map. A computed key whose value is
|
|
27
|
-
* still uninitialised becomes the literal string "undefined" — no error, just a
|
|
28
|
-
* stage that silently never announces itself.
|
|
29
|
-
*/
|
|
30
16
|
export const STAGE_OF_SLUG = {
|
|
31
17
|
[TRIAGE_SLUG]: "0",
|
|
32
18
|
[BRIEF_SLUG]: "1",
|
package/dist/noesis/stage.js
CHANGED
|
@@ -3,7 +3,6 @@ import { askThrough, QUESTION_FIELD } from "./question.js";
|
|
|
3
3
|
import { attempting } from "./retry.js";
|
|
4
4
|
import { announce, failed } from "./started.js";
|
|
5
5
|
import { usageOf } from "./usage.js";
|
|
6
|
-
/** The reply came back, but not as the object the stage is supposed to return. */
|
|
7
6
|
export class StageReplyError extends Error {
|
|
8
7
|
slug;
|
|
9
8
|
raw;
|
|
@@ -14,15 +13,6 @@ export class StageReplyError extends Error {
|
|
|
14
13
|
this.name = "StageReplyError";
|
|
15
14
|
}
|
|
16
15
|
}
|
|
17
|
-
/**
|
|
18
|
-
* One stored prompt, called and parsed.
|
|
19
|
-
*
|
|
20
|
-
* Every stage does the same four things — pull by slug, put the request through
|
|
21
|
-
* `{question}`, force the schema, hand back the object with what it cost — so
|
|
22
|
-
* they do it here rather than thirteen times over. The call is recorded once,
|
|
23
|
-
* as the prompt's rollout: the model client is told not to trace, or the same
|
|
24
|
-
* tokens land again as usage and every run prices double.
|
|
25
|
-
*/
|
|
26
16
|
export async function runStage(client, llm, slug, schema, input, opts = {}) {
|
|
27
17
|
announce(opts.emit, slug, opts.model);
|
|
28
18
|
return attempting(async () => {
|
package/dist/noesis/started.js
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import { STAGE_NAMES } from "./run/ledger.js";
|
|
2
2
|
import { STAGE_OF_SLUG } from "./slugs.js";
|
|
3
|
-
/**
|
|
4
|
-
* Say a stage has begun, if anyone is listening and the slug is one of ours.
|
|
5
|
-
*
|
|
6
|
-
* Shared by the three call paths — `runStage` and the copies triage and 6C keep
|
|
7
|
-
* — so the three of them cannot drift on what a `stage.started` looks like.
|
|
8
|
-
*/
|
|
9
3
|
export function announce(emit, slug, model) {
|
|
10
4
|
if (!emit)
|
|
11
5
|
return;
|
|
@@ -15,15 +9,6 @@ export function announce(emit, slug, model) {
|
|
|
15
9
|
emit({ type: "stage.started", stage, name: STAGE_NAMES[stage], slug, model });
|
|
16
10
|
}
|
|
17
11
|
const message = (err) => (err instanceof Error ? err.message : String(err));
|
|
18
|
-
/**
|
|
19
|
-
* Say a stage did not produce what it was asked for.
|
|
20
|
-
*
|
|
21
|
-
* The kind is read off the error's name rather than by importing the three
|
|
22
|
-
* reply errors, which would be a cycle — `stage.ts` reaches this file, and the
|
|
23
|
-
* errors live beside the calls that throw them. Every one of them ends in
|
|
24
|
-
* `ReplyError`, and that is the distinction worth drawing: a reply that came
|
|
25
|
-
* back wrong is the prompt's problem, anything else is the provider's.
|
|
26
|
-
*/
|
|
27
12
|
export function failed(emit, slug, err) {
|
|
28
13
|
if (!emit)
|
|
29
14
|
return;
|
package/dist/noesis/usage.js
CHANGED
|
@@ -6,14 +6,7 @@ export const NO_USAGE = {
|
|
|
6
6
|
citationTokens: 0,
|
|
7
7
|
reasoningTokens: 0,
|
|
8
8
|
};
|
|
9
|
-
/**
|
|
10
|
-
* Sums two costs without inventing one.
|
|
11
|
-
*
|
|
12
|
-
* Undefined on both sides stays undefined; a priced call landing beside an
|
|
13
|
-
* unpriced one gives the priced total, which is a floor rather than a fiction.
|
|
14
|
-
*/
|
|
15
9
|
const addCost = (a, b) => a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0);
|
|
16
|
-
/** Every token field filled, so a caller never has to guard one. Cost stays absent when it is. */
|
|
17
10
|
export const usageOf = (reported) => ({
|
|
18
11
|
inputTokens: reported.inputTokens ?? 0,
|
|
19
12
|
outputTokens: reported.outputTokens ?? 0,
|
|
@@ -35,22 +28,13 @@ export const addUsage = (total, next) => ({
|
|
|
35
28
|
return cost === undefined ? {} : { costMicros: cost };
|
|
36
29
|
})(),
|
|
37
30
|
});
|
|
38
|
-
/** Every token the call is billed for, however the provider split them. */
|
|
39
31
|
export const billableTokens = (usage) => usage.inputTokens +
|
|
40
32
|
usage.outputTokens +
|
|
41
33
|
usage.cacheReadTokens +
|
|
42
34
|
usage.cacheWriteTokens +
|
|
43
35
|
usage.citationTokens +
|
|
44
36
|
usage.reasoningTokens;
|
|
45
|
-
/** What the run is known to have cost, in micro-USD, or undefined where nothing priced it. */
|
|
46
37
|
export const spend = (usage) => usage.costMicros;
|
|
47
|
-
/**
|
|
48
|
-
* One price for several calls: undefined unless at least one of them was priced.
|
|
49
|
-
*
|
|
50
|
-
* The cascade in 6C runs a tier at a time, so a run can hold several prices
|
|
51
|
-
* still in flight. Summing them with `?? 0` alone would turn "nobody priced
|
|
52
|
-
* any of these" into a confident zero.
|
|
53
|
-
*/
|
|
54
38
|
export const totalCost = (costs) => costs.some((cost) => cost !== undefined)
|
|
55
39
|
? costs.reduce((sum, cost) => sum + (cost ?? 0), 0)
|
|
56
40
|
: undefined;
|