@spendgraph/workflows 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -0
- package/dist/noesis/attribution.js +0 -20
- package/dist/noesis/context.js +0 -8
- package/dist/noesis/draft/draft.js +0 -13
- package/dist/noesis/events.js +0 -10
- package/dist/noesis/execute/execute.js +2 -17
- package/dist/noesis/execute/index.d.ts +2 -0
- package/dist/noesis/execute/index.js +1 -0
- package/dist/noesis/execute/order.js +0 -8
- package/dist/noesis/execute/spend.d.ts +36 -0
- package/dist/noesis/execute/spend.js +79 -0
- package/dist/noesis/execute/types.d.ts +2 -0
- package/dist/noesis/gates/closure-checklist/invoke.js +0 -1
- package/dist/noesis/gates/closure-checklist/prompt.js +0 -2
- package/dist/noesis/gates/cqot-gate/invoke.js +0 -1
- package/dist/noesis/gates/cqot-gate/prompt.js +0 -2
- package/dist/noesis/gates/premortem/invoke.js +0 -1
- package/dist/noesis/gates/premortem/prompt.js +0 -2
- package/dist/noesis/gates/red-team/invoke.js +0 -1
- package/dist/noesis/gates/red-team/prompt.js +0 -2
- package/dist/noesis/gates/standards-review/invoke.js +0 -1
- package/dist/noesis/gates/standards-review/prompt.js +0 -2
- package/dist/noesis/gates/think-checkpoint/invoke.js +0 -1
- package/dist/noesis/gates/think-checkpoint/prompt.js +0 -2
- package/dist/noesis/index.d.ts +2 -2
- package/dist/noesis/index.js +1 -1
- package/dist/noesis/producers/cove-answers/invoke.js +0 -1
- package/dist/noesis/producers/cove-answers/prompt.js +0 -2
- package/dist/noesis/producers/cove-questions/invoke.js +0 -1
- package/dist/noesis/producers/cove-questions/prompt.js +0 -2
- package/dist/noesis/producers/decompose/invoke.js +0 -1
- package/dist/noesis/producers/decompose/prompt.js +0 -18
- package/dist/noesis/producers/discriminating-test/invoke.js +0 -1
- package/dist/noesis/producers/discriminating-test/prompt.js +0 -2
- package/dist/noesis/producers/plan-candidates/invoke.js +0 -1
- package/dist/noesis/producers/plan-candidates/prompt.js +0 -2
- package/dist/noesis/producers/qn-brief/invoke.js +0 -1
- package/dist/noesis/producers/qn-brief/prompt.js +0 -9
- package/dist/noesis/producers/toulmin/answer.js +0 -21
- package/dist/noesis/producers/toulmin/invoke.js +0 -10
- package/dist/noesis/producers/toulmin/prompt.js +0 -9
- package/dist/noesis/producers/toulmin/render.js +0 -24
- package/dist/noesis/question.js +0 -19
- package/dist/noesis/repair/reflection/invoke.js +0 -1
- package/dist/noesis/repair/reflection/prompt.js +0 -2
- package/dist/noesis/retry.js +0 -34
- package/dist/noesis/route.js +0 -27
- package/dist/noesis/router/triage/invoke.js +0 -23
- package/dist/noesis/router/triage/prompt.js +0 -8
- package/dist/noesis/run/artifacts.js +0 -17
- package/dist/noesis/run/errors.js +0 -18
- package/dist/noesis/run/index.d.ts +2 -2
- package/dist/noesis/run/index.js +1 -1
- package/dist/noesis/run/ledger.d.ts +49 -1
- package/dist/noesis/run/ledger.js +19 -13
- package/dist/noesis/run/loop.js +4 -22
- package/dist/noesis/run/options.d.ts +11 -0
- package/dist/noesis/run/options.js +18 -0
- package/dist/noesis/run/plan.js +4 -16
- package/dist/noesis/run/produce.js +8 -37
- package/dist/noesis/run/redo.js +0 -20
- package/dist/noesis/run/result.js +1 -10
- package/dist/noesis/run/run.js +5 -21
- package/dist/noesis/run/stages/classify.js +2 -2
- package/dist/noesis/run/stages/direct.js +0 -1
- package/dist/noesis/run/stages/full.js +0 -9
- package/dist/noesis/run/stages/retrieve.js +0 -8
- package/dist/noesis/run/stages/standard.js +0 -7
- package/dist/noesis/run/state.d.ts +5 -0
- package/dist/noesis/run/state.js +5 -4
- package/dist/noesis/run/stream.js +0 -13
- package/dist/noesis/run/verify.js +5 -22
- package/dist/noesis/slugs.js +0 -14
- package/dist/noesis/stage.js +0 -10
- package/dist/noesis/started.js +0 -15
- package/dist/noesis/types.d.ts +17 -5
- package/dist/noesis/usage.js +0 -16
- package/dist/noesis/verify/verify.js +0 -29
- package/package.json +7 -7
|
@@ -3,7 +3,6 @@ import { attempting } from "../../retry.js";
|
|
|
3
3
|
import { announce, failed } from "../../started.js";
|
|
4
4
|
import { usageOf } from "../../usage.js";
|
|
5
5
|
import { TOULMIN_SCHEMA, TOULMIN_SLUG } from "./prompt.js";
|
|
6
|
-
/** The reply came back, but not as the object the stage is supposed to return. */
|
|
7
6
|
export class ToulminReplyError extends Error {
|
|
8
7
|
raw;
|
|
9
8
|
constructor(raw) {
|
|
@@ -15,15 +14,6 @@ export class ToulminReplyError extends Error {
|
|
|
15
14
|
function asText(artifacts) {
|
|
16
15
|
return typeof artifacts === "string" ? artifacts : JSON.stringify(artifacts, null, 2);
|
|
17
16
|
}
|
|
18
|
-
/**
|
|
19
|
-
* Stage 6C. Compiles the artifacts into the final answer and its qualifier.
|
|
20
|
-
*
|
|
21
|
-
* The artifacts are the request, so they go through `{question}` like every
|
|
22
|
-
* other stage's. They used to arrive as a history turn instead, which worked
|
|
23
|
-
* only while the prompt's question box was empty: `renderMessages` puts history
|
|
24
|
-
* between the system turn and the question, so anything saved in that box would
|
|
25
|
-
* have been the last thing the model read.
|
|
26
|
-
*/
|
|
27
17
|
export async function invoke(client, llm, artifacts, opts = {}) {
|
|
28
18
|
announce(opts.emit, TOULMIN_SLUG, opts.model);
|
|
29
19
|
return attempting(async () => {
|
|
@@ -1,14 +1,5 @@
|
|
|
1
|
-
/** The stored answer-assembly prompt. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const TOULMIN_SLUG = "ctr-6c-toulmin-answer-d28bb8";
|
|
3
2
|
const UNIT = { type: "number", minimum: 0, maximum: 1 };
|
|
4
|
-
/**
|
|
5
|
-
* What the reply must be, as the prompt itself specifies it.
|
|
6
|
-
*
|
|
7
|
-
* `rebuttals` and `would_change_my_mind` are required rather than optional on
|
|
8
|
-
* purpose: an empty list is a claim the stage may make, but a missing key would
|
|
9
|
-
* let a surviving objection be dropped silently, which is what this stage
|
|
10
|
-
* exists to prevent.
|
|
11
|
-
*/
|
|
12
3
|
export const TOULMIN_SCHEMA = {
|
|
13
4
|
type: "object",
|
|
14
5
|
properties: {
|
|
@@ -9,21 +9,9 @@ function confidencePhrase(confidence) {
|
|
|
9
9
|
return "low";
|
|
10
10
|
return "very low";
|
|
11
11
|
}
|
|
12
|
-
/**
|
|
13
|
-
* Whether the model actually said this.
|
|
14
|
-
*
|
|
15
|
-
* Takes `string | undefined` although `ToulminAnswer` types every field as a
|
|
16
|
-
* string, because nothing enforces that: the schema's `required` list is sent
|
|
17
|
-
* to the provider and never checked on the way back, so a field the model
|
|
18
|
-
* omitted arrives as `undefined` and reads here as unsaid.
|
|
19
|
-
*/
|
|
20
12
|
function said(value) {
|
|
21
13
|
return !!value && value.trim().length > 0;
|
|
22
14
|
}
|
|
23
|
-
/**
|
|
24
|
-
* The model writes these as sentences, and the label already supplies the
|
|
25
|
-
* conjunction — without this they read "holds when when the trace shows".
|
|
26
|
-
*/
|
|
27
15
|
function clause(text) {
|
|
28
16
|
return text.trim().replace(/^(when|if)\s+/i, "");
|
|
29
17
|
}
|
|
@@ -54,18 +42,6 @@ function groundLines(answer) {
|
|
|
54
42
|
const warrant = said(answer.warrant) ? `Warrant: ${answer.warrant}` : "";
|
|
55
43
|
return [grounds, warrant].filter(said).join("\n\n");
|
|
56
44
|
}
|
|
57
|
-
/**
|
|
58
|
-
* The answer as a person reads it.
|
|
59
|
-
*
|
|
60
|
-
* Pure, and deliberately not a seventh prompt. Every part of a `ToulminAnswer`
|
|
61
|
-
* is something the stage was forbidden to soften — a surviving objection, a
|
|
62
|
-
* calibrated number, an abstention — and handing the object to a model to write
|
|
63
|
-
* up returns each of those chances on the last call before anyone sees it.
|
|
64
|
-
*
|
|
65
|
-
* Objections are never dropped. Where something has to give, grounds go first,
|
|
66
|
-
* which is why they are off by default: they are the audit trail, and the
|
|
67
|
-
* rollout already has them.
|
|
68
|
-
*/
|
|
69
45
|
export function render(answer, opts = {}) {
|
|
70
46
|
const rebuttals = rebuttalLines(answer.rebuttals);
|
|
71
47
|
const changes = said(answer.would_change_my_mind)
|
package/dist/noesis/question.js
CHANGED
|
@@ -1,28 +1,9 @@
|
|
|
1
|
-
/** The variable every stored prompt takes its request through. */
|
|
2
1
|
export const QUESTION_FIELD = "question";
|
|
3
2
|
const warned = new Set();
|
|
4
3
|
function carries(messages, question) {
|
|
5
4
|
const last = messages[messages.length - 1];
|
|
6
5
|
return last?.role === "user" && last.content.includes(question);
|
|
7
6
|
}
|
|
8
|
-
/**
|
|
9
|
-
* The messages to send, with the request in the turn the prompt renders.
|
|
10
|
-
*
|
|
11
|
-
* A prompt whose question box is `{question}` needs nothing from this: the
|
|
12
|
-
* value passed alongside fills it and the rendered turn already carries the
|
|
13
|
-
* request. Asked of the messages rather than of `prompt.fields`, because
|
|
14
|
-
* `applyVariables` substitutes any `{name}` it finds against the values it is
|
|
15
|
-
* given and never reads the field list — so a placeholder typed into the box
|
|
16
|
-
* works with no field declared, and would not show up in `fields` at all.
|
|
17
|
-
*
|
|
18
|
-
* Where the box holds something else — a question typed while testing, or
|
|
19
|
-
* nothing at all — the trailing turn is overwritten with the request and the
|
|
20
|
-
* stored text is not sent. That is what the placeholder would have done, and it
|
|
21
|
-
* keeps the shape the prompt expects: one system turn, one user turn. It warns
|
|
22
|
-
* once per prompt, because the alternative failure is silent: an unmigrated
|
|
23
|
-
* prompt would answer the question saved on it and return a well-formed result
|
|
24
|
-
* about the wrong thing.
|
|
25
|
-
*/
|
|
26
7
|
export function askThrough(messages, question, prompt) {
|
|
27
8
|
if (carries(messages, question))
|
|
28
9
|
return messages;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { REFLECTION_SCHEMA, REFLECTION_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage R. The earliest wrong decision behind a failure, not the place it surfaced. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, REFLECTION_SLUG, REFLECTION_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/** Stage R. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const REFLECTION_SLUG = "ctr-r-root-cause-reflection-7x0x90";
|
|
3
|
-
/** What the reply must be, as the prompt itself specifies it. */
|
|
4
2
|
export const REFLECTION_SCHEMA = {
|
|
5
3
|
type: "object",
|
|
6
4
|
properties: {
|
package/dist/noesis/retry.js
CHANGED
|
@@ -1,33 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* How many times a stage call may be made, counting the first.
|
|
3
|
-
*
|
|
4
|
-
* Two, not more. A flake is usually gone on the second try, and a call that
|
|
5
|
-
* failed has often still been billed — so the budget is deliberately small and
|
|
6
|
-
* the classifier below deliberately narrow.
|
|
7
|
-
*/
|
|
8
1
|
const DEFAULT_ATTEMPTS = 2;
|
|
9
|
-
/** Milliseconds before the next attempt, multiplied by the attempt number. */
|
|
10
2
|
const BACKOFF_MS = 300;
|
|
11
3
|
const pause = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
12
|
-
/**
|
|
13
|
-
* Signals that a second attempt could plausibly land.
|
|
14
|
-
*
|
|
15
|
-
* Matched on the message because a stage is handed whatever the caller's client
|
|
16
|
-
* threw, and nothing here can assume a status code survived. Anything else is
|
|
17
|
-
* not retried: a 400 naming a model that does not exist and a 402 for credits
|
|
18
|
-
* that are not there fail the same way however many times they are sent, and
|
|
19
|
-
* spending three round trips to learn that is worse than failing once.
|
|
20
|
-
*/
|
|
21
4
|
const TRANSIENT = /\b(429|500|502|503|504|522|524)\b|rate.?limit|overloaded|timed? ?out|timeout|ECONNRESET|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed|network error/i;
|
|
22
|
-
/**
|
|
23
|
-
* Whether the same call is worth making again.
|
|
24
|
-
*
|
|
25
|
-
* Two families qualify. A reply error means a schema-forced call came back with
|
|
26
|
-
* nothing to parse — the model declined to emit the tool call, which is a flake
|
|
27
|
-
* rather than a fault in the prompt, and the bounded budget covers the case
|
|
28
|
-
* where it is not. A transient transport error is the provider asking to be
|
|
29
|
-
* asked later.
|
|
30
|
-
*/
|
|
31
5
|
export function retryable(err) {
|
|
32
6
|
if (!(err instanceof Error))
|
|
33
7
|
return false;
|
|
@@ -35,14 +9,6 @@ export function retryable(err) {
|
|
|
35
9
|
return true;
|
|
36
10
|
return TRANSIENT.test(err.message);
|
|
37
11
|
}
|
|
38
|
-
/**
|
|
39
|
-
* Runs a stage call, once more if the first failure looks worth repeating.
|
|
40
|
-
*
|
|
41
|
-
* This is deliberately not the gate retry budget. That one is spent when a gate
|
|
42
|
-
* refuses and reflection sends work backwards — the loop thinking again. This
|
|
43
|
-
* one covers a call that never produced anything to think about, and mixing the
|
|
44
|
-
* two would let a flaky provider eat the allowance reserved for reasoning.
|
|
45
|
-
*/
|
|
46
12
|
export async function attempting(run, opts = {}) {
|
|
47
13
|
const total = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS);
|
|
48
14
|
for (let attempt = 1;; attempt++) {
|
package/dist/noesis/route.js
CHANGED
|
@@ -8,33 +8,6 @@ const NEARLY = 0.6;
|
|
|
8
8
|
const HARD = new Set(["expert", "research"]);
|
|
9
9
|
const EASY = new Set(["trivial", "easy"]);
|
|
10
10
|
const SERIOUS = new Set(["high", "critical"]);
|
|
11
|
-
/**
|
|
12
|
-
* The estimates, turned into a route.
|
|
13
|
-
*
|
|
14
|
-
* Deliberately code and deliberately dull: the router prompt is told not to
|
|
15
|
-
* choose its own budget, and a threshold written here can be moved after an
|
|
16
|
-
* eval run without touching a prompt or re-running anything.
|
|
17
|
-
*
|
|
18
|
-
* A lookup is answered as a lookup, even when the stakes are high. `redTeam` is
|
|
19
|
-
* a flag rather than a route, so it still fires — the care is kept and the
|
|
20
|
-
* planning stages that have nothing to plan are not paid for. `critical` is the
|
|
21
|
-
* exception, and keeps the whole loop.
|
|
22
|
-
*
|
|
23
|
-
* Difficulty alone does not buy the audit set. The premortem and the standards
|
|
24
|
-
* review bite on risk and vagueness, not on how hard the reasoning is, so a
|
|
25
|
-
* hard question that is low-stakes, well-posed and already understood takes
|
|
26
|
-
* `standard`. Hard and *unsure* still takes `full`.
|
|
27
|
-
*
|
|
28
|
-
* Vagueness and doubt compound. Either one alone is survivable; a question that
|
|
29
|
-
* is both under-specified and one the model cannot place is what 3B and 3C
|
|
30
|
-
* exist for, and it takes `full` below the ambiguity bar that would catch it on
|
|
31
|
-
* its own.
|
|
32
|
-
*
|
|
33
|
-
* `retrieve` splits the two reasons a clear question fails `direct`: high tool
|
|
34
|
-
* need, and a confidence that only just misses. Both mean the model should not
|
|
35
|
-
* answer from memory, and neither makes it a hard question — looking it up is
|
|
36
|
-
* cheaper than planning it, and likelier to be right.
|
|
37
|
-
*/
|
|
38
11
|
export function routeFor(estimates) {
|
|
39
12
|
const redTeam = SERIOUS.has(estimates.stakes);
|
|
40
13
|
const clear = EASY.has(estimates.difficulty) && estimates.ambiguity < CLEAR;
|
|
@@ -7,21 +7,10 @@ function reTriage(currentTask) {
|
|
|
7
7
|
const task = currentTask?.trim();
|
|
8
8
|
return task ? [{ role: "user", content: `<CURRENT_TASK>\n${task}\n</CURRENT_TASK>` }] : [];
|
|
9
9
|
}
|
|
10
|
-
/** Stands in for an attachment nobody sent. */
|
|
11
10
|
const NOTHING_ATTACHED = "(none)";
|
|
12
|
-
/**
|
|
13
|
-
* An attachment slot, filled or explicitly empty.
|
|
14
|
-
*
|
|
15
|
-
* The prompt's question box wraps every request in an `<attachments>` block, so
|
|
16
|
-
* the tags are sent whether or not anything was attached. Left empty they read
|
|
17
|
-
* as an attachment the router cannot open, which it prices as ambiguity and
|
|
18
|
-
* tool need — enough on its own to push a plain question from `standard` to
|
|
19
|
-
* `full` and pay for the long route on every call that attaches nothing.
|
|
20
|
-
*/
|
|
21
11
|
export function attached(content) {
|
|
22
12
|
return content?.trim() ? content : NOTHING_ATTACHED;
|
|
23
13
|
}
|
|
24
|
-
/** The reply came back, but not as the object the router is supposed to return. */
|
|
25
14
|
export class TriageReplyError extends Error {
|
|
26
15
|
raw;
|
|
27
16
|
constructor(raw) {
|
|
@@ -30,18 +19,6 @@ export class TriageReplyError extends Error {
|
|
|
30
19
|
this.name = "TriageReplyError";
|
|
31
20
|
}
|
|
32
21
|
}
|
|
33
|
-
/**
|
|
34
|
-
* Stage 0. Pulls the router, calls it, and hands back the parsed signals.
|
|
35
|
-
*
|
|
36
|
-
* The call is recorded once, as the prompt's rollout — the model client is told
|
|
37
|
-
* not to trace, or the same tokens land again as usage and every run prices
|
|
38
|
-
* double.
|
|
39
|
-
*
|
|
40
|
-
* An open task is sent as `<CURRENT_TASK>…</CURRENT_TASK>`. The prompt names
|
|
41
|
-
* the block but never shows its markup, so this is the form it is given — keep
|
|
42
|
-
* the two matching, or the router is pattern-matching on something it was never
|
|
43
|
-
* shown.
|
|
44
|
-
*/
|
|
45
22
|
export async function invoke(client, llm, input, opts = {}) {
|
|
46
23
|
announce(opts.emit, TRIAGE_SLUG, opts.model);
|
|
47
24
|
return attempting(async () => {
|
|
@@ -1,13 +1,5 @@
|
|
|
1
|
-
/** The stored router prompt. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const TRIAGE_SLUG = "triage-router-v1-81c2cc";
|
|
3
2
|
const UNIT = { type: "number", minimum: 0, maximum: 1 };
|
|
4
|
-
/**
|
|
5
|
-
* What the reply must be, as the prompt itself specifies it.
|
|
6
|
-
*
|
|
7
|
-
* `route`, `next_stage` and the rest are deliberately absent: the prompt forbids
|
|
8
|
-
* them and code derives them, so a model that returns one fails here rather
|
|
9
|
-
* than having it read downstream.
|
|
10
|
-
*/
|
|
11
3
|
export const TRIAGE_SCHEMA = {
|
|
12
4
|
type: "object",
|
|
13
5
|
properties: {
|
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* How much of any one string an artifact may carry into a prompt.
|
|
3
|
-
*
|
|
4
|
-
* A tool answer is the one artifact with no bound on its size: everything else
|
|
5
|
-
* is a model reply against a schema, and a wide `deep_recall` can be larger
|
|
6
|
-
* than every other stage put together.
|
|
7
|
-
*/
|
|
8
1
|
const DEFAULT_LIMIT = 2000;
|
|
9
2
|
const cut = (text, limit) => `${text.slice(0, limit)}… (${text.length - limit} more characters)`;
|
|
10
3
|
function shorten(value, limit) {
|
|
@@ -17,15 +10,5 @@ function shorten(value, limit) {
|
|
|
17
10
|
}
|
|
18
11
|
return value;
|
|
19
12
|
}
|
|
20
|
-
/**
|
|
21
|
-
* The artifacts as a stage that reasons about them reads them, long strings cut.
|
|
22
|
-
*
|
|
23
|
-
* Every stage handed the whole ledger — 3A, 3B, 3C, 4A and R — is reading it
|
|
24
|
-
* for the decisions it records, and a decision is legible from the head of a
|
|
25
|
-
* tool answer. 6C is the exception and takes the artifacts whole: it cites
|
|
26
|
-
* evidence rather than reviewing choices, and a citation cut in half is a
|
|
27
|
-
* citation of nothing.
|
|
28
|
-
*/
|
|
29
13
|
export const digest = (artifacts, limit = DEFAULT_LIMIT) => shorten(artifacts, limit);
|
|
30
|
-
/** One artifact, cut the same way — what an event carries rather than a prompt. */
|
|
31
14
|
export const digestOne = (artifact, limit = DEFAULT_LIMIT) => shorten(artifact, limit);
|
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* A route the table names with nothing behind it.
|
|
3
|
-
*
|
|
4
|
-
* Thrown only where the fallback ran — `RouteResult.route` is null — which is
|
|
5
|
-
* the one case where the branch really does not exist. A branch that ran and
|
|
6
|
-
* threw is a `RouteFailedError`, and the two are separate classes because the
|
|
7
|
-
* answer to them is: write the branch, versus try again.
|
|
8
|
-
*/
|
|
9
1
|
export class RouteNotBuiltError extends Error {
|
|
10
2
|
route;
|
|
11
3
|
estimates;
|
|
@@ -20,14 +12,6 @@ export class RouteNotBuiltError extends Error {
|
|
|
20
12
|
this.name = "RouteNotBuiltError";
|
|
21
13
|
}
|
|
22
14
|
}
|
|
23
|
-
/**
|
|
24
|
-
* A branch that ran and threw, with the estimates that chose it.
|
|
25
|
-
*
|
|
26
|
-
* `detail` is the branch's own message, carried up because the graph underneath
|
|
27
|
-
* catches whatever a branch throws and reports only a failed run. A caller
|
|
28
|
-
* catching this should decide whether the cause is worth retrying; catching
|
|
29
|
-
* `RouteNotBuiltError` never is.
|
|
30
|
-
*/
|
|
31
15
|
export class RouteFailedError extends Error {
|
|
32
16
|
route;
|
|
33
17
|
estimates;
|
|
@@ -42,11 +26,9 @@ export class RouteFailedError extends Error {
|
|
|
42
26
|
this.name = "RouteFailedError";
|
|
43
27
|
}
|
|
44
28
|
}
|
|
45
|
-
/** A branch named in the routing table with nothing behind it yet. */
|
|
46
29
|
export const notBuilt = (route) => () => {
|
|
47
30
|
throw new RouteNotBuiltError(route);
|
|
48
31
|
};
|
|
49
|
-
/** A route that has to look something up, with nothing to look it up with. */
|
|
50
32
|
export class NoToolsError extends Error {
|
|
51
33
|
constructor() {
|
|
52
34
|
super("This route looks something up, so it needs a ToolBus in opts.tools.");
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { NoToolsError, RouteFailedError, RouteNotBuiltError } from "./errors.js";
|
|
2
|
-
export type { Entry, Escalation, Ledger, Recorded, StageId, StageUsage } from "./ledger.js";
|
|
3
|
-
export { newLedger, STAGE_NAMES, STAGES } from "./ledger.js";
|
|
2
|
+
export type { Entry, Escalation, Ledger, Recorded, StageId, StageKey, StageName, StageUsage } from "./ledger.js";
|
|
3
|
+
export { newLedger, STAGE_KEYS, STAGE_NAMES, STAGES } from "./ledger.js";
|
|
4
4
|
export { run } from "./run.js";
|
|
5
5
|
export type { NoesisStream } from "./stream.js";
|
|
6
6
|
export { runStream } from "./stream.js";
|
package/dist/noesis/run/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { NoToolsError, RouteFailedError, RouteNotBuiltError } from "./errors.js";
|
|
2
|
-
export { newLedger, STAGE_NAMES, STAGES } from "./ledger.js";
|
|
2
|
+
export { newLedger, STAGE_KEYS, STAGE_NAMES, STAGES } from "./ledger.js";
|
|
3
3
|
export { run } from "./run.js";
|
|
4
4
|
export { runStream } from "./stream.js";
|
|
@@ -4,7 +4,55 @@ import { type PartialUsage, type Usage } from "../usage.js";
|
|
|
4
4
|
export declare const STAGES: readonly ["0", "1", "2", "3A", "3B", "3C", "4", "4A", "4B", "5A", "5B", "5", "5C", "6A", "6B", "6C", "R"];
|
|
5
5
|
export type StageId = (typeof STAGES)[number];
|
|
6
6
|
/** What a stage is called where a person or a model has to read it. */
|
|
7
|
-
export declare const STAGE_NAMES:
|
|
7
|
+
export declare const STAGE_NAMES: {
|
|
8
|
+
readonly "0": "triage";
|
|
9
|
+
readonly "1": "question brief";
|
|
10
|
+
readonly "2": "decomposition";
|
|
11
|
+
readonly "3A": "plan candidates";
|
|
12
|
+
readonly "3B": "premortem";
|
|
13
|
+
readonly "3C": "cqot gate";
|
|
14
|
+
readonly "4": "execution";
|
|
15
|
+
readonly "4A": "think checkpoint";
|
|
16
|
+
readonly "4B": "discriminating test";
|
|
17
|
+
readonly "5A": "verification questions";
|
|
18
|
+
readonly "5B": "verification answers";
|
|
19
|
+
readonly "5": "verification";
|
|
20
|
+
readonly "5C": "standards review";
|
|
21
|
+
readonly "6A": "closure checklist";
|
|
22
|
+
readonly "6B": "red team";
|
|
23
|
+
readonly "6C": "answer";
|
|
24
|
+
readonly R: "root cause";
|
|
25
|
+
};
|
|
26
|
+
/** A stage by the name it announces itself under. */
|
|
27
|
+
export type StageName = (typeof STAGE_NAMES)[StageId];
|
|
28
|
+
/**
|
|
29
|
+
* What a stage is called where it is a key rather than prose.
|
|
30
|
+
*
|
|
31
|
+
* `STAGE_NAMES` with the spaces closed up, kept apart from it because those
|
|
32
|
+
* names are read back out of the artifacts by the answer and cannot be renamed
|
|
33
|
+
* for a config's convenience.
|
|
34
|
+
*/
|
|
35
|
+
export declare const STAGE_KEYS: {
|
|
36
|
+
readonly "0": "triage";
|
|
37
|
+
readonly "1": "question_brief";
|
|
38
|
+
readonly "2": "decomposition";
|
|
39
|
+
readonly "3A": "plan_candidates";
|
|
40
|
+
readonly "3B": "premortem";
|
|
41
|
+
readonly "3C": "cqot_gate";
|
|
42
|
+
readonly "4": "execution";
|
|
43
|
+
readonly "4A": "think_checkpoint";
|
|
44
|
+
readonly "4B": "discriminating_test";
|
|
45
|
+
readonly "5A": "verification_questions";
|
|
46
|
+
readonly "5B": "verification_answers";
|
|
47
|
+
readonly "5": "verification";
|
|
48
|
+
readonly "5C": "standards_review";
|
|
49
|
+
readonly "6A": "closure_checklist";
|
|
50
|
+
readonly "6B": "red_team";
|
|
51
|
+
readonly "6C": "answer";
|
|
52
|
+
readonly R: "root_cause";
|
|
53
|
+
};
|
|
54
|
+
/** A stage by the key a config names it under. */
|
|
55
|
+
export type StageKey = (typeof STAGE_KEYS)[StageId];
|
|
8
56
|
/** What one stage left behind, and what it cost. */
|
|
9
57
|
export interface Entry extends Usage {
|
|
10
58
|
stage: StageId;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { addUsage, NO_USAGE, usageOf } from "../usage.js";
|
|
2
2
|
import { digestOne } from "./artifacts.js";
|
|
3
|
-
/** Every stage the loop can run, in the order it runs them. */
|
|
4
3
|
export const STAGES = [
|
|
5
4
|
"0",
|
|
6
5
|
"1",
|
|
@@ -20,7 +19,6 @@ export const STAGES = [
|
|
|
20
19
|
"6C",
|
|
21
20
|
"R",
|
|
22
21
|
];
|
|
23
|
-
/** What a stage is called where a person or a model has to read it. */
|
|
24
22
|
export const STAGE_NAMES = {
|
|
25
23
|
"0": "triage",
|
|
26
24
|
"1": "question brief",
|
|
@@ -40,19 +38,27 @@ export const STAGE_NAMES = {
|
|
|
40
38
|
"6C": "answer",
|
|
41
39
|
R: "root cause",
|
|
42
40
|
};
|
|
41
|
+
export const STAGE_KEYS = {
|
|
42
|
+
"0": "triage",
|
|
43
|
+
"1": "question_brief",
|
|
44
|
+
"2": "decomposition",
|
|
45
|
+
"3A": "plan_candidates",
|
|
46
|
+
"3B": "premortem",
|
|
47
|
+
"3C": "cqot_gate",
|
|
48
|
+
"4": "execution",
|
|
49
|
+
"4A": "think_checkpoint",
|
|
50
|
+
"4B": "discriminating_test",
|
|
51
|
+
"5A": "verification_questions",
|
|
52
|
+
"5B": "verification_answers",
|
|
53
|
+
"5": "verification",
|
|
54
|
+
"5C": "standards_review",
|
|
55
|
+
"6A": "closure_checklist",
|
|
56
|
+
"6B": "red_team",
|
|
57
|
+
"6C": "answer",
|
|
58
|
+
R: "root_cause",
|
|
59
|
+
};
|
|
43
60
|
const DEFAULT_MAX_RETRIES = 5;
|
|
44
61
|
const rolloutsOf = (result) => "rolloutIds" in result ? result.rolloutIds : [result.rolloutId];
|
|
45
|
-
/**
|
|
46
|
-
* What a run leaves behind, as one thing.
|
|
47
|
-
*
|
|
48
|
-
* A branch that keeps its own artifacts hands 6C a different shape from the
|
|
49
|
-
* next branch, and nothing can invalidate a stale one on a retry. Cost is
|
|
50
|
-
* counted here too, because the graph underneath reads a node's tokens off its
|
|
51
|
-
* return value — and a branch that returns text reports none.
|
|
52
|
-
*
|
|
53
|
-
* `maxRetries` is a total across every gate: one gate flapping and five gates
|
|
54
|
-
* failing once each are the same amount of a run going nowhere.
|
|
55
|
-
*/
|
|
56
62
|
export function newLedger(maxRetries = DEFAULT_MAX_RETRIES, emit) {
|
|
57
63
|
const filed = [];
|
|
58
64
|
const spentOn = [];
|
package/dist/noesis/run/loop.js
CHANGED
|
@@ -4,17 +4,7 @@ import { invoke as standardsReview } from "../gates/standards-review/index.js";
|
|
|
4
4
|
import { STAGE_NAMES } from "./ledger.js";
|
|
5
5
|
import { dropPoint, resumeAt } from "./redo.js";
|
|
6
6
|
import { checks, reflect } from "./verify.js";
|
|
7
|
-
|
|
8
|
-
* One return trip: spend a retry, find the root cause, cut the ledger back.
|
|
9
|
-
*
|
|
10
|
-
* The budget is checked before R runs, because a run with nothing left to spend
|
|
11
|
-
* has no use for knowing where it went wrong.
|
|
12
|
-
*
|
|
13
|
-
* The ledger is cut where the loop actually restarts, not where R pointed. A
|
|
14
|
-
* gate is the same gate on the same input, so anything it names redoes the
|
|
15
|
-
* checks — and cutting only from the gate would leave the stale checks filed
|
|
16
|
-
* beside the fresh ones.
|
|
17
|
-
*/
|
|
7
|
+
import { stageOptionsFor } from "./options.js";
|
|
18
8
|
async function stepBack(client, llm, question, opts, state, gates, gate, reason, hint) {
|
|
19
9
|
if (!state.ledger.retry(gate)) {
|
|
20
10
|
state.ledger.escalate(gate, `Out of retries. ${gate} last said: ${reason}`);
|
|
@@ -25,16 +15,9 @@ async function stepBack(client, llm, question, opts, state, gates, gate, reason,
|
|
|
25
15
|
state.ledger.dropFrom(again === "produce" ? dropPoint(redo) : "5A");
|
|
26
16
|
return { stopped: false, draft: again === "produce" ? await gates.reproduce() : undefined };
|
|
27
17
|
}
|
|
28
|
-
/**
|
|
29
|
-
* The gates, in the order they earn their place.
|
|
30
|
-
*
|
|
31
|
-
* 5C reads the answer against its standards, 6A asks the last questions before
|
|
32
|
-
* closing, and 6B attacks what survived both. The first refusal returns, so a
|
|
33
|
-
* gate that fires early saves the later ones the work.
|
|
34
|
-
*/
|
|
35
18
|
async function judge(client, llm, opts, state, checked, gates) {
|
|
36
19
|
if (gates.standards) {
|
|
37
|
-
const review = state.ledger.record("5C", await standardsReview(client, llm, { question: checked, ...opts.context }, opts
|
|
20
|
+
const review = state.ledger.record("5C", await standardsReview(client, llm, { question: checked, ...opts.context }, stageOptionsFor(opts, "5C")));
|
|
38
21
|
if (review.overall !== "accept") {
|
|
39
22
|
return {
|
|
40
23
|
stage: "5C",
|
|
@@ -43,7 +26,7 @@ async function judge(client, llm, opts, state, checked, gates) {
|
|
|
43
26
|
}
|
|
44
27
|
state.emit?.({ type: "gate.passed", stage: "5C", name: STAGE_NAMES["5C"] });
|
|
45
28
|
}
|
|
46
|
-
const closure = state.ledger.record("6A", await closureChecklist(client, llm, { question: checked, ...opts.context }, opts
|
|
29
|
+
const closure = state.ledger.record("6A", await closureChecklist(client, llm, { question: checked, ...opts.context }, stageOptionsFor(opts, "6A")));
|
|
47
30
|
if (closure.gate !== "close") {
|
|
48
31
|
return {
|
|
49
32
|
stage: "6A",
|
|
@@ -54,7 +37,7 @@ async function judge(client, llm, opts, state, checked, gates) {
|
|
|
54
37
|
}
|
|
55
38
|
state.emit?.({ type: "gate.passed", stage: "6A", name: STAGE_NAMES["6A"] });
|
|
56
39
|
if (gates.redTeam) {
|
|
57
|
-
const attacked = state.ledger.record("6B", await redTeam(client, llm, { question: checked, ...opts.context }, opts
|
|
40
|
+
const attacked = state.ledger.record("6B", await redTeam(client, llm, { question: checked, ...opts.context }, stageOptionsFor(opts, "6B")));
|
|
58
41
|
if (attacked.verdict !== "survives") {
|
|
59
42
|
return { stage: "6B", reason: `${attacked.verdict}: ${attacked.would_break_it}` };
|
|
60
43
|
}
|
|
@@ -62,7 +45,6 @@ async function judge(client, llm, opts, state, checked, gates) {
|
|
|
62
45
|
}
|
|
63
46
|
return undefined;
|
|
64
47
|
}
|
|
65
|
-
/** Check the draft, and keep going back until every gate lets it close. */
|
|
66
48
|
export async function settle(client, llm, question, opts, state, first, gates) {
|
|
67
49
|
if (state.ledger.escalated())
|
|
68
50
|
return;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { AnswerOptions } from "../producers/toulmin/index.js";
|
|
2
|
+
import type { StageOptions } from "../stage.js";
|
|
3
|
+
import type { NoesisOptions } from "../types.js";
|
|
4
|
+
import { type StageId } from "./ledger.js";
|
|
5
|
+
/**
|
|
6
|
+
* What one stage runs on: `stages[id]` over `stage`, with `params` dropped
|
|
7
|
+
* rather than inherited when the stage names a model of its own.
|
|
8
|
+
*/
|
|
9
|
+
export declare function stageOptionsFor(opts: NoesisOptions, id: StageId): StageOptions;
|
|
10
|
+
/** The answer's tiers, defaulting to whatever `stages.answer` named. */
|
|
11
|
+
export declare function answerOptions(opts: NoesisOptions): AnswerOptions;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { STAGE_KEYS } from "./ledger.js";
|
|
2
|
+
export function stageOptionsFor(opts, id) {
|
|
3
|
+
const base = opts.stage ?? {};
|
|
4
|
+
const per = opts.stages?.[STAGE_KEYS[id]];
|
|
5
|
+
if (!per)
|
|
6
|
+
return base;
|
|
7
|
+
const ownModel = per.model !== undefined && per.model !== base.model;
|
|
8
|
+
const params = ownModel ? per.params : { ...base.params, ...per.params };
|
|
9
|
+
const rest = Object.fromEntries(Object.entries({ ...base, ...per }).filter(([field]) => field !== "params"));
|
|
10
|
+
return params && Object.keys(params).length > 0 ? { ...rest, params } : rest;
|
|
11
|
+
}
|
|
12
|
+
export function answerOptions(opts) {
|
|
13
|
+
const base = opts.answer ?? {};
|
|
14
|
+
const sixC = opts.stages?.[STAGE_KEYS["6C"]];
|
|
15
|
+
if (base.tiers || !sixC?.model)
|
|
16
|
+
return base;
|
|
17
|
+
return { ...base, tiers: [{ model: sixC.model, ...(sixC.params ? { params: sixC.params } : {}) }] };
|
|
18
|
+
}
|
package/dist/noesis/run/plan.js
CHANGED
|
@@ -2,33 +2,21 @@ import { invoke as cqotGate } from "../gates/cqot-gate/index.js";
|
|
|
2
2
|
import { invoke as premortem } from "../gates/premortem/index.js";
|
|
3
3
|
import { invoke as planCandidates } from "../producers/plan-candidates/index.js";
|
|
4
4
|
import { soFar } from "./produce.js";
|
|
5
|
-
|
|
6
|
-
* Stage 3A, and the audits only `full` pays for.
|
|
7
|
-
*
|
|
8
|
-
* Both long routes plan, because 4A judges each step against the `tolerance` 3A
|
|
9
|
-
* wrote for it and a run with no plan has nothing to check against. What `full`
|
|
10
|
-
* adds is 3B and 3C: the plan assumed dead, then gated.
|
|
11
|
-
*
|
|
12
|
-
* Chained rather than forked, because 3C points at the step at fault and 3B may
|
|
13
|
-
* have patched that step. Run side by side, 3C would gate a plan 3B had already
|
|
14
|
-
* said was wrong. A patch is prose, so `patch_and_proceed` proceeds and the
|
|
15
|
-
* premortem is filed — 3C reads the plan with the patches beside it rather than
|
|
16
|
-
* a plan nobody rewrote.
|
|
17
|
-
*/
|
|
5
|
+
import { stageOptionsFor } from "./options.js";
|
|
18
6
|
export async function plan(client, llm, question, opts, state, audits) {
|
|
19
7
|
for (;;) {
|
|
20
8
|
state.ledger.dropFrom("3A");
|
|
21
|
-
state.ledger.record("3A", await planCandidates(client, llm, { question: soFar(state, question), ...opts.context }, opts
|
|
9
|
+
state.ledger.record("3A", await planCandidates(client, llm, { question: soFar(state, question), ...opts.context }, stageOptionsFor(opts, "3A")));
|
|
22
10
|
if (!audits)
|
|
23
11
|
return;
|
|
24
|
-
const dead = state.ledger.record("3B", await premortem(client, llm, { question: soFar(state, question), ...opts.context }, opts
|
|
12
|
+
const dead = state.ledger.record("3B", await premortem(client, llm, { question: soFar(state, question), ...opts.context }, stageOptionsFor(opts, "3B")));
|
|
25
13
|
if (dead.fatal || dead.verdict === "replan") {
|
|
26
14
|
if (state.ledger.retry("3B"))
|
|
27
15
|
continue;
|
|
28
16
|
state.ledger.escalate("3B", "Out of retries: the premortem keeps killing the plan.");
|
|
29
17
|
return;
|
|
30
18
|
}
|
|
31
|
-
const gate = state.ledger.record("3C", await cqotGate(client, llm, { question: soFar(state, question), ...opts.context }, opts
|
|
19
|
+
const gate = state.ledger.record("3C", await cqotGate(client, llm, { question: soFar(state, question), ...opts.context }, stageOptionsFor(opts, "3C")));
|
|
32
20
|
if (gate.gate === "pass")
|
|
33
21
|
return;
|
|
34
22
|
if (state.ledger.retry("3C"))
|