@spendgraph/workflows 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/noesis/attribution.js +0 -20
- package/dist/noesis/context.js +0 -8
- package/dist/noesis/draft/draft.js +0 -13
- package/dist/noesis/events.js +0 -10
- package/dist/noesis/execute/execute.js +0 -17
- package/dist/noesis/execute/order.js +0 -8
- package/dist/noesis/gates/closure-checklist/invoke.js +0 -1
- package/dist/noesis/gates/closure-checklist/prompt.js +0 -2
- package/dist/noesis/gates/cqot-gate/invoke.js +0 -1
- package/dist/noesis/gates/cqot-gate/prompt.js +0 -2
- package/dist/noesis/gates/premortem/invoke.js +0 -1
- package/dist/noesis/gates/premortem/prompt.js +0 -2
- package/dist/noesis/gates/red-team/invoke.js +0 -1
- package/dist/noesis/gates/red-team/prompt.js +0 -2
- package/dist/noesis/gates/standards-review/invoke.js +0 -1
- package/dist/noesis/gates/standards-review/prompt.js +0 -2
- package/dist/noesis/gates/think-checkpoint/invoke.js +0 -1
- package/dist/noesis/gates/think-checkpoint/prompt.js +0 -2
- package/dist/noesis/producers/cove-answers/invoke.js +0 -1
- package/dist/noesis/producers/cove-answers/prompt.js +0 -2
- package/dist/noesis/producers/cove-questions/invoke.js +0 -1
- package/dist/noesis/producers/cove-questions/prompt.js +0 -2
- package/dist/noesis/producers/decompose/invoke.js +0 -1
- package/dist/noesis/producers/decompose/prompt.js +0 -18
- package/dist/noesis/producers/discriminating-test/invoke.js +0 -1
- package/dist/noesis/producers/discriminating-test/prompt.js +0 -2
- package/dist/noesis/producers/plan-candidates/invoke.js +0 -1
- package/dist/noesis/producers/plan-candidates/prompt.js +0 -2
- package/dist/noesis/producers/qn-brief/invoke.js +0 -1
- package/dist/noesis/producers/qn-brief/prompt.js +0 -9
- package/dist/noesis/producers/toulmin/answer.js +0 -21
- package/dist/noesis/producers/toulmin/invoke.js +0 -10
- package/dist/noesis/producers/toulmin/prompt.js +0 -9
- package/dist/noesis/producers/toulmin/render.js +0 -24
- package/dist/noesis/question.js +0 -19
- package/dist/noesis/repair/reflection/invoke.js +0 -1
- package/dist/noesis/repair/reflection/prompt.js +0 -2
- package/dist/noesis/retry.js +0 -34
- package/dist/noesis/route.js +0 -27
- package/dist/noesis/router/triage/invoke.js +0 -23
- package/dist/noesis/router/triage/prompt.js +0 -8
- package/dist/noesis/run/artifacts.js +0 -17
- package/dist/noesis/run/errors.js +0 -18
- package/dist/noesis/run/ledger.js +0 -13
- package/dist/noesis/run/loop.js +0 -19
- package/dist/noesis/run/plan.js +0 -13
- package/dist/noesis/run/produce.js +0 -34
- package/dist/noesis/run/redo.js +0 -20
- package/dist/noesis/run/result.js +0 -8
- package/dist/noesis/run/run.js +0 -21
- package/dist/noesis/run/stages/classify.js +0 -1
- package/dist/noesis/run/stages/direct.js +0 -1
- package/dist/noesis/run/stages/full.js +0 -9
- package/dist/noesis/run/stages/retrieve.js +0 -8
- package/dist/noesis/run/stages/standard.js +0 -7
- package/dist/noesis/run/state.js +0 -4
- package/dist/noesis/run/stream.js +0 -13
- package/dist/noesis/run/verify.js +0 -18
- package/dist/noesis/slugs.js +0 -14
- package/dist/noesis/stage.js +0 -10
- package/dist/noesis/started.js +0 -15
- package/dist/noesis/usage.js +0 -16
- package/dist/noesis/verify/verify.js +0 -29
- package/package.json +7 -7
package/README.md
CHANGED
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
Ready-made workflows assembled from the other packages. One so far: **noesis**,
|
|
4
4
|
a question-answering loop that decides how much of itself a question is worth.
|
|
5
5
|
|
|
6
|
+
```sh
|
|
7
|
+
npm install @spendgraph/workflows
|
|
8
|
+
```
|
|
9
|
+
|
|
6
10
|
```ts
|
|
7
11
|
import { run } from "@spendgraph/workflows";
|
|
8
12
|
|
|
@@ -147,3 +151,7 @@ with `04-retrieve.mjs`: it is the cheapest, and it exercises triage, the
|
|
|
147
151
|
dispatcher, the ledger and 6C. The routes that look something up need one of
|
|
148
152
|
the two lookups configured — `LOCUSGRAPH_AGENT_SECRET` for `deep_recall`, or
|
|
149
153
|
`PERPLEXITY_API_KEY` for `web_search` — and say so rather than half-running.
|
|
154
|
+
|
|
155
|
+
## License
|
|
156
|
+
|
|
157
|
+
MIT
|
|
@@ -1,14 +1,6 @@
|
|
|
1
1
|
import { STAGE_NAMES, STAGES } from "./run/ledger.js";
|
|
2
2
|
const BY_NAME = new Map(Object.entries(STAGE_NAMES).map(([id, name]) => [name.toLowerCase(), id]));
|
|
3
3
|
const IDS = new Set(STAGES);
|
|
4
|
-
/**
|
|
5
|
-
* The stage a ground says it came from, or nothing.
|
|
6
|
-
*
|
|
7
|
-
* Lenient because the field is free text on a model reply, and "5B", "stage 5B"
|
|
8
|
-
* and "verification answers" are the same citation. It will not guess: a string
|
|
9
|
-
* that resolves to nothing is a ground that cannot be traced, which is the
|
|
10
|
-
* finding rather than a parsing problem to work around.
|
|
11
|
-
*/
|
|
12
4
|
export function stageCited(source) {
|
|
13
5
|
const raw = source.trim();
|
|
14
6
|
const id = raw
|
|
@@ -19,18 +11,6 @@ export function stageCited(source) {
|
|
|
19
11
|
return id;
|
|
20
12
|
return BY_NAME.get(raw.toLowerCase());
|
|
21
13
|
}
|
|
22
|
-
/**
|
|
23
|
-
* The grounds citing a stage this run never ran.
|
|
24
|
-
*
|
|
25
|
-
* 6C compiles `grounds[]` out of the artifacts and each one names its
|
|
26
|
-
* `source_stage`; the ledger knows which stages were actually paid for. A
|
|
27
|
-
* citation of anything else is the stage writing evidence rather than
|
|
28
|
-
* assembling it, and it is the cheapest check available that catches a
|
|
29
|
-
* fabricated answer without a judge and without a reference.
|
|
30
|
-
*
|
|
31
|
-
* Returns the `source_stage` strings as written, because what the model claimed
|
|
32
|
-
* is the useful thing to read back.
|
|
33
|
-
*/
|
|
34
14
|
export function unattributed(answer, ran) {
|
|
35
15
|
const filed = new Set(ran);
|
|
36
16
|
const grounds = answer.grounds ?? [];
|
package/dist/noesis/context.js
CHANGED
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The request with the `<context>` block ahead of it.
|
|
3
|
-
*
|
|
4
|
-
* The prompts name `now` and `locale` but never show the markup they expect
|
|
5
|
-
* them in, so this is the form they are given. An absent fact is omitted rather
|
|
6
|
-
* than sent empty: a missing one is recorded as something to find out, an empty
|
|
7
|
-
* one reads as a fact already in hand.
|
|
8
|
-
*/
|
|
9
1
|
export function asked(input) {
|
|
10
2
|
const facts = [
|
|
11
3
|
input.now ? `now: ${input.now}` : "",
|
|
@@ -1,11 +1,4 @@
|
|
|
1
1
|
const NEVER_RAN = "It depended on something that was never answered.";
|
|
2
|
-
/**
|
|
3
|
-
* The sub-answers, paired back to the sub-questions that asked for them.
|
|
4
|
-
*
|
|
5
|
-
* A sub-question that produced nothing is named rather than dropped: 6A asks
|
|
6
|
-
* whether every one was answered, and a draft that quietly omits the failures
|
|
7
|
-
* would let it say yes.
|
|
8
|
-
*/
|
|
9
2
|
export function assemble(question, decomposition, execution) {
|
|
10
3
|
const asked = new Map(decomposition.sub_questions.map((sub) => [sub.id, sub.question]));
|
|
11
4
|
const findings = [];
|
|
@@ -29,12 +22,6 @@ export function assemble(question, decomposition, execution) {
|
|
|
29
22
|
}
|
|
30
23
|
const block = (tag, body) => `<${tag}>\n${body}\n</${tag}>`;
|
|
31
24
|
const asText = (answer) => typeof answer === "string" ? answer : JSON.stringify(answer, null, 2);
|
|
32
|
-
/**
|
|
33
|
-
* The draft as a stage reads it.
|
|
34
|
-
*
|
|
35
|
-
* Tagged the way `asked()` tags the context block, because every stage
|
|
36
|
-
* downstream takes this through `{question}` and has to tell the parts apart.
|
|
37
|
-
*/
|
|
38
25
|
export function render(draft) {
|
|
39
26
|
const parts = [
|
|
40
27
|
block("question", draft.question),
|
package/dist/noesis/events.js
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* One emitter per run, stamping order and elapsed time.
|
|
3
|
-
*
|
|
4
|
-
* Deliberately synchronous and deliberately swallowing: a listener that throws
|
|
5
|
-
* is a bug in the listener, and letting it take the run down would make
|
|
6
|
-
* watching a run more dangerous than not watching one. Returns `undefined`
|
|
7
|
-
* where nobody is listening, so every emit site can stay a `?.` call and cost
|
|
8
|
-
* nothing.
|
|
9
|
-
*/
|
|
10
1
|
export function emitter(listener, now = Date.now) {
|
|
11
2
|
if (!listener)
|
|
12
3
|
return undefined;
|
|
@@ -18,7 +9,6 @@ export function emitter(listener, now = Date.now) {
|
|
|
18
9
|
listener(stamped);
|
|
19
10
|
}
|
|
20
11
|
catch {
|
|
21
|
-
// A listener that throws does not get to fail the run.
|
|
22
12
|
}
|
|
23
13
|
};
|
|
24
14
|
}
|
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
import { order } from "./order.js";
|
|
2
|
-
/**
|
|
3
|
-
* What each hint reaches for, and where the question goes.
|
|
4
|
-
*
|
|
5
|
-
* `code_exec` and `get_documents` are unwired because nothing ships them —
|
|
6
|
-
* a sub-question asking for one is reported rather than quietly dropped.
|
|
7
|
-
*/
|
|
8
2
|
export const DEFAULT_TOOLS = {
|
|
9
3
|
calculator: { name: "calculate", arg: "expression" },
|
|
10
4
|
deeprecall: { name: "deep_recall", arg: "question" },
|
|
@@ -54,17 +48,6 @@ async function answerOne(bus, wiring, sub, wave, emit) {
|
|
|
54
48
|
? { ...base, status: "failed", reason: result.error, latencyMs: result.latencyMs }
|
|
55
49
|
: { ...base, status: "answered", result: parsed(result.output), latencyMs: result.latencyMs });
|
|
56
50
|
}
|
|
57
|
-
/**
|
|
58
|
-
* Stage 4. Every sub-question put through the tool its hint named.
|
|
59
|
-
*
|
|
60
|
-
* The dispatcher dispatches and does not reason: `deep_recall` and `web_search`
|
|
61
|
-
* already return a synthesised answer, and a sub-question with no tool is
|
|
62
|
-
* recorded as unanswered rather than guessed at.
|
|
63
|
-
*
|
|
64
|
-
* `done_when` is carried rather than acted on. It describes an observable
|
|
65
|
-
* condition, which is 4A's job to judge against what actually happened, not
|
|
66
|
-
* something this can evaluate.
|
|
67
|
-
*/
|
|
68
51
|
export async function execute(bus, plan, opts = {}) {
|
|
69
52
|
const wiring = { ...DEFAULT_TOOLS, ...opts.tools };
|
|
70
53
|
const { waves, unresolved } = order(plan);
|
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The plan, grouped into what may run together.
|
|
3
|
-
*
|
|
4
|
-
* `read_only` siblings share a wave and `synthesis` takes one alone, which is
|
|
5
|
-
* the distinction the stage went to the trouble of typing. A dependency on an
|
|
6
|
-
* id nothing created can never be satisfied, so it lands in `unresolved`
|
|
7
|
-
* alongside a cycle rather than being run in the wrong order.
|
|
8
|
-
*/
|
|
9
1
|
export function order(plan) {
|
|
10
2
|
const known = new Set(plan.map((sub) => sub.id));
|
|
11
3
|
const done = new Set();
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { CLOSURE_SCHEMA, CLOSURE_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 6A. The last questions before an answer is allowed to close. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, CLOSURE_SLUG, CLOSURE_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { GATE_SCHEMA, GATE_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 3C. Eight questions asked of the plan, and whether it may proceed. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, GATE_SLUG, GATE_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { PREMORTEM_SCHEMA, PREMORTEM_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 3B. The plan assumed dead, and what killed it. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, PREMORTEM_SLUG, PREMORTEM_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { RED_TEAM_SCHEMA, RED_TEAM_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 6B. The answer attacked, and whether it survived. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, RED_TEAM_SLUG, RED_TEAM_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { STANDARDS_SCHEMA, STANDARDS_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 5C. The answer against eight standards, each passed or failed on the evidence. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, STANDARDS_SLUG, STANDARDS_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/** Stage 5C. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const STANDARDS_SLUG = "ctr-5c-standards-review-17c6w4";
|
|
3
|
-
/** What the reply must be, as the prompt itself specifies it. */
|
|
4
2
|
export const STANDARDS_SCHEMA = {
|
|
5
3
|
type: "object",
|
|
6
4
|
properties: {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { CHECKPOINT_SCHEMA, CHECKPOINT_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 4A. What was predicted against what happened, and whether to carry on. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, CHECKPOINT_SLUG, CHECKPOINT_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/** Stage 4A. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const CHECKPOINT_SLUG = "ctr-4a-think-checkpoint-8d5302";
|
|
3
|
-
/** What the reply must be, as the prompt itself specifies it. */
|
|
4
2
|
export const CHECKPOINT_SCHEMA = {
|
|
5
3
|
type: "object",
|
|
6
4
|
properties: {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { COVE_ANSWERS_SCHEMA, COVE_ANSWERS_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 5B. Each verification question answered blind, without sight of the answer under test. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, COVE_ANSWERS_SLUG, COVE_ANSWERS_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/** Stage 5B. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const COVE_ANSWERS_SLUG = "ctr-5b-cove-blind-answer-1w1479";
|
|
3
|
-
/** What the reply must be, as the prompt itself specifies it. */
|
|
4
2
|
export const COVE_ANSWERS_SCHEMA = {
|
|
5
3
|
type: "object",
|
|
6
4
|
properties: {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { COVE_QUESTIONS_SCHEMA, COVE_QUESTIONS_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 5A. The verification questions the answer's own claims invite. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, COVE_QUESTIONS_SLUG, COVE_QUESTIONS_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/** Stage 5A. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const COVE_QUESTIONS_SLUG = "ctr-5a-cove-questions-d9633d";
|
|
3
|
-
/** What the reply must be, as the prompt itself specifies it. */
|
|
4
2
|
export const COVE_QUESTIONS_SCHEMA = {
|
|
5
3
|
type: "object",
|
|
6
4
|
properties: {
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { DECOMPOSE_SCHEMA, DECOMPOSE_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 2. Splits the question into ordered sub-questions, and answers none. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, DECOMPOSE_SLUG, DECOMPOSE_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,22 +1,4 @@
|
|
|
1
|
-
/** The stored decomposition prompt. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const DECOMPOSE_SLUG = "ctr-2-decompose-b559f2";
|
|
3
|
-
/**
|
|
4
|
-
* What the reply must be, as the prompt itself specifies it.
|
|
5
|
-
*
|
|
6
|
-
* `maxItems` is 7 because the prompt caps it there, and the cap is the point:
|
|
7
|
-
* unbounded splitting — task lists breeding task lists — is the failure this
|
|
8
|
-
* stage exists to avoid.
|
|
9
|
-
*/
|
|
10
|
-
/**
|
|
11
|
-
* What each hint means, sent with the schema.
|
|
12
|
-
*
|
|
13
|
-
* Seven bare identifiers left the choice to be guessed from the name, and two of
|
|
14
|
-
* them are wired to nothing by default — a sub-question hinting `code_exec` or
|
|
15
|
-
* `get_documents` comes back skipped, so a guess between `deeprecall` and
|
|
16
|
-
* `get_documents` silently costs an answer. Saying which is which is the whole
|
|
17
|
-
* of the fix. `tests/tool-hints.test.ts` holds this text to what
|
|
18
|
-
* `DEFAULT_TOOLS` actually wires.
|
|
19
|
-
*/
|
|
20
2
|
export const TOOL_HINT = [
|
|
21
3
|
"Which tool the executor should reach for. Name the cheapest one that can answer, and prefer none over a call that is not needed.",
|
|
22
4
|
"- none: answerable from the question and the brief alone. Nothing is called.",
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { TEST_SCHEMA, TEST_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 4B. The one test whose result tells the live hypotheses apart. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, TEST_SLUG, TEST_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { PLAN_SCHEMA, PLAN_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 3A. Three approaches, the one chosen, and the steps that follow from it. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, PLAN_SLUG, PLAN_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { BRIEF_SCHEMA, BRIEF_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage 1. Interrogates the question and hands back the brief, never an answer. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, BRIEF_SLUG, BRIEF_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,15 +1,6 @@
|
|
|
1
|
-
/** The stored question-brief prompt. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const BRIEF_SLUG = "ctr-1-question-brief-f2w9fd";
|
|
3
2
|
const STRINGS = { type: "array", items: { type: "string" } };
|
|
4
3
|
const NULLABLE_STRING = { type: ["string", "null"] };
|
|
5
|
-
/**
|
|
6
|
-
* What the reply must be, as the prompt itself specifies it.
|
|
7
|
-
*
|
|
8
|
-
* Every key is required, including the two that are usually null: the prompt
|
|
9
|
-
* returns `better_question` and `interpretation` as null rather than omitting
|
|
10
|
-
* them, and a missing key would read downstream as "nothing was ambiguous"
|
|
11
|
-
* instead of "the stage did not say".
|
|
12
|
-
*/
|
|
13
4
|
export const BRIEF_SCHEMA = {
|
|
14
5
|
type: "object",
|
|
15
6
|
properties: {
|
|
@@ -3,19 +3,7 @@ import { addUsage, totalCost, NO_USAGE } from "../../usage.js";
|
|
|
3
3
|
import { invoke } from "./invoke.js";
|
|
4
4
|
import { render } from "./render.js";
|
|
5
5
|
const DEFAULT_MIN_CONFIDENCE = 0.6;
|
|
6
|
-
/**
|
|
7
|
-
* The tiers to run, when the caller named none.
|
|
8
|
-
*
|
|
9
|
-
* One tier, and it is the model the `Llm` was built with. A default list of
|
|
10
|
-
* model ids chosen here would answer on a model the caller never picked, and on
|
|
11
|
-
* a provider that has never heard of those ids it fails outright — at the last
|
|
12
|
-
* stage, after every earlier one has been paid for.
|
|
13
|
-
*
|
|
14
|
-
* So a cascade is something you ask for. Naming two tiers is what buys the
|
|
15
|
-
* second opinion; naming none gets the model already configured, once.
|
|
16
|
-
*/
|
|
17
6
|
const tiersFor = (llm, named) => named?.length ? named : [{ model: llm.model }];
|
|
18
|
-
/** Every tier fell over, so there is no answer to render. */
|
|
19
7
|
export class NoAnswerError extends Error {
|
|
20
8
|
escalated;
|
|
21
9
|
constructor(escalated) {
|
|
@@ -24,15 +12,6 @@ export class NoAnswerError extends Error {
|
|
|
24
12
|
this.name = "NoAnswerError";
|
|
25
13
|
}
|
|
26
14
|
}
|
|
27
|
-
/**
|
|
28
|
-
* Stage 6C, cheapest model first.
|
|
29
|
-
*
|
|
30
|
-
* A tier is accepted when it committed to a claim and its own calibrated
|
|
31
|
-
* confidence clears the bar; an abstention escalates whatever the bar is,
|
|
32
|
-
* because a stage that could not answer has not answered. Nothing is rewritten
|
|
33
|
-
* on the way up — each rung answers from the artifacts again, so an escalation
|
|
34
|
-
* is a second opinion rather than a polish of the first.
|
|
35
|
-
*/
|
|
36
15
|
export async function answer(client, llm, artifacts, opts = {}) {
|
|
37
16
|
const tiers = tiersFor(llm, opts.tiers);
|
|
38
17
|
const floor = opts.minConfidence ?? DEFAULT_MIN_CONFIDENCE;
|
|
@@ -3,7 +3,6 @@ import { attempting } from "../../retry.js";
|
|
|
3
3
|
import { announce, failed } from "../../started.js";
|
|
4
4
|
import { usageOf } from "../../usage.js";
|
|
5
5
|
import { TOULMIN_SCHEMA, TOULMIN_SLUG } from "./prompt.js";
|
|
6
|
-
/** The reply came back, but not as the object the stage is supposed to return. */
|
|
7
6
|
export class ToulminReplyError extends Error {
|
|
8
7
|
raw;
|
|
9
8
|
constructor(raw) {
|
|
@@ -15,15 +14,6 @@ export class ToulminReplyError extends Error {
|
|
|
15
14
|
function asText(artifacts) {
|
|
16
15
|
return typeof artifacts === "string" ? artifacts : JSON.stringify(artifacts, null, 2);
|
|
17
16
|
}
|
|
18
|
-
/**
|
|
19
|
-
* Stage 6C. Compiles the artifacts into the final answer and its qualifier.
|
|
20
|
-
*
|
|
21
|
-
* The artifacts are the request, so they go through `{question}` like every
|
|
22
|
-
* other stage's. They used to arrive as a history turn instead, which worked
|
|
23
|
-
* only while the prompt's question box was empty: `renderMessages` puts history
|
|
24
|
-
* between the system turn and the question, so anything saved in that box would
|
|
25
|
-
* have been the last thing the model read.
|
|
26
|
-
*/
|
|
27
17
|
export async function invoke(client, llm, artifacts, opts = {}) {
|
|
28
18
|
announce(opts.emit, TOULMIN_SLUG, opts.model);
|
|
29
19
|
return attempting(async () => {
|
|
@@ -1,14 +1,5 @@
|
|
|
1
|
-
/** The stored answer-assembly prompt. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const TOULMIN_SLUG = "ctr-6c-toulmin-answer-d28bb8";
|
|
3
2
|
const UNIT = { type: "number", minimum: 0, maximum: 1 };
|
|
4
|
-
/**
|
|
5
|
-
* What the reply must be, as the prompt itself specifies it.
|
|
6
|
-
*
|
|
7
|
-
* `rebuttals` and `would_change_my_mind` are required rather than optional on
|
|
8
|
-
* purpose: an empty list is a claim the stage may make, but a missing key would
|
|
9
|
-
* let a surviving objection be dropped silently, which is what this stage
|
|
10
|
-
* exists to prevent.
|
|
11
|
-
*/
|
|
12
3
|
export const TOULMIN_SCHEMA = {
|
|
13
4
|
type: "object",
|
|
14
5
|
properties: {
|
|
@@ -9,21 +9,9 @@ function confidencePhrase(confidence) {
|
|
|
9
9
|
return "low";
|
|
10
10
|
return "very low";
|
|
11
11
|
}
|
|
12
|
-
/**
|
|
13
|
-
* Whether the model actually said this.
|
|
14
|
-
*
|
|
15
|
-
* Takes `string | undefined` although `ToulminAnswer` types every field as a
|
|
16
|
-
* string, because nothing enforces that: the schema's `required` list is sent
|
|
17
|
-
* to the provider and never checked on the way back, so a field the model
|
|
18
|
-
* omitted arrives as `undefined` and reads here as unsaid.
|
|
19
|
-
*/
|
|
20
12
|
function said(value) {
|
|
21
13
|
return !!value && value.trim().length > 0;
|
|
22
14
|
}
|
|
23
|
-
/**
|
|
24
|
-
* The model writes these as sentences, and the label already supplies the
|
|
25
|
-
* conjunction — without this they read "holds when when the trace shows".
|
|
26
|
-
*/
|
|
27
15
|
function clause(text) {
|
|
28
16
|
return text.trim().replace(/^(when|if)\s+/i, "");
|
|
29
17
|
}
|
|
@@ -54,18 +42,6 @@ function groundLines(answer) {
|
|
|
54
42
|
const warrant = said(answer.warrant) ? `Warrant: ${answer.warrant}` : "";
|
|
55
43
|
return [grounds, warrant].filter(said).join("\n\n");
|
|
56
44
|
}
|
|
57
|
-
/**
|
|
58
|
-
* The answer as a person reads it.
|
|
59
|
-
*
|
|
60
|
-
* Pure, and deliberately not a seventh prompt. Every part of a `ToulminAnswer`
|
|
61
|
-
* is something the stage was forbidden to soften — a surviving objection, a
|
|
62
|
-
* calibrated number, an abstention — and handing the object to a model to write
|
|
63
|
-
* up returns each of those chances on the last call before anyone sees it.
|
|
64
|
-
*
|
|
65
|
-
* Objections are never dropped. Where something has to give, grounds go first,
|
|
66
|
-
* which is why they are off by default: they are the audit trail, and the
|
|
67
|
-
* rollout already has them.
|
|
68
|
-
*/
|
|
69
45
|
export function render(answer, opts = {}) {
|
|
70
46
|
const rebuttals = rebuttalLines(answer.rebuttals);
|
|
71
47
|
const changes = said(answer.would_change_my_mind)
|
package/dist/noesis/question.js
CHANGED
|
@@ -1,28 +1,9 @@
|
|
|
1
|
-
/** The variable every stored prompt takes its request through. */
|
|
2
1
|
export const QUESTION_FIELD = "question";
|
|
3
2
|
const warned = new Set();
|
|
4
3
|
function carries(messages, question) {
|
|
5
4
|
const last = messages[messages.length - 1];
|
|
6
5
|
return last?.role === "user" && last.content.includes(question);
|
|
7
6
|
}
|
|
8
|
-
/**
|
|
9
|
-
* The messages to send, with the request in the turn the prompt renders.
|
|
10
|
-
*
|
|
11
|
-
* A prompt whose question box is `{question}` needs nothing from this: the
|
|
12
|
-
* value passed alongside fills it and the rendered turn already carries the
|
|
13
|
-
* request. Asked of the messages rather than of `prompt.fields`, because
|
|
14
|
-
* `applyVariables` substitutes any `{name}` it finds against the values it is
|
|
15
|
-
* given and never reads the field list — so a placeholder typed into the box
|
|
16
|
-
* works with no field declared, and would not show up in `fields` at all.
|
|
17
|
-
*
|
|
18
|
-
* Where the box holds something else — a question typed while testing, or
|
|
19
|
-
* nothing at all — the trailing turn is overwritten with the request and the
|
|
20
|
-
* stored text is not sent. That is what the placeholder would have done, and it
|
|
21
|
-
* keeps the shape the prompt expects: one system turn, one user turn. It warns
|
|
22
|
-
* once per prompt, because the alternative failure is silent: an unmigrated
|
|
23
|
-
* prompt would answer the question saved on it and return a well-formed result
|
|
24
|
-
* about the wrong thing.
|
|
25
|
-
*/
|
|
26
7
|
export function askThrough(messages, question, prompt) {
|
|
27
8
|
if (carries(messages, question))
|
|
28
9
|
return messages;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { runStage } from "../../stage.js";
|
|
2
2
|
import { REFLECTION_SCHEMA, REFLECTION_SLUG } from "./prompt.js";
|
|
3
|
-
/** Stage R. The earliest wrong decision behind a failure, not the place it surfaced. */
|
|
4
3
|
export function invoke(client, llm, input, opts = {}) {
|
|
5
4
|
return runStage(client, llm, REFLECTION_SLUG, REFLECTION_SCHEMA, input, opts);
|
|
6
5
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
/** Stage R. Pulled by slug, so an edit needs no release here. */
|
|
2
1
|
export const REFLECTION_SLUG = "ctr-r-root-cause-reflection-7x0x90";
|
|
3
|
-
/** What the reply must be, as the prompt itself specifies it. */
|
|
4
2
|
export const REFLECTION_SCHEMA = {
|
|
5
3
|
type: "object",
|
|
6
4
|
properties: {
|
package/dist/noesis/retry.js
CHANGED
|
@@ -1,33 +1,7 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* How many times a stage call may be made, counting the first.
|
|
3
|
-
*
|
|
4
|
-
* Two, not more. A flake is usually gone on the second try, and a call that
|
|
5
|
-
* failed has often still been billed — so the budget is deliberately small and
|
|
6
|
-
* the classifier below deliberately narrow.
|
|
7
|
-
*/
|
|
8
1
|
const DEFAULT_ATTEMPTS = 2;
|
|
9
|
-
/** Milliseconds before the next attempt, multiplied by the attempt number. */
|
|
10
2
|
const BACKOFF_MS = 300;
|
|
11
3
|
const pause = (ms) => new Promise((done) => setTimeout(done, ms));
|
|
12
|
-
/**
|
|
13
|
-
* Signals that a second attempt could plausibly land.
|
|
14
|
-
*
|
|
15
|
-
* Matched on the message because a stage is handed whatever the caller's client
|
|
16
|
-
* threw, and nothing here can assume a status code survived. Anything else is
|
|
17
|
-
* not retried: a 400 naming a model that does not exist and a 402 for credits
|
|
18
|
-
* that are not there fail the same way however many times they are sent, and
|
|
19
|
-
* spending three round trips to learn that is worse than failing once.
|
|
20
|
-
*/
|
|
21
4
|
const TRANSIENT = /\b(429|500|502|503|504|522|524)\b|rate.?limit|overloaded|timed? ?out|timeout|ECONNRESET|ETIMEDOUT|EAI_AGAIN|socket hang up|fetch failed|network error/i;
|
|
22
|
-
/**
|
|
23
|
-
* Whether the same call is worth making again.
|
|
24
|
-
*
|
|
25
|
-
* Two families qualify. A reply error means a schema-forced call came back with
|
|
26
|
-
* nothing to parse — the model declined to emit the tool call, which is a flake
|
|
27
|
-
* rather than a fault in the prompt, and the bounded budget covers the case
|
|
28
|
-
* where it is not. A transient transport error is the provider asking to be
|
|
29
|
-
* asked later.
|
|
30
|
-
*/
|
|
31
5
|
export function retryable(err) {
|
|
32
6
|
if (!(err instanceof Error))
|
|
33
7
|
return false;
|
|
@@ -35,14 +9,6 @@ export function retryable(err) {
|
|
|
35
9
|
return true;
|
|
36
10
|
return TRANSIENT.test(err.message);
|
|
37
11
|
}
|
|
38
|
-
/**
|
|
39
|
-
* Runs a stage call, once more if the first failure looks worth repeating.
|
|
40
|
-
*
|
|
41
|
-
* This is deliberately not the gate retry budget. That one is spent when a gate
|
|
42
|
-
* refuses and reflection sends work backwards — the loop thinking again. This
|
|
43
|
-
* one covers a call that never produced anything to think about, and mixing the
|
|
44
|
-
* two would let a flaky provider eat the allowance reserved for reasoning.
|
|
45
|
-
*/
|
|
46
12
|
export async function attempting(run, opts = {}) {
|
|
47
13
|
const total = Math.max(1, opts.attempts ?? DEFAULT_ATTEMPTS);
|
|
48
14
|
for (let attempt = 1;; attempt++) {
|
package/dist/noesis/route.js
CHANGED
|
@@ -8,33 +8,6 @@ const NEARLY = 0.6;
|
|
|
8
8
|
const HARD = new Set(["expert", "research"]);
|
|
9
9
|
const EASY = new Set(["trivial", "easy"]);
|
|
10
10
|
const SERIOUS = new Set(["high", "critical"]);
|
|
11
|
-
/**
|
|
12
|
-
* The estimates, turned into a route.
|
|
13
|
-
*
|
|
14
|
-
* Deliberately code and deliberately dull: the router prompt is told not to
|
|
15
|
-
* choose its own budget, and a threshold written here can be moved after an
|
|
16
|
-
* eval run without touching a prompt or re-running anything.
|
|
17
|
-
*
|
|
18
|
-
* A lookup is answered as a lookup, even when the stakes are high. `redTeam` is
|
|
19
|
-
* a flag rather than a route, so it still fires — the care is kept and the
|
|
20
|
-
* planning stages that have nothing to plan are not paid for. `critical` is the
|
|
21
|
-
* exception, and keeps the whole loop.
|
|
22
|
-
*
|
|
23
|
-
* Difficulty alone does not buy the audit set. The premortem and the standards
|
|
24
|
-
* review bite on risk and vagueness, not on how hard the reasoning is, so a
|
|
25
|
-
* hard question that is low-stakes, well-posed and already understood takes
|
|
26
|
-
* `standard`. Hard and *unsure* still takes `full`.
|
|
27
|
-
*
|
|
28
|
-
* Vagueness and doubt compound. Either one alone is survivable; a question that
|
|
29
|
-
* is both under-specified and one the model cannot place is what 3B and 3C
|
|
30
|
-
* exist for, and it takes `full` below the ambiguity bar that would catch it on
|
|
31
|
-
* its own.
|
|
32
|
-
*
|
|
33
|
-
* `retrieve` splits the two reasons a clear question fails `direct`: high tool
|
|
34
|
-
* need, and a confidence that only just misses. Both mean the model should not
|
|
35
|
-
* answer from memory, and neither makes it a hard question — looking it up is
|
|
36
|
-
* cheaper than planning it, and likelier to be right.
|
|
37
|
-
*/
|
|
38
11
|
export function routeFor(estimates) {
|
|
39
12
|
const redTeam = SERIOUS.has(estimates.stakes);
|
|
40
13
|
const clear = EASY.has(estimates.difficulty) && estimates.ambiguity < CLEAR;
|