@gr8ful/spf 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -17
- package/assets/defaults/spf.config.yaml +68 -0
- package/assets/prompts/refiner/system.md +32 -6
- package/assets/prompts/refiner/user.md +37 -5
- package/assets/skill/SKILL.md +1 -0
- package/assets/skill/references/config.md +153 -4
- package/assets/templates/ts-flue-ollama.spf.config.yaml +25 -0
- package/assets/templates/ts.spf.config.yaml +6 -0
- package/dist/chains/index.d.ts +11 -0
- package/dist/chains/index.js +38 -3
- package/dist/chains/repo_chains.js +1 -0
- package/dist/chains/simple_sdlc.js +1 -1
- package/dist/chains/steps.d.ts +27 -9
- package/dist/chains/steps.js +82 -13
- package/dist/cli/commands/doctor.js +108 -1
- package/dist/cli/commands/estimate.d.ts +82 -0
- package/dist/cli/commands/estimate.js +317 -0
- package/dist/cli/commands/fanout.d.ts +40 -0
- package/dist/cli/commands/fanout.js +401 -0
- package/dist/cli/commands/init.js +19 -0
- package/dist/cli/commands/trace.d.ts +18 -0
- package/dist/cli/commands/trace.js +22 -3
- package/dist/cli/commands/watch.js +40 -26
- package/dist/cli/index.js +12 -0
- package/dist/core/agents.d.ts +56 -0
- package/dist/core/agents.js +152 -1
- package/dist/core/data_types.d.ts +232 -0
- package/dist/core/data_types.js +135 -0
- package/dist/core/fanout.d.ts +229 -0
- package/dist/core/fanout.js +313 -0
- package/dist/core/gates.d.ts +8 -0
- package/dist/core/gates.js +24 -1
- package/dist/core/git_helper.d.ts +98 -0
- package/dist/core/git_helper.js +127 -0
- package/dist/core/issues/github_provider.d.ts +7 -1
- package/dist/core/issues/github_provider.js +33 -2
- package/dist/core/issues/jira_provider.d.ts +5 -1
- package/dist/core/issues/jira_provider.js +26 -3
- package/dist/core/issues/provider.d.ts +53 -2
- package/dist/core/notify/channel.d.ts +1 -1
- package/dist/core/notify/notifier.d.ts +16 -0
- package/dist/core/notify/notifier.js +36 -0
- package/dist/core/otel.d.ts +64 -9
- package/dist/core/otel.js +78 -14
- package/dist/core/runner.d.ts +8 -0
- package/dist/core/runner.js +7 -0
- package/dist/core/session.d.ts +25 -0
- package/dist/core/session.js +97 -28
- package/dist/core/tiering.d.ts +145 -0
- package/dist/core/tiering.js +235 -0
- package/dist/core/tracer.d.ts +10 -0
- package/dist/core/tracer.js +12 -0
- package/dist/core/watch.d.ts +89 -9
- package/dist/core/watch.js +193 -19
- package/dist/ui/server/db.d.ts +39 -0
- package/dist/ui/server/db.js +61 -0
- package/package.json +2 -1
- package/dist/test/agent_cc.test.d.ts +0 -1
- package/dist/test/agent_cc.test.js +0 -95
- package/dist/test/agent_flue.test.d.ts +0 -1
- package/dist/test/agent_flue.test.js +0 -83
- package/dist/test/chains.test.d.ts +0 -12
- package/dist/test/chains.test.js +0 -92
- package/dist/test/data_types.test.d.ts +0 -10
- package/dist/test/data_types.test.js +0 -220
- package/dist/test/env_file.test.d.ts +0 -1
- package/dist/test/env_file.test.js +0 -74
- package/dist/test/fake_asker.d.ts +0 -23
- package/dist/test/fake_asker.js +0 -30
- package/dist/test/git_helper.test.d.ts +0 -1
- package/dist/test/git_helper.test.js +0 -59
- package/dist/test/hermetic_git.d.ts +0 -1
- package/dist/test/hermetic_git.js +0 -22
- package/dist/test/init_command.test.d.ts +0 -14
- package/dist/test/init_command.test.js +0 -136
- package/dist/test/interview.test.d.ts +0 -15
- package/dist/test/interview.test.js +0 -425
- package/dist/test/notify.test.d.ts +0 -1
- package/dist/test/notify.test.js +0 -174
- package/dist/test/ollama_provider.test.d.ts +0 -1
- package/dist/test/ollama_provider.test.js +0 -103
- package/dist/test/otel.test.d.ts +0 -26
- package/dist/test/otel.test.js +0 -512
- package/dist/test/paths.test.d.ts +0 -1
- package/dist/test/paths.test.js +0 -68
- package/dist/test/refine.test.d.ts +0 -1
- package/dist/test/refine.test.js +0 -189
- package/dist/test/repo_chains.test.d.ts +0 -21
- package/dist/test/repo_chains.test.js +0 -416
- package/dist/test/signoff.test.d.ts +0 -1
- package/dist/test/signoff.test.js +0 -329
- package/dist/test/ui_server.test.d.ts +0 -7
- package/dist/test/ui_server.test.js +0 -120
- package/dist/test/watch.test.d.ts +0 -1
- package/dist/test/watch.test.js +0 -687
package/dist/chains/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import * as steps from "./steps.js";
|
|
2
2
|
import * as simpleSdlc from "./simple_sdlc.js";
|
|
3
|
+
import * as otel from "../core/otel.js";
|
|
4
|
+
import * as session from "../core/session.js";
|
|
3
5
|
/**
|
|
4
6
|
* Build a step-based ChainDefinition, deriving phases/requiredAgents/
|
|
5
7
|
* requiredSuites from its steps.
|
|
@@ -66,6 +68,15 @@ export const CHAINS = [
|
|
|
66
68
|
stepChain("document", "write up the work that was just done, from the diff", [steps.request(), steps.changes(), steps.document()]),
|
|
67
69
|
stepChain("refine", "decompose a product spec into a feature/story-or-bug tree of tracker issues — spf watch's spec-ready lane", [
|
|
68
70
|
steps.request(),
|
|
71
|
+
// Grounds the decomposition in code that actually exists, rather than
|
|
72
|
+
// leaving "explore the codebase" as prose the refiner may or may not
|
|
73
|
+
// follow (see assets/prompts/refiner/system.md). Its ScoutOutput flows
|
|
74
|
+
// into refine() as `previous` — agentStep() threads state.previous into
|
|
75
|
+
// every agent phase, and refiner/user.md already renders it as
|
|
76
|
+
// {{previous_envelope}}. This makes `scout` a required agent for this
|
|
77
|
+
// chain: a roster that pruned it fails agents.validate() by name at
|
|
78
|
+
// `spf watch` startup, same as any other missing required agent.
|
|
79
|
+
steps.scout({ description: "Map the subsystems this spec touches — change nothing" }),
|
|
69
80
|
steps.refine(),
|
|
70
81
|
steps.publishIssues(),
|
|
71
82
|
]),
|
|
@@ -147,9 +158,33 @@ export function resolveRequiredSuites(chain, options) {
|
|
|
147
158
|
* CLI dispatch sites (`spf <chain>` and `spf watch`) go through this, never
|
|
148
159
|
* `chain.run(...)` directly — the whole reason to route through here is that
|
|
149
160
|
* a `steps`-only chain has no `run` to call.
|
|
161
|
+
*
|
|
162
|
+
* Also the one call site both dispatch paths share on the way OUT — success
|
|
163
|
+
* or thrown error alike, `finally` runs either way — which is what makes it
|
|
164
|
+
* the right seam for `otel.releaseOtelExporter()` (see otel.ts's RUN-SCOPED
|
|
165
|
+
* CLEANUP note / #26): a `spf watch` daemon calls this once per issue, in
|
|
166
|
+
* the same process, for as long as it runs, so a run's exporter has to be
|
|
167
|
+
* dropped here rather than living until the whole daemon exits. `session.
|
|
168
|
+
* finalize()` is the same idea for the run itself — it stops the process-
|
|
169
|
+
* wide signal handler from reaching this (now finished) run, and closes its
|
|
170
|
+
* Tracer's sqlite handle, so a long `spf watch` daemon does not hold every
|
|
171
|
+
* issue it has ever processed reachable for the rest of its life.
|
|
150
172
|
*/
|
|
151
173
|
export async function runChain(chain, ctx, options = {}) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
174
|
+
try {
|
|
175
|
+
if (chain.run)
|
|
176
|
+
return await chain.run(ctx, options);
|
|
177
|
+
return await steps.runSteps(ctx, resolveRequiredAgents(chain, options), resolveRequiredSuites(chain, options), chain.steps, options);
|
|
178
|
+
}
|
|
179
|
+
finally {
|
|
180
|
+
// Both are no-ops for a one-shot `spf <chain>` invocation with no
|
|
181
|
+
// explicit `--adw-id` (ctx.adw_id is null; the id session.ensure()
|
|
182
|
+
// actually minted never makes it back up here) — that process exits
|
|
183
|
+
// right after this call returns, and the existing end-of-process
|
|
184
|
+
// `flushAll()` in `cli/index.ts`'s `finally` still drains the exporter
|
|
185
|
+
// exactly as it always did (the signal listener needs no such fallback:
|
|
186
|
+
// the process is gone).
|
|
187
|
+
await otel.releaseOtelExporter(ctx.adw_id);
|
|
188
|
+
session.finalize(ctx.adw_id);
|
|
189
|
+
}
|
|
155
190
|
}
|
|
@@ -148,7 +148,7 @@ export function trailerFor(outcome, identity) {
|
|
|
148
148
|
}
|
|
149
149
|
export async function main(ctx) {
|
|
150
150
|
const { prompt } = ctx;
|
|
151
|
-
const run = startRun(ctx, REQUIRED_AGENTS, REQUIRED_SUITES);
|
|
151
|
+
const run = await startRun(ctx, REQUIRED_AGENTS, REQUIRED_SUITES);
|
|
152
152
|
const baseline = run.git.rev("HEAD"); // pinned before this run commits anything
|
|
153
153
|
await run.phase(makePhaseParams({ name: "request", kind: "engineer", owner: run.engineer, description: "Capture the incoming ask" }), async (ph) => {
|
|
154
154
|
ph.log({ input: prompt, baseline: run.git.shortSha(baseline) });
|
package/dist/chains/steps.d.ts
CHANGED
|
@@ -82,8 +82,14 @@ export interface Step {
|
|
|
82
82
|
/** Display fragment for derivePhases() — e.g. "planner", "git(commit)". */
|
|
83
83
|
label?: string;
|
|
84
84
|
}
|
|
85
|
-
/**
|
|
86
|
-
|
|
85
|
+
/**
|
|
86
|
+
* The identical loadConfig -> validate -> session.ensure prologue every
|
|
87
|
+
* chain repeated. `async` since the tiering availability probe below is an
|
|
88
|
+
* awaited `fetch` — firing it unawaited would let `agents.execute` read
|
|
89
|
+
* `run.tiering` before the probe resolved. Both call sites are already
|
|
90
|
+
* inside `async` functions, so this stays a one-token change for each.
|
|
91
|
+
*/
|
|
92
|
+
export declare function startRun(ctx: ChainContext, requiredAgents: string[], requiredSuites: string[]): Promise<Run>;
|
|
87
93
|
/**
|
|
88
94
|
* Commit an envelope in its own author's words — the message-fallback four
|
|
89
95
|
* chains repeated.
|
|
@@ -317,13 +323,25 @@ export declare function refine(opts?: {
|
|
|
317
323
|
* rendering) is `core/refine.ts`'s job; this step is sequencing only, per
|
|
318
324
|
* SKILL.md's "chains stay thin" rule. Requires a preceding refine() step.
|
|
319
325
|
*
|
|
320
|
-
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
* `
|
|
326
|
-
* to
|
|
326
|
+
* When the refiner raised material ambiguity instead of a tree
|
|
327
|
+
* (`envelope.questions` non-empty — `gates.refinementWellFormed` already
|
|
328
|
+
* guarantees `issues` is empty whenever that's true), this publishes
|
|
329
|
+
* nothing: it writes those questions to
|
|
330
|
+
* `<context_handoff_dir>/refine_questions.json` instead of
|
|
331
|
+
* `refine_publish.json` and returns. That is a successful phase outcome, not
|
|
332
|
+
* a failure — escalating to a human is a legitimate way for this chain to
|
|
333
|
+
* end, same as publishing a tree is.
|
|
334
|
+
*
|
|
335
|
+
* Writes what it created (or asked) to `<context_handoff_dir>/`, one of
|
|
336
|
+
* `refine_publish.json` or `refine_questions.json` — the side channel
|
|
337
|
+
* `cli/commands/watch.ts`'s `runRefine` reads after the chain returns, since
|
|
338
|
+
* a chain's own return value is just an exit code. `spf watch`'s own
|
|
339
|
+
* marker/comment/transition bookkeeping for the spec issue (including
|
|
340
|
+
* posting the questions and moving it to `needs-feedback`) lives entirely in
|
|
341
|
+
* `core/watch.ts`'s `runSpec`/`escalateSpec`, never here — a bare `spf
|
|
342
|
+
* refine` run (no daemon, no spec issue in play) still needs this step to
|
|
343
|
+
* work standalone, and just leaves the questions on disk for a human to
|
|
344
|
+
* read.
|
|
327
345
|
*/
|
|
328
346
|
export declare function publishIssues(opts?: {
|
|
329
347
|
description?: string;
|
package/dist/chains/steps.js
CHANGED
|
@@ -48,6 +48,7 @@ import * as quality from "../core/quality.js";
|
|
|
48
48
|
import * as agentsCfg from "../core/agents.js";
|
|
49
49
|
import * as session from "../core/session.js";
|
|
50
50
|
import * as refineLib from "../core/refine.js";
|
|
51
|
+
import * as tiering from "../core/tiering.js";
|
|
51
52
|
import { DOCUMENT_NOTES } from "../core/prompts.js";
|
|
52
53
|
import { BuildOutput, DocumentOutput, GenericOutput, PlanOutput, RefineOutput, ReviewOutput, ScoutOutput, makeAgentCall, makeChangeCapture, makeEventRecord, makePhaseParams, } from "../core/data_types.js";
|
|
53
54
|
import { Run } from "../core/runner.js";
|
|
@@ -62,8 +63,14 @@ function makeStep(fn, meta = {}) {
|
|
|
62
63
|
return step;
|
|
63
64
|
}
|
|
64
65
|
// ── layer 1: the shared prologue ────────────────────────────────────────
|
|
65
|
-
/**
|
|
66
|
-
|
|
66
|
+
/**
|
|
67
|
+
* The identical loadConfig -> validate -> session.ensure prologue every
|
|
68
|
+
* chain repeated. `async` since the tiering availability probe below is an
|
|
69
|
+
* awaited `fetch` — firing it unawaited would let `agents.execute` read
|
|
70
|
+
* `run.tiering` before the probe resolved. Both call sites are already
|
|
71
|
+
* inside `async` functions, so this stays a one-token change for each.
|
|
72
|
+
*/
|
|
73
|
+
export async function startRun(ctx, requiredAgents, requiredSuites) {
|
|
67
74
|
const cfg = agentsCfg.loadConfig(ctx.config_paths);
|
|
68
75
|
agentsCfg.validate(cfg, requiredAgents, requiredSuites, ctx.cwd);
|
|
69
76
|
const run = session.ensure(cfg, ctx.adw_id, ctx.cwd, ctx.chain_name);
|
|
@@ -83,6 +90,41 @@ export function startRun(ctx, requiredAgents, requiredSuites) {
|
|
|
83
90
|
if (ctx.chain_source) {
|
|
84
91
|
run.tracer.event(makeEventRecord({ adw_id: run.adw_id, type: "log", name: "chain_source", payload: { source: ctx.chain_source } }));
|
|
85
92
|
}
|
|
93
|
+
// Tiering resolution (SPF #14) — one more run-scoped fact, computed once,
|
|
94
|
+
// before any phase opens, beside chain_source above. `risk`/`signals` are
|
|
95
|
+
// always computed (a pure function of chain name + prompt); `routing`/
|
|
96
|
+
// `notes` are only ever non-empty when `cfg.tiering.enabled` — see
|
|
97
|
+
// `resolveTiering`'s own no-op guarantee.
|
|
98
|
+
const servedOllamaTags = await tiering.probeServedOllamaTags(cfg);
|
|
99
|
+
run.tiering = tiering.resolveTiering({
|
|
100
|
+
cfg,
|
|
101
|
+
chainName: ctx.chain_name,
|
|
102
|
+
prompt: ctx.prompt,
|
|
103
|
+
servedOllamaTags,
|
|
104
|
+
required: requiredAgents,
|
|
105
|
+
});
|
|
106
|
+
run.tracer.event(makeEventRecord({
|
|
107
|
+
adw_id: run.adw_id,
|
|
108
|
+
type: "log",
|
|
109
|
+
name: "tiering",
|
|
110
|
+
payload: {
|
|
111
|
+
risk: run.tiering.risk,
|
|
112
|
+
signals: run.tiering.signals,
|
|
113
|
+
routing: run.tiering.routing,
|
|
114
|
+
notes: run.tiering.notes,
|
|
115
|
+
},
|
|
116
|
+
}));
|
|
117
|
+
// One console line per RETIERED agent (changedModels — not per routing
|
|
118
|
+
// entry: a role whose tier resolves to the model it was already
|
|
119
|
+
// configured with is not news). Routed through the existing
|
|
120
|
+
// `Console.note()` — no new Console method. This is what reconciles
|
|
121
|
+
// `run.cfg.agents[].model` (which keeps the configured value) with
|
|
122
|
+
// `agent_sessions.model` (which records the effective one) for a human
|
|
123
|
+
// reading the console when the two disagree.
|
|
124
|
+
for (const [agentName, effective] of Object.entries(tiering.changedModels(run.tiering))) {
|
|
125
|
+
const route = run.tiering.routing[agentName];
|
|
126
|
+
run.console.note(`[spf] tiering ${agentName} ${route.tier} (${route.configured} -> ${effective}) risk=${run.tiering.risk}`);
|
|
127
|
+
}
|
|
86
128
|
return run;
|
|
87
129
|
}
|
|
88
130
|
/** A trailer line: `Key: value...`, one per line, no blank lines inside the block. */
|
|
@@ -622,10 +664,19 @@ export function refine(opts = {}) {
|
|
|
622
664
|
output_type: RefineOutput,
|
|
623
665
|
description: opts.description ?? "Decompose the spec into a feature/story tree of vertical slices",
|
|
624
666
|
// refinementWellFormed is NOT in GATE_ALLOWLIST — it is meaningless on
|
|
625
|
-
// any other envelope type (it reads `issues`), so there is
|
|
626
|
-
// gain by letting a definition name it, and it stays
|
|
667
|
+
// any other envelope type (it reads `issues`/`questions`), so there is
|
|
668
|
+
// nothing to gain by letting a definition name it, and it stays
|
|
669
|
+
// non-removable here.
|
|
670
|
+
//
|
|
671
|
+
// retries: 1, not 0 — this gate's own doc comment says a violation
|
|
672
|
+
// "re-prompts the SAME refiner session before publishIssues() ever
|
|
673
|
+
// runs." With 0 retries that was never true: agents.ts throws
|
|
674
|
+
// GateFailure on the first violation and the whole spec goes straight to
|
|
675
|
+
// spf:blocked with no correction round-trip. The mutual-exclusion rule
|
|
676
|
+
// between `issues` and `questions` (see refinementWellFormed) makes that
|
|
677
|
+
// mismatch more likely to bite in practice, not less.
|
|
627
678
|
gates: withExtraGates([gates.refinementWellFormed], opts.extraGates),
|
|
628
|
-
retries: opts.retries ??
|
|
679
|
+
retries: opts.retries ?? 1,
|
|
629
680
|
});
|
|
630
681
|
}
|
|
631
682
|
/**
|
|
@@ -635,13 +686,25 @@ export function refine(opts = {}) {
|
|
|
635
686
|
* rendering) is `core/refine.ts`'s job; this step is sequencing only, per
|
|
636
687
|
* SKILL.md's "chains stay thin" rule. Requires a preceding refine() step.
|
|
637
688
|
*
|
|
638
|
-
*
|
|
639
|
-
*
|
|
640
|
-
*
|
|
641
|
-
*
|
|
642
|
-
*
|
|
643
|
-
* `
|
|
644
|
-
* to
|
|
689
|
+
* When the refiner raised material ambiguity instead of a tree
|
|
690
|
+
* (`envelope.questions` non-empty — `gates.refinementWellFormed` already
|
|
691
|
+
* guarantees `issues` is empty whenever that's true), this publishes
|
|
692
|
+
* nothing: it writes those questions to
|
|
693
|
+
* `<context_handoff_dir>/refine_questions.json` instead of
|
|
694
|
+
* `refine_publish.json` and returns. That is a successful phase outcome, not
|
|
695
|
+
* a failure — escalating to a human is a legitimate way for this chain to
|
|
696
|
+
* end, same as publishing a tree is.
|
|
697
|
+
*
|
|
698
|
+
* Writes what it created (or asked) to `<context_handoff_dir>/`, one of
|
|
699
|
+
* `refine_publish.json` or `refine_questions.json` — the side channel
|
|
700
|
+
* `cli/commands/watch.ts`'s `runRefine` reads after the chain returns, since
|
|
701
|
+
* a chain's own return value is just an exit code. `spf watch`'s own
|
|
702
|
+
* marker/comment/transition bookkeeping for the spec issue (including
|
|
703
|
+
* posting the questions and moving it to `needs-feedback`) lives entirely in
|
|
704
|
+
* `core/watch.ts`'s `runSpec`/`escalateSpec`, never here — a bare `spf
|
|
705
|
+
* refine` run (no daemon, no spec issue in play) still needs this step to
|
|
706
|
+
* work standalone, and just leaves the questions on disk for a human to
|
|
707
|
+
* read.
|
|
645
708
|
*/
|
|
646
709
|
export function publishIssues(opts = {}) {
|
|
647
710
|
preflightDescription("publish", opts.description);
|
|
@@ -650,12 +713,18 @@ export function publishIssues(opts = {}) {
|
|
|
650
713
|
if (!envelope || !Array.isArray(envelope.issues)) {
|
|
651
714
|
throw new Error("publishIssues() requires a preceding refine() step in the chain's step list");
|
|
652
715
|
}
|
|
716
|
+
const questions = envelope.questions ?? [];
|
|
653
717
|
await run.phase(makePhaseParams({
|
|
654
718
|
name: "publish",
|
|
655
719
|
kind: "code",
|
|
656
720
|
owner: "tracker",
|
|
657
721
|
description: opts.description ?? "Create the feature/story tree on the tracker, in dependency order, and link each to its parent",
|
|
658
722
|
}), async (ph) => {
|
|
723
|
+
if (questions.length > 0) {
|
|
724
|
+
writeFileSync(path.join(run.context_handoff_dir, "refine_questions.json"), JSON.stringify(questions, null, 2));
|
|
725
|
+
ph.log({ escalated: questions.length });
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
659
728
|
const tracker = refineLib.resolveAuthoringProvider(run.cfg);
|
|
660
729
|
const created = await refineLib.publish(tracker, envelope.issues, {
|
|
661
730
|
labelPrefix: run.cfg.watch.label_prefix,
|
|
@@ -709,7 +778,7 @@ export function derivePhases(steps) {
|
|
|
709
778
|
// ── the driver ────────────────────────────────────────────────────────────
|
|
710
779
|
/** Run a chain's step list start to finish: prologue, every step in order, then run.finish(). */
|
|
711
780
|
export async function runSteps(ctx, requiredAgents, requiredSuites, steps, options = {}) {
|
|
712
|
-
const run = startRun(ctx, requiredAgents, requiredSuites);
|
|
781
|
+
const run = await startRun(ctx, requiredAgents, requiredSuites);
|
|
713
782
|
const state = makeState(ctx.prompt, options, ctx.issue_id ?? null);
|
|
714
783
|
for (const step of steps) {
|
|
715
784
|
await step(run, state);
|
|
@@ -19,6 +19,7 @@ import { isKnownToolName as isKnownFlueToolName, resolveModel } from "../../core
|
|
|
19
19
|
import { ollamaBaseUrl } from "../../core/ollama_provider.js";
|
|
20
20
|
import { binaryOnPath, parseCli } from "../../core/utils.js";
|
|
21
21
|
import { PROVIDER_ENV_KEYS } from "../../core/providers.js";
|
|
22
|
+
import { probeServedOllamaTags, resolveTiering } from "../../core/tiering.js";
|
|
22
23
|
import { isRepoAt } from "../../core/git_helper.js";
|
|
23
24
|
import { allChains, findChain, repoChainProblems, resolveRequiredAgents, resolveRequiredSuites } from "../../chains/index.js";
|
|
24
25
|
/**
|
|
@@ -265,7 +266,13 @@ export async function doctorCommand(argv) {
|
|
|
265
266
|
// handled in the per-agent loop below) but DO have a server that might
|
|
266
267
|
// simply not be running — worth a reachability probe the same way
|
|
267
268
|
// ANTHROPIC_BASE_URL gets one above.
|
|
268
|
-
|
|
269
|
+
// Tiering (SPF #14): an ollama/* ladder rung reaches for the same server
|
|
270
|
+
// whether or not any cfg.agents[] entry itself names one — gated on
|
|
271
|
+
// tiering.enabled so a disabled (or never-opted-into) ladder can't turn on
|
|
272
|
+
// a probe nobody asked for. Without this, an all-hosted roster paired with
|
|
273
|
+
// an ollama ladder got NO reachability probe at all (integration point 18a).
|
|
274
|
+
const usesOllamaFlue = cfg.agents.some((a) => a.coding_agent !== "claude_code" && a.model.startsWith("ollama/")) ||
|
|
275
|
+
(cfg.tiering.enabled && cfg.tiering.tiers.some((t) => t.coding_agent !== "claude_code" && t.model.startsWith("ollama/")));
|
|
269
276
|
if (usesOllamaFlue && !flags["no-probe"]) {
|
|
270
277
|
// `ollamaBaseUrl()` (ollama_provider.ts) is the SAME default-substitution
|
|
271
278
|
// logic `registerOllamaModel` uses for a real dispatch, including
|
|
@@ -318,6 +325,106 @@ export async function doctorCommand(argv) {
|
|
|
318
325
|
check(report, `${label} tool "${toolName}"`, false, "not a known tool name");
|
|
319
326
|
}
|
|
320
327
|
}
|
|
328
|
+
// Tiering (SPF #14) — three checks, ALL gated on tiering.enabled: a
|
|
329
|
+
// disabled (or never-opted-into) ladder must be invisible to `spf doctor`,
|
|
330
|
+
// the same gate agents.validate() uses (see the "roster + suites
|
|
331
|
+
// validate" check above, which already fails loudly on a rule-T backend
|
|
332
|
+
// mismatch for the whole roster — nothing here duplicates that failure,
|
|
333
|
+
// this section only adds VISIBILITY on top of it). Without this gate,
|
|
334
|
+
// `spf init --template ts && spf doctor` would fail on three unset
|
|
335
|
+
// provider keys for a ladder that is off and will never dispatch
|
|
336
|
+
// (integration point 18's own regression note).
|
|
337
|
+
if (cfg.tiering.enabled) {
|
|
338
|
+
// 18(b): the SAME provider-key check the per-agent loop above just ran,
|
|
339
|
+
// over cfg.tiering.tiers[].model instead of cfg.agents[].model,
|
|
340
|
+
// branching on the TIER's OWN declared coding_agent (never an agent's —
|
|
341
|
+
// a tier changes `model` and nothing else, see core/tiering.ts's rule
|
|
342
|
+
// T). Without this, a hosted rung naming a provider whose key is unset
|
|
343
|
+
// is a green doctor and a first-dispatch failure.
|
|
344
|
+
for (const tier of cfg.tiering.tiers) {
|
|
345
|
+
const tierLabel = `tiering tier "${tier.name}"`;
|
|
346
|
+
if (tier.coding_agent === "claude_code") {
|
|
347
|
+
check(report, `${tierLabel} provider key`, true, process.env["ANTHROPIC_API_KEY"] ? "ANTHROPIC_API_KEY is set" : "ANTHROPIC_API_KEY not set — fine if authenticated via `claude login` instead");
|
|
348
|
+
}
|
|
349
|
+
else {
|
|
350
|
+
try {
|
|
351
|
+
const [provider] = resolveModel(tier.model);
|
|
352
|
+
const envKeys = PROVIDER_ENV_KEYS[provider];
|
|
353
|
+
if (!envKeys) {
|
|
354
|
+
check(report, `${tierLabel} provider key`, true, `provider "${provider}" not in doctor's known list — skipped, not a failure`);
|
|
355
|
+
}
|
|
356
|
+
else if (envKeys.length === 0) {
|
|
357
|
+
check(report, `${tierLabel} provider key`, true, `provider "${provider}" is keyless — no key required`);
|
|
358
|
+
}
|
|
359
|
+
else {
|
|
360
|
+
const set = envKeys.find((k) => process.env[k]);
|
|
361
|
+
check(report, `${tierLabel} provider key`, Boolean(set), set ? `${set} is set` : `none of ${envKeys.join(", ")} is set`);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
catch (error) {
|
|
365
|
+
check(report, `${tierLabel} model`, false, error.message);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
// 18(c): the resolved ladder, each routed role's effective model, and
|
|
370
|
+
// the served-tag report — imports probeServedOllamaTags FROM
|
|
371
|
+
// core/tiering.ts (not doctor's own probeGet, which discards the
|
|
372
|
+
// response body that's the only part that matters here).
|
|
373
|
+
//
|
|
374
|
+
// chainName/prompt are deliberately "" — an unlisted chain name weighs
|
|
375
|
+
// 0 and a zero-word prompt weighs -1, summing to -1, which classifies
|
|
376
|
+
// as "standard" (short of the -2 "low" needs). That reports the
|
|
377
|
+
// LADDER'S OWN baseline resolution (step 0 — no risk shift), not any
|
|
378
|
+
// one chain's risk-shifted view; a risk-shifted projection for a real
|
|
379
|
+
// chain+prompt is `spf estimate`'s job, not doctor's. `required` is the
|
|
380
|
+
// WHOLE roster, matching the "roster + suites validate" call above —
|
|
381
|
+
// every routed role doctor's job covers, not just what one chain needs.
|
|
382
|
+
const servedOllamaTags = flags["no-probe"] ? null : await probeServedOllamaTags(cfg);
|
|
383
|
+
const resolution = resolveTiering({
|
|
384
|
+
cfg,
|
|
385
|
+
chainName: "",
|
|
386
|
+
prompt: "",
|
|
387
|
+
servedOllamaTags,
|
|
388
|
+
required: cfg.agents.map((a) => a.name),
|
|
389
|
+
});
|
|
390
|
+
check(report, "tiering ladder", true, cfg.tiering.tiers.length > 0 ? cfg.tiering.tiers.map((t) => `${t.name}(${t.coding_agent}:${t.model})`).join(" -> ") : "(no tiers declared)", "info");
|
|
391
|
+
// FULL routing map — an entry for every routed role including the ones
|
|
392
|
+
// whose effective model equals their configured one (integration point
|
|
393
|
+
// 18c: "roles whose effective equals their configured model" still
|
|
394
|
+
// print), never a diff.
|
|
395
|
+
for (const [agentName, route] of Object.entries(resolution.routing)) {
|
|
396
|
+
check(report, `tiering role "${agentName}"`, true, `-> tier "${route.tier}" (${route.effective})${route.configured === route.effective ? " [unchanged from configured]" : ` (was ${route.configured})`}`, "info");
|
|
397
|
+
}
|
|
398
|
+
// One line per note resolveTiering produced (an unserved tag, a rule-T
|
|
399
|
+
// backend mismatch, an unknown tier name, ...) — resolveTiering never
|
|
400
|
+
// assigns severity (agents.validate() owns that; this call only owns
|
|
401
|
+
// detection), so every note here is informational: the hard failure for
|
|
402
|
+
// a rule-T mismatch or a malformed rung already came from the "roster +
|
|
403
|
+
// suites validate" check above, on the SAME resolveTiering-adjacent
|
|
404
|
+
// detection agents.validate() runs. This just names it again, in context.
|
|
405
|
+
for (const note of resolution.notes) {
|
|
406
|
+
check(report, "tiering note", true, note, "warn");
|
|
407
|
+
}
|
|
408
|
+
// Served-tag report, fail-open in the SAME direction the probe itself
|
|
409
|
+
// does: a stopped/unreachable server reports "could not check" once,
|
|
410
|
+
// never per-rung "not served" — flagging every rung as unserved on a
|
|
411
|
+
// probe failure would look identical to the prefix-strip bug
|
|
412
|
+
// core/tiering.ts's usable() guards against, and is exactly the outcome
|
|
413
|
+
// fail-open exists to prevent.
|
|
414
|
+
const ollamaTiers = cfg.tiering.tiers.filter((t) => t.coding_agent !== "claude_code" && t.model.startsWith("ollama/"));
|
|
415
|
+
if (ollamaTiers.length > 0) {
|
|
416
|
+
if (servedOllamaTags === null) {
|
|
417
|
+
check(report, "tiering served-tag probe", true, flags["no-probe"] ? "skipped (--no-probe)" : "could not check — Ollama server unreachable (fail-open: nothing is dropped this run)", "warn");
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
for (const tier of ollamaTiers) {
|
|
421
|
+
const tag = tier.model.slice("ollama/".length);
|
|
422
|
+
const served = servedOllamaTags.has(tag);
|
|
423
|
+
check(report, `tiering tier "${tier.name}" served`, true, served ? `"${tag}" is in \`ollama list\`` : `"${tag}" is NOT in \`ollama list\` — a run will walk down to a lower usable rung`, served ? "info" : "warn");
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
321
428
|
for (const spec of cfg.quality.checks) {
|
|
322
429
|
check(report, `quality check "${spec.name}"`, binaryOnPath(spec.argv[0]), `${spec.argv[0]} ${binaryOnPath(spec.argv[0]) ? "found on PATH" : "NOT found on PATH"}`);
|
|
323
430
|
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { type TierResolution } from "../../core/tiering.ts";
|
|
2
|
+
import type { ChainHistorySession } from "../../ui/server/db.ts";
|
|
3
|
+
export declare function usage(): string;
|
|
4
|
+
export type SampleStatus = "success" | "any" | "none";
|
|
5
|
+
export interface Sample {
|
|
6
|
+
sessions: ChainHistorySession[];
|
|
7
|
+
status: SampleStatus;
|
|
8
|
+
fanoutCollapsed: number;
|
|
9
|
+
joinedExcluded: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Two tiers, stopping at the first that qualifies, and an honest refusal.
|
|
13
|
+
* `sessions` is already most-recent-first (db.ts's own ORDER BY).
|
|
14
|
+
*/
|
|
15
|
+
export declare function selectSample(sessions: ChainHistorySession[], joinedExcluded: number): Sample;
|
|
16
|
+
/** p50, no interpolation — a 3-sample percentile is false precision (design doc §5.3). Even-length arrays average the two middle values, the ordinary definition. */
|
|
17
|
+
export declare function median(values: number[]): number;
|
|
18
|
+
/** Strip a loop factory's trailing `_<digits>` (`steps.ts:572-579,597-600,670,683`) — `fix_1`/`fix_2` -> `fix`, `test_1` -> `test`. Reads the recorded name; never predicts it. */
|
|
19
|
+
export declare function normalizePhaseName(name: string): string;
|
|
20
|
+
export interface PhaseProjection {
|
|
21
|
+
name: string;
|
|
22
|
+
p50: number;
|
|
23
|
+
min: number;
|
|
24
|
+
max: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Per run: sum every iteration of a normalized phase into one observation.
|
|
28
|
+
* Across runs: median (p50) and min/max of those per-run sums. A phase
|
|
29
|
+
* absent from a given run (a fix loop that never triggered) counts as a
|
|
30
|
+
* `0` observation for that run — not a skip — so its median reflects how
|
|
31
|
+
* often the phase actually ran, not just its size when it did.
|
|
32
|
+
*/
|
|
33
|
+
export declare function aggregatePhases(sessions: ChainHistorySession[]): PhaseProjection[];
|
|
34
|
+
export interface RunTotalsProjection {
|
|
35
|
+
p50: number;
|
|
36
|
+
min: number;
|
|
37
|
+
max: number;
|
|
38
|
+
}
|
|
39
|
+
/** The HEADLINE: median of sampled runs' `sessions.total_tokens` — NOT the sum of the per-phase medians (those are different numbers; median of sums != sum of medians). */
|
|
40
|
+
export declare function projectRunTotals(sessions: ChainHistorySession[]): RunTotalsProjection | null;
|
|
41
|
+
/** Where a p50 run would be cut off by `max_run_tokens`, walking the phase breakdown cumulatively in the order phases first appeared. `undefined` means unset; `null` means set but never reached on the p50 path. */
|
|
42
|
+
export declare function findCutoffPhase(phases: PhaseProjection[], maxRunTokens: number | undefined): string | null | undefined;
|
|
43
|
+
export interface DriftWarning {
|
|
44
|
+
agent: string;
|
|
45
|
+
historical: string;
|
|
46
|
+
current: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Compares each sampled agent's HISTORICAL dispatch model against the
|
|
50
|
+
* resolution's EFFECTIVE model — `res.routing[agent]?.effective ?? agent's
|
|
51
|
+
* own configured model` — never against `roles`/`tiers` in isolation. That
|
|
52
|
+
* is what makes the warning fire correctly whether or not tiering is
|
|
53
|
+
* enabled: with tiering off, "effective" is just the agent's own `model:`,
|
|
54
|
+
* so an operator who edited the roster directly between runs (no tiering
|
|
55
|
+
* involved) still gets an accurate warning.
|
|
56
|
+
*/
|
|
57
|
+
export declare function detectModelDrift(historicalModels: Map<string, string>, currentModel: (agent: string) => string | undefined): DriftWarning[];
|
|
58
|
+
export interface EstimateReport {
|
|
59
|
+
chain: string;
|
|
60
|
+
risk: TierResolution["risk"];
|
|
61
|
+
signals: TierResolution["signals"];
|
|
62
|
+
routing: TierResolution["routing"];
|
|
63
|
+
notes: string[];
|
|
64
|
+
sample: {
|
|
65
|
+
n: number;
|
|
66
|
+
status: SampleStatus;
|
|
67
|
+
joined_excluded: number;
|
|
68
|
+
fanout_collapsed: number;
|
|
69
|
+
};
|
|
70
|
+
phases: PhaseProjection[];
|
|
71
|
+
projected: RunTotalsProjection | null;
|
|
72
|
+
observed_cost: number;
|
|
73
|
+
drift: DriftWarning[];
|
|
74
|
+
validate_error: string | null;
|
|
75
|
+
ceilings: {
|
|
76
|
+
max_run_tokens: number | undefined;
|
|
77
|
+
max_run_cost: number | undefined;
|
|
78
|
+
cutoff_phase: string | null | undefined;
|
|
79
|
+
};
|
|
80
|
+
fanout_n: number;
|
|
81
|
+
}
|
|
82
|
+
export declare function estimateCommand(argv: string[]): Promise<number>;
|