@gr8ful/spf 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -17
- package/assets/defaults/spf.config.yaml +68 -0
- package/assets/prompts/refiner/system.md +32 -6
- package/assets/prompts/refiner/user.md +37 -5
- package/assets/skill/SKILL.md +1 -0
- package/assets/skill/references/config.md +153 -4
- package/assets/templates/ts-flue-ollama.spf.config.yaml +25 -0
- package/assets/templates/ts.spf.config.yaml +6 -0
- package/dist/chains/index.d.ts +11 -0
- package/dist/chains/index.js +38 -3
- package/dist/chains/repo_chains.js +1 -0
- package/dist/chains/simple_sdlc.js +1 -1
- package/dist/chains/steps.d.ts +27 -9
- package/dist/chains/steps.js +82 -13
- package/dist/cli/commands/doctor.js +108 -1
- package/dist/cli/commands/estimate.d.ts +82 -0
- package/dist/cli/commands/estimate.js +317 -0
- package/dist/cli/commands/fanout.d.ts +40 -0
- package/dist/cli/commands/fanout.js +401 -0
- package/dist/cli/commands/init.js +19 -0
- package/dist/cli/commands/trace.d.ts +18 -0
- package/dist/cli/commands/trace.js +22 -3
- package/dist/cli/commands/watch.js +51 -26
- package/dist/cli/index.js +12 -0
- package/dist/core/agents.d.ts +56 -0
- package/dist/core/agents.js +152 -1
- package/dist/core/data_types.d.ts +232 -0
- package/dist/core/data_types.js +135 -0
- package/dist/core/fanout.d.ts +229 -0
- package/dist/core/fanout.js +313 -0
- package/dist/core/gates.d.ts +8 -0
- package/dist/core/gates.js +24 -1
- package/dist/core/git_helper.d.ts +98 -0
- package/dist/core/git_helper.js +127 -0
- package/dist/core/issues/github_provider.d.ts +7 -1
- package/dist/core/issues/github_provider.js +33 -2
- package/dist/core/issues/jira_provider.d.ts +5 -1
- package/dist/core/issues/jira_provider.js +26 -3
- package/dist/core/issues/provider.d.ts +53 -2
- package/dist/core/notify/channel.d.ts +1 -1
- package/dist/core/notify/notifier.d.ts +16 -0
- package/dist/core/notify/notifier.js +36 -0
- package/dist/core/otel.d.ts +64 -9
- package/dist/core/otel.js +78 -14
- package/dist/core/paths.d.ts +3 -0
- package/dist/core/paths.js +48 -1
- package/dist/core/runner.d.ts +8 -0
- package/dist/core/runner.js +7 -0
- package/dist/core/session.d.ts +25 -0
- package/dist/core/session.js +97 -28
- package/dist/core/tiering.d.ts +145 -0
- package/dist/core/tiering.js +235 -0
- package/dist/core/tracer.d.ts +10 -0
- package/dist/core/tracer.js +12 -0
- package/dist/core/watch.d.ts +89 -9
- package/dist/core/watch.js +193 -19
- package/dist/ui/server/db.d.ts +39 -0
- package/dist/ui/server/db.js +61 -0
- package/package.json +2 -1
- package/dist/test/agent_cc.test.d.ts +0 -1
- package/dist/test/agent_cc.test.js +0 -95
- package/dist/test/agent_flue.test.d.ts +0 -1
- package/dist/test/agent_flue.test.js +0 -83
- package/dist/test/chains.test.d.ts +0 -12
- package/dist/test/chains.test.js +0 -92
- package/dist/test/data_types.test.d.ts +0 -10
- package/dist/test/data_types.test.js +0 -220
- package/dist/test/env_file.test.d.ts +0 -1
- package/dist/test/env_file.test.js +0 -74
- package/dist/test/fake_asker.d.ts +0 -23
- package/dist/test/fake_asker.js +0 -30
- package/dist/test/git_helper.test.d.ts +0 -1
- package/dist/test/git_helper.test.js +0 -59
- package/dist/test/hermetic_git.d.ts +0 -1
- package/dist/test/hermetic_git.js +0 -22
- package/dist/test/init_command.test.d.ts +0 -14
- package/dist/test/init_command.test.js +0 -136
- package/dist/test/interview.test.d.ts +0 -15
- package/dist/test/interview.test.js +0 -425
- package/dist/test/notify.test.d.ts +0 -1
- package/dist/test/notify.test.js +0 -174
- package/dist/test/ollama_provider.test.d.ts +0 -1
- package/dist/test/ollama_provider.test.js +0 -103
- package/dist/test/otel.test.d.ts +0 -26
- package/dist/test/otel.test.js +0 -512
- package/dist/test/refine.test.d.ts +0 -1
- package/dist/test/refine.test.js +0 -189
- package/dist/test/repo_chains.test.d.ts +0 -21
- package/dist/test/repo_chains.test.js +0 -416
- package/dist/test/signoff.test.d.ts +0 -1
- package/dist/test/signoff.test.js +0 -329
- package/dist/test/ui_server.test.d.ts +0 -7
- package/dist/test/ui_server.test.js +0 -120
- package/dist/test/watch.test.d.ts +0 -1
- package/dist/test/watch.test.js +0 -687
package/dist/core/session.d.ts
CHANGED
|
@@ -7,6 +7,31 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { Run } from "./runner.ts";
|
|
9
9
|
import type { SFConfig } from "./data_types.ts";
|
|
10
|
+
/**
|
|
11
|
+
* The symmetric teardown for `finalizeWhenKilled()` above: drop `adwId` from
|
|
12
|
+
* `ACTIVE` (so a later signal can no longer reach it) and close its Tracer's
|
|
13
|
+
* sqlite handle. Call once a run's own dispatch has fully settled — success
|
|
14
|
+
* or thrown error alike; `chains/index.ts`'s `runChain()` finally is the one
|
|
15
|
+
* seam every dispatch path (one-shot CLI and `spf watch` alike) shares on
|
|
16
|
+
* the way out, exactly where `otel.releaseOtelExporter()` already lives for
|
|
17
|
+
* the identical reason (see otel.ts's RUN-SCOPED CLEANUP note / #26).
|
|
18
|
+
*
|
|
19
|
+
* Without this, `spf watch` held every finished run's `Run` (and its
|
|
20
|
+
* Tracer, its open sqlite handle, and its Notifier) strongly reachable from
|
|
21
|
+
* the signal listener for the rest of the daemon's life — the listener
|
|
22
|
+
* closes over the `run` a fresh `finalizeWhenKilled()` call captured, but
|
|
23
|
+
* since #26 only that one listener installs once now, and `ACTIVE` is the
|
|
24
|
+
* only thing keeping a finished run reachable from it.
|
|
25
|
+
*
|
|
26
|
+
* A one-shot invocation with no explicit `--adw-id` makes `adwId` here the
|
|
27
|
+
* caller's `ctx.adw_id` (`null`) rather than the id `session.ensure()`
|
|
28
|
+
* actually minted, so this is a harmless no-op for it — same caveat as
|
|
29
|
+
* `releaseOtelExporter`, and harmless for the same reason: that process
|
|
30
|
+
* exits right after anyway.
|
|
31
|
+
*/
|
|
32
|
+
export declare function finalize(adwId: string | null | undefined): void;
|
|
33
|
+
/** Tests only: which adw_ids the process-wide signal handler currently considers active. */
|
|
34
|
+
export declare function activeRunIdsForTest(): string[];
|
|
10
35
|
/**
|
|
11
36
|
* `cwd` anchors this run's repo_root and data_dir — it is NOT where the
|
|
12
37
|
* process happened to start; it is an explicit decision, threaded down from
|
package/dist/core/session.js
CHANGED
|
@@ -10,16 +10,35 @@ import * as paths from "./paths.js";
|
|
|
10
10
|
import { Run } from "./runner.js";
|
|
11
11
|
import { Tracer } from "./tracer.js";
|
|
12
12
|
import { engineerName, newId } from "./utils.js";
|
|
13
|
-
import { resolveNotifier } from "./notify/notifier.js";
|
|
13
|
+
import { resolveNotifier, drainAll as drainNotifiers } from "./notify/notifier.js";
|
|
14
14
|
import * as otel from "./otel.js";
|
|
15
15
|
/**
|
|
16
|
-
* How long a signalled run may spend pushing spans before it
|
|
17
|
-
* Short on purpose: someone who just pressed ^C is waiting, and
|
|
18
|
-
* observability projection is
|
|
19
|
-
* budget is enforced inside `otel.flushAll()`
|
|
20
|
-
*
|
|
16
|
+
* How long a signalled run may spend pushing spans / webhook sends before it
|
|
17
|
+
* exits anyway. Short on purpose: someone who just pressed ^C is waiting, and
|
|
18
|
+
* neither an observability projection nor a Slack ping is ever worth making a
|
|
19
|
+
* kill feel broken. The budget is enforced inside `otel.flushAll()` and
|
|
20
|
+
* `notify.drainAll()` (each a raced, unref'd deadline), so an unreachable
|
|
21
|
+
* collector or webhook host costs exactly this and not one tick more.
|
|
21
22
|
*/
|
|
22
23
|
const SIGNAL_DRAIN_MS = 750;
|
|
24
|
+
/**
|
|
25
|
+
* Runs currently in flight in THIS process, keyed by adw_id — what the one
|
|
26
|
+
* shared signal handler below acts on, and what `finalize()` removes a run
|
|
27
|
+
* from once its own dispatch has settled.
|
|
28
|
+
*
|
|
29
|
+
* `spf watch`'s daemon loop calls `ensure()` once per claimed issue, in the
|
|
30
|
+
* same process, for the life of the daemon. Installing a FRESH
|
|
31
|
+
* SIGTERM/SIGINT listener per `ensure()` call (the old shape) meant every
|
|
32
|
+
* listener ever registered stayed registered and fired on the NEXT signal —
|
|
33
|
+
* so ^C during issue 72 would also re-run issue 70 and 71's long-finished
|
|
34
|
+
* handlers, flipping their already-`success` sessions back to `fail`, on top
|
|
35
|
+
* of leaking a `MaxListenersExceededWarning` at the 11th run. One listener,
|
|
36
|
+
* installed once, acting on whichever runs are still in `ACTIVE`, fixes
|
|
37
|
+
* both: a signal only ever finalizes runs that are actually still running.
|
|
38
|
+
*/
|
|
39
|
+
const ACTIVE = new Map();
|
|
40
|
+
let installed = false;
|
|
41
|
+
let draining = false;
|
|
23
42
|
/**
|
|
24
43
|
* A killed run still closes its own trace.
|
|
25
44
|
*
|
|
@@ -31,32 +50,82 @@ const SIGNAL_DRAIN_MS = 750;
|
|
|
31
50
|
* (best-effort: a signal can still land mid-write).
|
|
32
51
|
*
|
|
33
52
|
* SQLite is written FIRST and synchronously, exactly as before — the otel
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
53
|
+
* and notify drains are appended after it and can only ever cost time, never
|
|
54
|
+
* correctness. `notify`'s in-flight Slack/Teams/webhook sends get the SAME
|
|
55
|
+
* timeout-and-swallow discipline as the otel drain (see `Notifier.drain()` /
|
|
56
|
+
* `notify.drainAll()` in `notify/notifier.ts`): bounded by the same
|
|
57
|
+
* `SIGNAL_DRAIN_MS` budget, run concurrently with the otel drain (not after
|
|
58
|
+
* it, so a killed run never pays both budgets back to back), and never able
|
|
59
|
+
* to throw into this handler.
|
|
38
60
|
* A second signal during the drain exits immediately — someone pressing ^C
|
|
39
61
|
* twice means "now", and a shutdown path that ignores that is a hang.
|
|
40
62
|
*/
|
|
41
|
-
function
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
function handleSignal(signal) {
|
|
64
|
+
const code = 128 + (signal === "SIGINT" ? 2 : 15);
|
|
65
|
+
if (draining)
|
|
66
|
+
process.exit(code);
|
|
67
|
+
draining = true;
|
|
68
|
+
const runs = [...ACTIVE.values()]; // snapshot: finalize() may mutate ACTIVE mid-drain
|
|
69
|
+
for (const run of runs)
|
|
48
70
|
run.tracer.sessionFinish(run.adw_id, false); // also closes process rows
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
71
|
+
const drains = [];
|
|
72
|
+
if (runs.some((run) => run.tracer.otel))
|
|
73
|
+
drains.push(otel.flushAll(SIGNAL_DRAIN_MS));
|
|
74
|
+
if (runs.some((run) => run.notify))
|
|
75
|
+
drains.push(drainNotifiers(SIGNAL_DRAIN_MS));
|
|
76
|
+
// Unconfigured (the default) exits SYNCHRONOUSLY, exactly as it did before
|
|
77
|
+
// otel/notify existed — no extra tick between the signal and the exit for
|
|
78
|
+
// the repos that never opted in to either.
|
|
79
|
+
if (drains.length === 0) {
|
|
80
|
+
process.exit(code);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
// Bounded and never-throwing: both drains swallow their own failures and
|
|
84
|
+
// resolve on their own deadline, so this always reaches process.exit().
|
|
85
|
+
void Promise.all(drains).then(() => process.exit(code), () => process.exit(code));
|
|
86
|
+
}
|
|
87
|
+
function finalizeWhenKilled(run) {
|
|
88
|
+
ACTIVE.set(run.adw_id, run);
|
|
89
|
+
if (installed)
|
|
90
|
+
return;
|
|
91
|
+
installed = true;
|
|
92
|
+
process.on("SIGTERM", handleSignal);
|
|
93
|
+
process.on("SIGINT", handleSignal);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The symmetric teardown for `finalizeWhenKilled()` above: drop `adwId` from
|
|
97
|
+
* `ACTIVE` (so a later signal can no longer reach it) and close its Tracer's
|
|
98
|
+
* sqlite handle. Call once a run's own dispatch has fully settled — success
|
|
99
|
+
* or thrown error alike; `chains/index.ts`'s `runChain()` finally is the one
|
|
100
|
+
* seam every dispatch path (one-shot CLI and `spf watch` alike) shares on
|
|
101
|
+
* the way out, exactly where `otel.releaseOtelExporter()` already lives for
|
|
102
|
+
* the identical reason (see otel.ts's RUN-SCOPED CLEANUP note / #26).
|
|
103
|
+
*
|
|
104
|
+
* Without this, `spf watch` held every finished run's `Run` (and its
|
|
105
|
+
* Tracer, its open sqlite handle, and its Notifier) strongly reachable from
|
|
106
|
+
* the signal listener for the rest of the daemon's life — the listener
|
|
107
|
+
* closes over the `run` a fresh `finalizeWhenKilled()` call captured, but
|
|
108
|
+
* since #26 only that one listener installs once now, and `ACTIVE` is the
|
|
109
|
+
* only thing keeping a finished run reachable from it.
|
|
110
|
+
*
|
|
111
|
+
* A one-shot invocation with no explicit `--adw-id` makes `adwId` here the
|
|
112
|
+
* caller's `ctx.adw_id` (`null`) rather than the id `session.ensure()`
|
|
113
|
+
* actually minted, so this is a harmless no-op for it — same caveat as
|
|
114
|
+
* `releaseOtelExporter`, and harmless for the same reason: that process
|
|
115
|
+
* exits right after anyway.
|
|
116
|
+
*/
|
|
117
|
+
export function finalize(adwId) {
|
|
118
|
+
if (!adwId)
|
|
119
|
+
return;
|
|
120
|
+
const run = ACTIVE.get(adwId);
|
|
121
|
+
if (!run)
|
|
122
|
+
return;
|
|
123
|
+
ACTIVE.delete(adwId);
|
|
124
|
+
run.tracer.close();
|
|
125
|
+
}
|
|
126
|
+
/** Tests only: which adw_ids the process-wide signal handler currently considers active. */
|
|
127
|
+
export function activeRunIdsForTest() {
|
|
128
|
+
return [...ACTIVE.keys()];
|
|
60
129
|
}
|
|
61
130
|
/**
|
|
62
131
|
* `cwd` anchors this run's repo_root and data_dir — it is NOT where the
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Risk-tiered per-role model routing — SPF #14.
|
|
3
|
+
*
|
|
4
|
+
* PURE except for `probeServedOllamaTags`, the one async/impure export (a
|
|
5
|
+
* network probe — see its own doc comment below). Everything else here is a
|
|
6
|
+
* function of its own arguments: no `Run`, no `ChainContext`, no git handle,
|
|
7
|
+
* no filesystem. That purity is deliberate — it is what lets `startRun`
|
|
8
|
+
* (`src/chains/steps.ts`) and `spf estimate` call the IDENTICAL
|
|
9
|
+
* `resolveTiering` and get identical answers with no drift between "what
|
|
10
|
+
* will dispatch" and "what would dispatch".
|
|
11
|
+
*
|
|
12
|
+
* No imports from `src/chains/` and none from `src/cli/` — `spf doctor` and
|
|
13
|
+
* `startRun` both import FROM this module, never the reverse.
|
|
14
|
+
*
|
|
15
|
+
* ── The mechanism, in one paragraph ─────────────────────────────────────
|
|
16
|
+
* A run's `risk` (`low`/`standard`/`high`) is a single run-global scalar,
|
|
17
|
+
* classified once from the chain's name and the prompt's word count
|
|
18
|
+
* (`classifyRisk`). Separately, `tiering.roles` names a baseline tier per
|
|
19
|
+
* ROLE (an agent name) on `tiering.tiers`, a ladder ordered weakest first.
|
|
20
|
+
* `resolveTiering` shifts every routed role's baseline by the SAME step in
|
|
21
|
+
* the SAME direction (down for `low`, up for `high`), walking down to the
|
|
22
|
+
* nearest rung that is both available (§ probe) and backend-compatible
|
|
23
|
+
* (rule T) — never up, because a degradation must never silently escalate
|
|
24
|
+
* spend. An agent not named in `roles` is never touched: its own `model:`
|
|
25
|
+
* stands. `effectiveAgent` is the one place that turns a resolution into an
|
|
26
|
+
* actual `AgentConfig` — overriding `model` and NOTHING else.
|
|
27
|
+
*/
|
|
28
|
+
import type { AgentConfig, SFConfig } from "./data_types.ts";
|
|
29
|
+
/**
|
|
30
|
+
* One explicit `name -> weight` table entry per BUILT-IN chain
|
|
31
|
+
* (`src/chains/index.ts`'s `CHAINS`), including every chain that weighs
|
|
32
|
+
* `0` — an entry, not a fallthrough. A name absent from this table (any
|
|
33
|
+
* repo-local `.spf/chains/*.yaml` chain, or a built-in nobody has wired in
|
|
34
|
+
* yet) falls to `0` through `chainWeight`'s own default — a DIFFERENT code
|
|
35
|
+
* path from being explicitly listed as `0`, which is what lets a test
|
|
36
|
+
* assert "every built-in chain has a real entry here" rather than merely
|
|
37
|
+
* "this returned 0", so a new built-in nobody wired in cannot land in the
|
|
38
|
+
* `0` bucket unnoticed.
|
|
39
|
+
*/
|
|
40
|
+
export declare const CHAIN_WEIGHTS: Record<string, -1 | 0 | 1>;
|
|
41
|
+
/** `name in CHAIN_WEIGHTS` distinguishes "explicitly 0" from "fell through" — see `CHAIN_WEIGHTS`'s own comment. */
|
|
42
|
+
export declare function chainWeight(name: string): number;
|
|
43
|
+
/** Whitespace-delimited word count, deliberately not tokens — no tokenizer exists for an arbitrary ollama/<tag>, and a word count is free, stable, and reproducible from the trace. */
|
|
44
|
+
export declare function promptWords(prompt: string): number;
|
|
45
|
+
/** `-1` for a terse ask, `+1` for a long one, `0` otherwise. */
|
|
46
|
+
export declare function promptWeight(words: number): number;
|
|
47
|
+
export type Risk = "low" | "standard" | "high";
|
|
48
|
+
export interface TierSignals {
|
|
49
|
+
chain: string;
|
|
50
|
+
chain_weight: number;
|
|
51
|
+
prompt_words: number;
|
|
52
|
+
prompt_weight: number;
|
|
53
|
+
sum: number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The classifier. Asymmetric on purpose: demotion (`low`) requires TWO
|
|
57
|
+
* agreeing signals (`sum <= -2` — both `chain_weight` and `prompt_weight`
|
|
58
|
+
* at `-1`); promotion (`high`) requires only ONE. Concretely that means a
|
|
59
|
+
* long prompt (`prompt_weight === 1`, ≥400 words) forces `high`
|
|
60
|
+
* UNCONDITIONALLY, even riding on top of a `-1` chain (`spf scout "<≥400
|
|
61
|
+
* words>"` → `high`, not the `sum === 0` "standard" a naive `sum >= 1`
|
|
62
|
+
* threshold would give) — the design doc's own §2 3×3 table and its §3.1
|
|
63
|
+
* reachability table both pin this cell as `high`, so the table (not the
|
|
64
|
+
* simpler-looking `sum` arithmetic sketched alongside it) is what
|
|
65
|
+
* `src/test/tiering.test.ts`'s full-grid test treats as the specification.
|
|
66
|
+
* An unnecessarily weak model produces a wrong answer that costs a whole
|
|
67
|
+
* re-run — the expensive failure; an unnecessarily strong one just costs
|
|
68
|
+
* some tokens — the cheap failure. Skewed toward the cheap one.
|
|
69
|
+
*/
|
|
70
|
+
export declare function classifyRisk(chainName: string, prompt: string): {
|
|
71
|
+
risk: Risk;
|
|
72
|
+
signals: TierSignals;
|
|
73
|
+
};
|
|
74
|
+
/** PURE input — no `Run`, no `ChainContext`. `required` scopes everything: an agent no phase in this run will dispatch is neither routed nor reported. */
|
|
75
|
+
export interface TierInput {
|
|
76
|
+
cfg: SFConfig;
|
|
77
|
+
chainName: string;
|
|
78
|
+
prompt: string;
|
|
79
|
+
/** `null` == "not probed / probe failed" == fail open, drop nothing. */
|
|
80
|
+
servedOllamaTags: Set<string> | null;
|
|
81
|
+
required: string[];
|
|
82
|
+
}
|
|
83
|
+
export interface TierRoute {
|
|
84
|
+
/** the tier NAME finally selected, after the shift and any walk-down */
|
|
85
|
+
tier: string;
|
|
86
|
+
/** agent.model as CONFIGURED (post-back-fill) */
|
|
87
|
+
configured: string;
|
|
88
|
+
/** the model that will actually dispatch */
|
|
89
|
+
effective: string;
|
|
90
|
+
}
|
|
91
|
+
export interface TierResolution {
|
|
92
|
+
risk: Risk;
|
|
93
|
+
signals: TierSignals;
|
|
94
|
+
/**
|
|
95
|
+
* FULL routing: one entry for every agent in `required` that tiering
|
|
96
|
+
* routes — including entries where `effective === configured`. NOT a
|
|
97
|
+
* diff; `changedModels()` below is the derived diff view. An agent not in
|
|
98
|
+
* `required`, or not named in `roles`, has no entry.
|
|
99
|
+
*/
|
|
100
|
+
routing: Record<string, TierRoute>;
|
|
101
|
+
/** One line per degradation or non-routable role. Empty when tiering is disabled or has nothing to say. */
|
|
102
|
+
notes: string[];
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* The ladder walk (design doc §4.3). Never throws and never reports a
|
|
106
|
+
* severity — a backend mismatch (rule T) or an unknown tier name is a
|
|
107
|
+
* `notes` line plus a role left unrouted; `agents.validate()` owns
|
|
108
|
+
* severity, this owns detection.
|
|
109
|
+
*/
|
|
110
|
+
export declare function resolveTiering(input: TierInput): TierResolution;
|
|
111
|
+
/**
|
|
112
|
+
* The diff view, derived — `{agent: effective}` for entries where
|
|
113
|
+
* `configured !== effective`. Empty means tiering changed nothing this run.
|
|
114
|
+
* Deriving it (rather than storing it) is what keeps this view and the
|
|
115
|
+
* full `routing` map from disagreeing: one source, one filter.
|
|
116
|
+
*/
|
|
117
|
+
export declare function changedModels(res: TierResolution): Record<string, string>;
|
|
118
|
+
/**
|
|
119
|
+
* The single dispatch-site change (design doc §4.6): `model` and ONLY
|
|
120
|
+
* `model`. `run` is a minimal structural shape — deliberately not
|
|
121
|
+
* `agents.ts`'s `RunForAgents`, which would import this module and create a
|
|
122
|
+
* cycle — so any run-shaped object with an (optional) `tiering` field
|
|
123
|
+
* satisfies it.
|
|
124
|
+
*/
|
|
125
|
+
export declare function effectiveAgent(run: {
|
|
126
|
+
tiering?: TierResolution | null;
|
|
127
|
+
}, base: AgentConfig): AgentConfig;
|
|
128
|
+
/**
|
|
129
|
+
* `Promise<Set<string> | null>` of bare (unprefixed) served Ollama tags, or
|
|
130
|
+
* `null` on any failure — non-200, timeout, connection refused, an
|
|
131
|
+
* unparseable/`data`-less body. `null` means "fail open, drop nothing",
|
|
132
|
+
* never "everything is unserved": an unreachable probe silently degrading
|
|
133
|
+
* every agent to the bottom rung is a worse outcome than the honest
|
|
134
|
+
* "Unknown model ID" error Ollama itself raises at first dispatch.
|
|
135
|
+
*
|
|
136
|
+
* Only actually probes when tiering is enabled AND at least one declared
|
|
137
|
+
* tier's model starts with `ollama/` — never on a run whose tiers are all
|
|
138
|
+
* hosted, and never when tiering is off. Memoized once per process.
|
|
139
|
+
*
|
|
140
|
+
* A NEW helper, not a reuse of `spf doctor`'s `probeGet`: that helper
|
|
141
|
+
* discards the response body, which is the only part that matters here.
|
|
142
|
+
*/
|
|
143
|
+
export declare function probeServedOllamaTags(cfg: SFConfig): Promise<Set<string> | null>;
|
|
144
|
+
/** Test-only: `probeServedOllamaTags` memoizes once per process, so a test that stubs `fetch` differently across cases must reset the cache between them. */
|
|
145
|
+
export declare function resetProbeCacheForTest(): void;
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Risk-tiered per-role model routing — SPF #14.
|
|
3
|
+
*
|
|
4
|
+
* PURE except for `probeServedOllamaTags`, the one async/impure export (a
|
|
5
|
+
* network probe — see its own doc comment below). Everything else here is a
|
|
6
|
+
* function of its own arguments: no `Run`, no `ChainContext`, no git handle,
|
|
7
|
+
* no filesystem. That purity is deliberate — it is what lets `startRun`
|
|
8
|
+
* (`src/chains/steps.ts`) and `spf estimate` call the IDENTICAL
|
|
9
|
+
* `resolveTiering` and get identical answers with no drift between "what
|
|
10
|
+
* will dispatch" and "what would dispatch".
|
|
11
|
+
*
|
|
12
|
+
* No imports from `src/chains/` and none from `src/cli/` — `spf doctor` and
|
|
13
|
+
* `startRun` both import FROM this module, never the reverse.
|
|
14
|
+
*
|
|
15
|
+
* ── The mechanism, in one paragraph ─────────────────────────────────────
|
|
16
|
+
* A run's `risk` (`low`/`standard`/`high`) is a single run-global scalar,
|
|
17
|
+
* classified once from the chain's name and the prompt's word count
|
|
18
|
+
* (`classifyRisk`). Separately, `tiering.roles` names a baseline tier per
|
|
19
|
+
* ROLE (an agent name) on `tiering.tiers`, a ladder ordered weakest first.
|
|
20
|
+
* `resolveTiering` shifts every routed role's baseline by the SAME step in
|
|
21
|
+
* the SAME direction (down for `low`, up for `high`), walking down to the
|
|
22
|
+
* nearest rung that is both available (§ probe) and backend-compatible
|
|
23
|
+
* (rule T) — never up, because a degradation must never silently escalate
|
|
24
|
+
* spend. An agent not named in `roles` is never touched: its own `model:`
|
|
25
|
+
* stands. `effectiveAgent` is the one place that turns a resolution into an
|
|
26
|
+
* actual `AgentConfig` — overriding `model` and NOTHING else.
|
|
27
|
+
*/
|
|
28
|
+
import { ollamaBaseUrl } from "./ollama_provider.js";
|
|
29
|
+
// ── the classifier (design doc §2) ──────────────────────────────────────────
|
|
30
|
+
/**
|
|
31
|
+
* One explicit `name -> weight` table entry per BUILT-IN chain
|
|
32
|
+
* (`src/chains/index.ts`'s `CHAINS`), including every chain that weighs
|
|
33
|
+
* `0` — an entry, not a fallthrough. A name absent from this table (any
|
|
34
|
+
* repo-local `.spf/chains/*.yaml` chain, or a built-in nobody has wired in
|
|
35
|
+
* yet) falls to `0` through `chainWeight`'s own default — a DIFFERENT code
|
|
36
|
+
* path from being explicitly listed as `0`, which is what lets a test
|
|
37
|
+
* assert "every built-in chain has a real entry here" rather than merely
|
|
38
|
+
* "this returned 0", so a new built-in nobody wired in cannot land in the
|
|
39
|
+
* `0` bucket unnoticed.
|
|
40
|
+
*/
|
|
41
|
+
export const CHAIN_WEIGHTS = {
|
|
42
|
+
scout: -1,
|
|
43
|
+
prompt: -1,
|
|
44
|
+
document: -1,
|
|
45
|
+
quality: -1,
|
|
46
|
+
"simple-sdlc": 1,
|
|
47
|
+
"plan-build-test-quality": 1,
|
|
48
|
+
refine: 1,
|
|
49
|
+
plan: 0,
|
|
50
|
+
build: 0,
|
|
51
|
+
"plan-build": 0,
|
|
52
|
+
"build-test": 0,
|
|
53
|
+
"plan-build-test": 0,
|
|
54
|
+
"build-review": 0,
|
|
55
|
+
};
|
|
56
|
+
/** `name in CHAIN_WEIGHTS` distinguishes "explicitly 0" from "fell through" — see `CHAIN_WEIGHTS`'s own comment. */
|
|
57
|
+
export function chainWeight(name) {
|
|
58
|
+
return name in CHAIN_WEIGHTS ? CHAIN_WEIGHTS[name] : 0;
|
|
59
|
+
}
|
|
60
|
+
/** Whitespace-delimited word count, deliberately not tokens — no tokenizer exists for an arbitrary ollama/<tag>, and a word count is free, stable, and reproducible from the trace. */
|
|
61
|
+
export function promptWords(prompt) {
|
|
62
|
+
return prompt.split(/\s+/).filter(Boolean).length;
|
|
63
|
+
}
|
|
64
|
+
/** `-1` for a terse ask, `+1` for a long one, `0` otherwise. */
|
|
65
|
+
export function promptWeight(words) {
|
|
66
|
+
if (words <= 60)
|
|
67
|
+
return -1;
|
|
68
|
+
if (words >= 400)
|
|
69
|
+
return 1;
|
|
70
|
+
return 0;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* The classifier. Asymmetric on purpose: demotion (`low`) requires TWO
|
|
74
|
+
* agreeing signals (`sum <= -2` — both `chain_weight` and `prompt_weight`
|
|
75
|
+
* at `-1`); promotion (`high`) requires only ONE. Concretely that means a
|
|
76
|
+
* long prompt (`prompt_weight === 1`, ≥400 words) forces `high`
|
|
77
|
+
* UNCONDITIONALLY, even riding on top of a `-1` chain (`spf scout "<≥400
|
|
78
|
+
* words>"` → `high`, not the `sum === 0` "standard" a naive `sum >= 1`
|
|
79
|
+
* threshold would give) — the design doc's own §2 3×3 table and its §3.1
|
|
80
|
+
* reachability table both pin this cell as `high`, so the table (not the
|
|
81
|
+
* simpler-looking `sum` arithmetic sketched alongside it) is what
|
|
82
|
+
* `src/test/tiering.test.ts`'s full-grid test treats as the specification.
|
|
83
|
+
* An unnecessarily weak model produces a wrong answer that costs a whole
|
|
84
|
+
* re-run — the expensive failure; an unnecessarily strong one just costs
|
|
85
|
+
* some tokens — the cheap failure. Skewed toward the cheap one.
|
|
86
|
+
*/
|
|
87
|
+
export function classifyRisk(chainName, prompt) {
|
|
88
|
+
const chain_weight = chainWeight(chainName);
|
|
89
|
+
const prompt_words = promptWords(prompt);
|
|
90
|
+
const prompt_weight = promptWeight(prompt_words);
|
|
91
|
+
const sum = chain_weight + prompt_weight;
|
|
92
|
+
const risk = prompt_weight === 1 || sum >= 1 ? "high" : sum <= -2 ? "low" : "standard";
|
|
93
|
+
return { risk, signals: { chain: chainName, chain_weight, prompt_words, prompt_weight, sum } };
|
|
94
|
+
}
|
|
95
|
+
const RISK_STEP = { low: -1, standard: 0, high: 1 };
|
|
96
|
+
function stripOllamaPrefix(model) {
|
|
97
|
+
return model.startsWith("ollama/") ? model.slice("ollama/".length) : model;
|
|
98
|
+
}
|
|
99
|
+
/** Usable == not dropped by the availability probe, AND backend-compatible (rule T). */
|
|
100
|
+
function usable(tier, agent, servedOllamaTags) {
|
|
101
|
+
if (tier.coding_agent !== agent.coding_agent)
|
|
102
|
+
return false; // rule T
|
|
103
|
+
if (servedOllamaTags !== null && tier.model.startsWith("ollama/")) {
|
|
104
|
+
// The comparison strips the "ollama/" prefix before lookup — getting
|
|
105
|
+
// this backwards fails CLOSED in the wrong direction: every ollama/*
|
|
106
|
+
// rung would look unserved and every ladder would walk to the bottom.
|
|
107
|
+
return servedOllamaTags.has(stripOllamaPrefix(tier.model));
|
|
108
|
+
}
|
|
109
|
+
return true;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* The ladder walk (design doc §4.3). Never throws and never reports a
|
|
113
|
+
* severity — a backend mismatch (rule T) or an unknown tier name is a
|
|
114
|
+
* `notes` line plus a role left unrouted; `agents.validate()` owns
|
|
115
|
+
* severity, this owns detection.
|
|
116
|
+
*/
|
|
117
|
+
export function resolveTiering(input) {
|
|
118
|
+
const { cfg, chainName, prompt, servedOllamaTags, required } = input;
|
|
119
|
+
const { risk, signals } = classifyRisk(chainName, prompt);
|
|
120
|
+
const routing = {};
|
|
121
|
+
const notes = [];
|
|
122
|
+
if (!cfg.tiering.enabled)
|
|
123
|
+
return { risk, signals, routing, notes };
|
|
124
|
+
const step = RISK_STEP[risk];
|
|
125
|
+
for (const agentName of required) {
|
|
126
|
+
const baseline = cfg.tiering.roles[agentName];
|
|
127
|
+
if (baseline === undefined)
|
|
128
|
+
continue; // precedence: not named in roles -> untouched
|
|
129
|
+
const agent = cfg.agents.find((a) => a.name === agentName);
|
|
130
|
+
if (!agent) {
|
|
131
|
+
notes.push(`tiering.roles.${agentName} names an agent not defined in this roster — left unrouted, no change`);
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
const i = cfg.tiering.tiers.findIndex((t) => t.name === baseline);
|
|
135
|
+
if (i === -1) {
|
|
136
|
+
notes.push(`tiering.roles.${agentName} names tier ${JSON.stringify(baseline)}, which is not declared in tiering.tiers — left unrouted`);
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
const target = Math.min(Math.max(i + step, 0), cfg.tiering.tiers.length - 1);
|
|
140
|
+
let j = target;
|
|
141
|
+
while (j >= 0 && !usable(cfg.tiering.tiers[j], agent, servedOllamaTags))
|
|
142
|
+
j--;
|
|
143
|
+
if (j < 0) {
|
|
144
|
+
notes.push(`${agentName}: no rung at or below ${cfg.tiering.tiers[target].name} is usable for this run (unserved tag or backend mismatch) — ` +
|
|
145
|
+
`left unrouted, dispatching on ${agent.model}`);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const tier = cfg.tiering.tiers[j];
|
|
149
|
+
routing[agentName] = { tier: tier.name, configured: agent.model, effective: tier.model };
|
|
150
|
+
}
|
|
151
|
+
return { risk, signals, routing, notes };
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* The diff view, derived — `{agent: effective}` for entries where
|
|
155
|
+
* `configured !== effective`. Empty means tiering changed nothing this run.
|
|
156
|
+
* Deriving it (rather than storing it) is what keeps this view and the
|
|
157
|
+
* full `routing` map from disagreeing: one source, one filter.
|
|
158
|
+
*/
|
|
159
|
+
export function changedModels(res) {
|
|
160
|
+
const changed = {};
|
|
161
|
+
for (const [agentName, route] of Object.entries(res.routing)) {
|
|
162
|
+
if (route.configured !== route.effective)
|
|
163
|
+
changed[agentName] = route.effective;
|
|
164
|
+
}
|
|
165
|
+
return changed;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* The single dispatch-site change (design doc §4.6): `model` and ONLY
|
|
169
|
+
* `model`. `run` is a minimal structural shape — deliberately not
|
|
170
|
+
* `agents.ts`'s `RunForAgents`, which would import this module and create a
|
|
171
|
+
* cycle — so any run-shaped object with an (optional) `tiering` field
|
|
172
|
+
* satisfies it.
|
|
173
|
+
*/
|
|
174
|
+
export function effectiveAgent(run, base) {
|
|
175
|
+
const effective = run.tiering?.routing[base.name]?.effective;
|
|
176
|
+
return effective === undefined ? base : { ...base, model: effective };
|
|
177
|
+
}
|
|
178
|
+
// ── availability probe (design doc §4.4B) ───────────────────────────────────
|
|
179
|
+
const PROBE_TIMEOUT_MS = 3_000; // same budget spf doctor's own probeGet uses
|
|
180
|
+
let cachedProbe;
|
|
181
|
+
async function fetchServedOllamaTags() {
|
|
182
|
+
const controller = new AbortController();
|
|
183
|
+
const timer = setTimeout(() => controller.abort(), PROBE_TIMEOUT_MS);
|
|
184
|
+
try {
|
|
185
|
+
// ollamaBaseUrl() is the SAME default-substitution a real dispatch
|
|
186
|
+
// uses, including treating a set-but-EMPTY OLLAMA_BASE_URL as unset —
|
|
187
|
+
// never re-derive that default here.
|
|
188
|
+
const url = ollamaBaseUrl().replace(/\/+$/, "") + "/models";
|
|
189
|
+
const res = await fetch(url, { signal: controller.signal });
|
|
190
|
+
if (res.status !== 200)
|
|
191
|
+
return null;
|
|
192
|
+
const body = (await res.json());
|
|
193
|
+
if (!body || !Array.isArray(body.data))
|
|
194
|
+
return null;
|
|
195
|
+
const tags = new Set();
|
|
196
|
+
for (const entry of body.data) {
|
|
197
|
+
if (entry && typeof entry === "object" && "id" in entry) {
|
|
198
|
+
tags.add(String(entry.id));
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
return tags; // bare tags — no "ollama/" prefix in the response
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return null; // timeout | connection refused | unparseable body all fail OPEN
|
|
205
|
+
}
|
|
206
|
+
finally {
|
|
207
|
+
clearTimeout(timer);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* `Promise<Set<string> | null>` of bare (unprefixed) served Ollama tags, or
|
|
212
|
+
* `null` on any failure — non-200, timeout, connection refused, an
|
|
213
|
+
* unparseable/`data`-less body. `null` means "fail open, drop nothing",
|
|
214
|
+
* never "everything is unserved": an unreachable probe silently degrading
|
|
215
|
+
* every agent to the bottom rung is a worse outcome than the honest
|
|
216
|
+
* "Unknown model ID" error Ollama itself raises at first dispatch.
|
|
217
|
+
*
|
|
218
|
+
* Only actually probes when tiering is enabled AND at least one declared
|
|
219
|
+
* tier's model starts with `ollama/` — never on a run whose tiers are all
|
|
220
|
+
* hosted, and never when tiering is off. Memoized once per process.
|
|
221
|
+
*
|
|
222
|
+
* A NEW helper, not a reuse of `spf doctor`'s `probeGet`: that helper
|
|
223
|
+
* discards the response body, which is the only part that matters here.
|
|
224
|
+
*/
|
|
225
|
+
export async function probeServedOllamaTags(cfg) {
|
|
226
|
+
if (!cfg.tiering.enabled || !cfg.tiering.tiers.some((t) => t.model.startsWith("ollama/")))
|
|
227
|
+
return null;
|
|
228
|
+
if (cachedProbe === undefined)
|
|
229
|
+
cachedProbe = fetchServedOllamaTags();
|
|
230
|
+
return cachedProbe;
|
|
231
|
+
}
|
|
232
|
+
/** Test-only: `probeServedOllamaTags` memoizes once per process, so a test that stubs `fetch` differently across cases must reset the cache between them. */
|
|
233
|
+
export function resetProbeCacheForTest() {
|
|
234
|
+
cachedProbe = undefined;
|
|
235
|
+
}
|
package/dist/core/tracer.d.ts
CHANGED
|
@@ -31,6 +31,16 @@ export declare class Tracer {
|
|
|
31
31
|
/** `null` unless `observability.otel` is configured — see the header. */
|
|
32
32
|
otel: OtelExporter | null;
|
|
33
33
|
constructor(dbPath: string, eventsJsonl: string, otel?: OtelExporter | null);
|
|
34
|
+
/**
|
|
35
|
+
* Close the sqlite handle. A one-shot CLI process never needs this — it
|
|
36
|
+
* exits right after its one Tracer anyway — but `spf watch`'s daemon loop
|
|
37
|
+
* builds a fresh Tracer (and a fresh `new Database(dbPath)`) per claimed
|
|
38
|
+
* issue, in-process, for the life of the daemon; without this, every
|
|
39
|
+
* issue's handle stayed open forever. Called from `session.ts`'s
|
|
40
|
+
* `finalize()`, once a run's own dispatch has fully settled — see its
|
|
41
|
+
* comment for why that timing is safe.
|
|
42
|
+
*/
|
|
43
|
+
close(): void;
|
|
34
44
|
/** Additive column migrations, so a db from an older SPF still opens. */
|
|
35
45
|
private migrate;
|
|
36
46
|
/**
|
package/dist/core/tracer.js
CHANGED
|
@@ -128,6 +128,18 @@ export class Tracer {
|
|
|
128
128
|
this.db.exec(SCHEMA);
|
|
129
129
|
this.migrate();
|
|
130
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* Close the sqlite handle. A one-shot CLI process never needs this — it
|
|
133
|
+
* exits right after its one Tracer anyway — but `spf watch`'s daemon loop
|
|
134
|
+
* builds a fresh Tracer (and a fresh `new Database(dbPath)`) per claimed
|
|
135
|
+
* issue, in-process, for the life of the daemon; without this, every
|
|
136
|
+
* issue's handle stayed open forever. Called from `session.ts`'s
|
|
137
|
+
* `finalize()`, once a run's own dispatch has fully settled — see its
|
|
138
|
+
* comment for why that timing is safe.
|
|
139
|
+
*/
|
|
140
|
+
close() {
|
|
141
|
+
this.db.close();
|
|
142
|
+
}
|
|
131
143
|
/** Additive column migrations, so a db from an older SPF still opens. */
|
|
132
144
|
migrate() {
|
|
133
145
|
for (const [table, column, decl] of MIGRATIONS) {
|