@gr8ful/spf 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +168 -29
  2. package/assets/defaults/spf.config.yaml +68 -0
  3. package/assets/prompts/refiner/system.md +42 -6
  4. package/assets/prompts/refiner/user.md +46 -8
  5. package/assets/skill/SKILL.md +1 -0
  6. package/assets/skill/references/config.md +192 -5
  7. package/assets/templates/ts-flue-ollama.spf.config.yaml +25 -0
  8. package/assets/templates/ts.spf.config.yaml +12 -2
  9. package/dist/chains/index.d.ts +11 -0
  10. package/dist/chains/index.js +38 -3
  11. package/dist/chains/repo_chains.js +1 -0
  12. package/dist/chains/simple_sdlc.js +1 -1
  13. package/dist/chains/steps.d.ts +8 -17
  14. package/dist/chains/steps.js +102 -14
  15. package/dist/cli/commands/doctor.js +112 -3
  16. package/dist/cli/commands/estimate.d.ts +82 -0
  17. package/dist/cli/commands/estimate.js +317 -0
  18. package/dist/cli/commands/fanout.d.ts +40 -0
  19. package/dist/cli/commands/fanout.js +401 -0
  20. package/dist/cli/commands/init.js +23 -1
  21. package/dist/cli/commands/run.js +9 -2
  22. package/dist/cli/commands/trace.d.ts +18 -0
  23. package/dist/cli/commands/trace.js +22 -3
  24. package/dist/cli/commands/watch.d.ts +8 -0
  25. package/dist/cli/commands/watch.js +95 -33
  26. package/dist/cli/index.js +13 -1
  27. package/dist/cli/interview.js +9 -5
  28. package/dist/core/agents.d.ts +56 -0
  29. package/dist/core/agents.js +152 -1
  30. package/dist/core/data_types.d.ts +340 -5
  31. package/dist/core/data_types.js +185 -5
  32. package/dist/core/fanout.d.ts +229 -0
  33. package/dist/core/fanout.js +313 -0
  34. package/dist/core/gates.d.ts +8 -0
  35. package/dist/core/gates.js +48 -2
  36. package/dist/core/git_helper.d.ts +98 -0
  37. package/dist/core/git_helper.js +127 -0
  38. package/dist/core/issues/github_provider.d.ts +46 -6
  39. package/dist/core/issues/github_provider.js +120 -5
  40. package/dist/core/issues/jira_provider.d.ts +83 -12
  41. package/dist/core/issues/jira_provider.js +110 -4
  42. package/dist/core/issues/provider.d.ts +124 -19
  43. package/dist/core/issues/provider.js +24 -7
  44. package/dist/core/notify/channel.d.ts +1 -1
  45. package/dist/core/notify/notifier.d.ts +16 -0
  46. package/dist/core/notify/notifier.js +36 -0
  47. package/dist/core/otel.d.ts +64 -9
  48. package/dist/core/otel.js +78 -14
  49. package/dist/core/refine.d.ts +45 -8
  50. package/dist/core/refine.js +98 -24
  51. package/dist/core/runner.d.ts +8 -0
  52. package/dist/core/runner.js +7 -0
  53. package/dist/core/session.d.ts +25 -0
  54. package/dist/core/session.js +97 -28
  55. package/dist/core/tiering.d.ts +145 -0
  56. package/dist/core/tiering.js +235 -0
  57. package/dist/core/tracer.d.ts +10 -0
  58. package/dist/core/tracer.js +12 -0
  59. package/dist/core/watch.d.ts +174 -11
  60. package/dist/core/watch.js +531 -33
  61. package/dist/ui/server/db.d.ts +39 -0
  62. package/dist/ui/server/db.js +61 -0
  63. package/package.json +2 -1
  64. package/dist/test/agent_cc.test.d.ts +0 -1
  65. package/dist/test/agent_cc.test.js +0 -95
  66. package/dist/test/agent_flue.test.d.ts +0 -1
  67. package/dist/test/agent_flue.test.js +0 -83
  68. package/dist/test/chains.test.d.ts +0 -12
  69. package/dist/test/chains.test.js +0 -92
  70. package/dist/test/data_types.test.d.ts +0 -10
  71. package/dist/test/data_types.test.js +0 -220
  72. package/dist/test/env_file.test.d.ts +0 -1
  73. package/dist/test/env_file.test.js +0 -74
  74. package/dist/test/fake_asker.d.ts +0 -23
  75. package/dist/test/fake_asker.js +0 -30
  76. package/dist/test/git_helper.test.d.ts +0 -1
  77. package/dist/test/git_helper.test.js +0 -59
  78. package/dist/test/hermetic_git.d.ts +0 -1
  79. package/dist/test/hermetic_git.js +0 -22
  80. package/dist/test/init_command.test.d.ts +0 -14
  81. package/dist/test/init_command.test.js +0 -136
  82. package/dist/test/interview.test.d.ts +0 -15
  83. package/dist/test/interview.test.js +0 -425
  84. package/dist/test/notify.test.d.ts +0 -1
  85. package/dist/test/notify.test.js +0 -174
  86. package/dist/test/ollama_provider.test.d.ts +0 -1
  87. package/dist/test/ollama_provider.test.js +0 -103
  88. package/dist/test/otel.test.d.ts +0 -26
  89. package/dist/test/otel.test.js +0 -512
  90. package/dist/test/paths.test.d.ts +0 -1
  91. package/dist/test/paths.test.js +0 -68
  92. package/dist/test/refine.test.d.ts +0 -1
  93. package/dist/test/refine.test.js +0 -189
  94. package/dist/test/repo_chains.test.d.ts +0 -21
  95. package/dist/test/repo_chains.test.js +0 -416
  96. package/dist/test/signoff.test.d.ts +0 -1
  97. package/dist/test/signoff.test.js +0 -329
  98. package/dist/test/ui_server.test.d.ts +0 -7
  99. package/dist/test/ui_server.test.js +0 -120
  100. package/dist/test/watch.test.d.ts +0 -1
  101. package/dist/test/watch.test.js +0 -687
@@ -19,6 +19,7 @@ import { isKnownToolName as isKnownFlueToolName, resolveModel } from "../../core
19
19
  import { ollamaBaseUrl } from "../../core/ollama_provider.js";
20
20
  import { binaryOnPath, parseCli } from "../../core/utils.js";
21
21
  import { PROVIDER_ENV_KEYS } from "../../core/providers.js";
22
+ import { probeServedOllamaTags, resolveTiering } from "../../core/tiering.js";
22
23
  import { isRepoAt } from "../../core/git_helper.js";
23
24
  import { allChains, findChain, repoChainProblems, resolveRequiredAgents, resolveRequiredSuites } from "../../chains/index.js";
24
25
  /**
@@ -265,7 +266,13 @@ export async function doctorCommand(argv) {
265
266
  // handled in the per-agent loop below) but DO have a server that might
266
267
  // simply not be running — worth a reachability probe the same way
267
268
  // ANTHROPIC_BASE_URL gets one above.
268
- const usesOllamaFlue = cfg.agents.some((a) => a.coding_agent !== "claude_code" && a.model.startsWith("ollama/"));
269
+ // Tiering (SPF #14): an ollama/* ladder rung reaches for the same server
270
+ // whether or not any cfg.agents[] entry itself names one — gated on
271
+ // tiering.enabled so a disabled (or never-opted-into) ladder can't turn on
272
+ // a probe nobody asked for. Without this, an all-hosted roster paired with
273
+ // an ollama ladder got NO reachability probe at all (integration point 18a).
274
+ const usesOllamaFlue = cfg.agents.some((a) => a.coding_agent !== "claude_code" && a.model.startsWith("ollama/")) ||
275
+ (cfg.tiering.enabled && cfg.tiering.tiers.some((t) => t.coding_agent !== "claude_code" && t.model.startsWith("ollama/")));
269
276
  if (usesOllamaFlue && !flags["no-probe"]) {
270
277
  // `ollamaBaseUrl()` (ollama_provider.ts) is the SAME default-substitution
271
278
  // logic `registerOllamaModel` uses for a real dispatch, including
@@ -318,6 +325,106 @@ export async function doctorCommand(argv) {
318
325
  check(report, `${label} tool "${toolName}"`, false, "not a known tool name");
319
326
  }
320
327
  }
328
+ // Tiering (SPF #14) — three checks, ALL gated on tiering.enabled: a
329
+ // disabled (or never-opted-into) ladder must be invisible to `spf doctor`,
330
+ // the same gate agents.validate() uses (see the "roster + suites
331
+ // validate" check above, which already fails loudly on a rule-T backend
332
+ // mismatch for the whole roster — nothing here duplicates that failure,
333
+ // this section only adds VISIBILITY on top of it). Without this gate,
334
+ // `spf init --template ts && spf doctor` would fail on three unset
335
+ // provider keys for a ladder that is off and will never dispatch
336
+ // (integration point 18's own regression note).
337
+ if (cfg.tiering.enabled) {
338
+ // 18(b): the SAME provider-key check the per-agent loop above just ran,
339
+ // over cfg.tiering.tiers[].model instead of cfg.agents[].model,
340
+ // branching on the TIER's OWN declared coding_agent (never an agent's —
341
+ // a tier changes `model` and nothing else, see core/tiering.ts's rule
342
+ // T). Without this, a hosted rung naming a provider whose key is unset
343
+ // is a green doctor and a first-dispatch failure.
344
+ for (const tier of cfg.tiering.tiers) {
345
+ const tierLabel = `tiering tier "${tier.name}"`;
346
+ if (tier.coding_agent === "claude_code") {
347
+ check(report, `${tierLabel} provider key`, true, process.env["ANTHROPIC_API_KEY"] ? "ANTHROPIC_API_KEY is set" : "ANTHROPIC_API_KEY not set — fine if authenticated via `claude login` instead");
348
+ }
349
+ else {
350
+ try {
351
+ const [provider] = resolveModel(tier.model);
352
+ const envKeys = PROVIDER_ENV_KEYS[provider];
353
+ if (!envKeys) {
354
+ check(report, `${tierLabel} provider key`, true, `provider "${provider}" not in doctor's known list — skipped, not a failure`);
355
+ }
356
+ else if (envKeys.length === 0) {
357
+ check(report, `${tierLabel} provider key`, true, `provider "${provider}" is keyless — no key required`);
358
+ }
359
+ else {
360
+ const set = envKeys.find((k) => process.env[k]);
361
+ check(report, `${tierLabel} provider key`, Boolean(set), set ? `${set} is set` : `none of ${envKeys.join(", ")} is set`);
362
+ }
363
+ }
364
+ catch (error) {
365
+ check(report, `${tierLabel} model`, false, error.message);
366
+ }
367
+ }
368
+ }
369
+ // 18(c): the resolved ladder, each routed role's effective model, and
370
+ // the served-tag report — imports probeServedOllamaTags FROM
371
+ // core/tiering.ts (not doctor's own probeGet, which discards the
372
+ // response body that's the only part that matters here).
373
+ //
374
+ // chainName/prompt are deliberately "" — an unlisted chain name weighs
375
+ // 0 and a zero-word prompt weighs -1, summing to -1, which classifies
376
+ // as "standard" (short of the -2 "low" needs). That reports the
377
+ // LADDER'S OWN baseline resolution (step 0 — no risk shift), not any
378
+ // one chain's risk-shifted view; a risk-shifted projection for a real
379
+ // chain+prompt is `spf estimate`'s job, not doctor's. `required` is the
380
+ // WHOLE roster, matching the "roster + suites validate" call above —
381
+ // every routed role doctor's job covers, not just what one chain needs.
382
+ const servedOllamaTags = flags["no-probe"] ? null : await probeServedOllamaTags(cfg);
383
+ const resolution = resolveTiering({
384
+ cfg,
385
+ chainName: "",
386
+ prompt: "",
387
+ servedOllamaTags,
388
+ required: cfg.agents.map((a) => a.name),
389
+ });
390
+ check(report, "tiering ladder", true, cfg.tiering.tiers.length > 0 ? cfg.tiering.tiers.map((t) => `${t.name}(${t.coding_agent}:${t.model})`).join(" -> ") : "(no tiers declared)", "info");
391
+ // FULL routing map — an entry for every routed role including the ones
392
+ // whose effective model equals their configured one (integration point
393
+ // 18c: "roles whose effective equals their configured model" still
394
+ // print), never a diff.
395
+ for (const [agentName, route] of Object.entries(resolution.routing)) {
396
+ check(report, `tiering role "${agentName}"`, true, `-> tier "${route.tier}" (${route.effective})${route.configured === route.effective ? " [unchanged from configured]" : ` (was ${route.configured})`}`, "info");
397
+ }
398
+ // One line per note resolveTiering produced (an unserved tag, a rule-T
399
+ // backend mismatch, an unknown tier name, ...) — resolveTiering never
400
+ // assigns severity (agents.validate() owns that; this call only owns
401
+ // detection), so every note here is informational: the hard failure for
402
+ // a rule-T mismatch or a malformed rung already came from the "roster +
403
+ // suites validate" check above, on the SAME resolveTiering-adjacent
404
+ // detection agents.validate() runs. This just names it again, in context.
405
+ for (const note of resolution.notes) {
406
+ check(report, "tiering note", true, note, "warn");
407
+ }
408
+ // Served-tag report, fail-open in the SAME direction the probe itself
409
+ // does: a stopped/unreachable server reports "could not check" once,
410
+ // never per-rung "not served" — flagging every rung as unserved on a
411
+ // probe failure would look identical to the prefix-strip bug
412
+ // core/tiering.ts's usable() guards against, and is exactly the outcome
413
+ // fail-open exists to prevent.
414
+ const ollamaTiers = cfg.tiering.tiers.filter((t) => t.coding_agent !== "claude_code" && t.model.startsWith("ollama/"));
415
+ if (ollamaTiers.length > 0) {
416
+ if (servedOllamaTags === null) {
417
+ check(report, "tiering served-tag probe", true, flags["no-probe"] ? "skipped (--no-probe)" : "could not check — Ollama server unreachable (fail-open: nothing is dropped this run)", "warn");
418
+ }
419
+ else {
420
+ for (const tier of ollamaTiers) {
421
+ const tag = tier.model.slice("ollama/".length);
422
+ const served = servedOllamaTags.has(tag);
423
+ check(report, `tiering tier "${tier.name}" served`, true, served ? `"${tag}" is in \`ollama list\`` : `"${tag}" is NOT in \`ollama list\` — a run will walk down to a lower usable rung`, served ? "info" : "warn");
424
+ }
425
+ }
426
+ }
427
+ }
321
428
  for (const spec of cfg.quality.checks) {
322
429
  check(report, `quality check "${spec.name}"`, binaryOnPath(spec.argv[0]), `${spec.argv[0]} ${binaryOnPath(spec.argv[0]) ? "found on PATH" : "NOT found on PATH"}`);
323
430
  }
@@ -427,9 +534,11 @@ export async function doctorCommand(argv) {
427
534
  if (cfg.watch.refine.enabled) {
428
535
  const refineChain = findChain(cfg.watch.refine.chain);
429
536
  check(report, "watch.refine.chain", Boolean(refineChain), refineChain ? `${cfg.watch.refine.chain}${refineChain.source ? ` (repo: ${repoChainLabel(refineChain.source)})` : ""}` : `"${cfg.watch.refine.chain}" is not a registered chain`);
430
- check(report, "watch.refine issue authoring", cfg.watch.issue_provider === "github", cfg.watch.issue_provider === "github"
537
+ check(report, "watch.refine issue authoring", cfg.watch.issue_provider === "github" || cfg.watch.issue_provider === "jira", cfg.watch.issue_provider === "github"
431
538
  ? "github supports issue authoring (createIssue/sub-issues)"
432
- : `watch.issue_provider is ${JSON.stringify(cfg.watch.issue_provider)} the refine lane needs "github" (Jira issue authoring isn't implemented yet)`);
539
+ : cfg.watch.issue_provider === "jira"
540
+ ? "jira supports issue authoring (createIssue/parent field) — run `spf watch init` to validate watch.jira.issue_types against the real project"
541
+ : `watch.issue_provider is ${JSON.stringify(cfg.watch.issue_provider)} — the refine lane needs "github" or "jira"`);
433
542
  }
434
543
  }
435
544
  // OTel span export: informational in every direction. It is off unless
@@ -0,0 +1,82 @@
1
+ import { type TierResolution } from "../../core/tiering.ts";
2
+ import type { ChainHistorySession } from "../../ui/server/db.ts";
3
+ export declare function usage(): string;
4
+ export type SampleStatus = "success" | "any" | "none";
5
+ export interface Sample {
6
+ sessions: ChainHistorySession[];
7
+ status: SampleStatus;
8
+ fanoutCollapsed: number;
9
+ joinedExcluded: number;
10
+ }
11
+ /**
12
+ * Two tiers, stopping at the first that qualifies, and an honest refusal.
13
+ * `sessions` is already most-recent-first (db.ts's own ORDER BY).
14
+ */
15
+ export declare function selectSample(sessions: ChainHistorySession[], joinedExcluded: number): Sample;
16
+ /** p50, no interpolation — a 3-sample percentile is false precision (design doc §5.3). Even-length arrays average the two middle values, the ordinary definition. */
17
+ export declare function median(values: number[]): number;
18
+ /** Strip a loop factory's trailing `_<digits>` (`steps.ts:572-579,597-600,670,683`) — `fix_1`/`fix_2` -> `fix`, `test_1` -> `test`. Reads the recorded name; never predicts it. */
19
+ export declare function normalizePhaseName(name: string): string;
20
+ export interface PhaseProjection {
21
+ name: string;
22
+ p50: number;
23
+ min: number;
24
+ max: number;
25
+ }
26
+ /**
27
+ * Per run: sum every iteration of a normalized phase into one observation.
28
+ * Across runs: median (p50) and min/max of those per-run sums. A phase
29
+ * absent from a given run (a fix loop that never triggered) counts as a
30
+ * `0` observation for that run — not a skip — so its median reflects how
31
+ * often the phase actually ran, not just its size when it did.
32
+ */
33
+ export declare function aggregatePhases(sessions: ChainHistorySession[]): PhaseProjection[];
34
+ export interface RunTotalsProjection {
35
+ p50: number;
36
+ min: number;
37
+ max: number;
38
+ }
39
+ /** The HEADLINE: median of sampled runs' `sessions.total_tokens` — NOT the sum of the per-phase medians (those are different numbers; median of sums != sum of medians). */
40
+ export declare function projectRunTotals(sessions: ChainHistorySession[]): RunTotalsProjection | null;
41
+ /** Where a p50 run would be cut off by `max_run_tokens`, walking the phase breakdown cumulatively in the order phases first appeared. `undefined` means unset; `null` means set but never reached on the p50 path. */
42
+ export declare function findCutoffPhase(phases: PhaseProjection[], maxRunTokens: number | undefined): string | null | undefined;
43
+ export interface DriftWarning {
44
+ agent: string;
45
+ historical: string;
46
+ current: string;
47
+ }
48
+ /**
49
+ * Compares each sampled agent's HISTORICAL dispatch model against the
50
+ * resolution's EFFECTIVE model — `res.routing[agent]?.effective ?? agent's
51
+ * own configured model` — never against `roles`/`tiers` in isolation. That
52
+ * is what makes the warning fire correctly whether or not tiering is
53
+ * enabled: with tiering off, "effective" is just the agent's own `model:`,
54
+ * so an operator who edited the roster directly between runs (no tiering
55
+ * involved) still gets an accurate warning.
56
+ */
57
+ export declare function detectModelDrift(historicalModels: Map<string, string>, currentModel: (agent: string) => string | undefined): DriftWarning[];
58
+ export interface EstimateReport {
59
+ chain: string;
60
+ risk: TierResolution["risk"];
61
+ signals: TierResolution["signals"];
62
+ routing: TierResolution["routing"];
63
+ notes: string[];
64
+ sample: {
65
+ n: number;
66
+ status: SampleStatus;
67
+ joined_excluded: number;
68
+ fanout_collapsed: number;
69
+ };
70
+ phases: PhaseProjection[];
71
+ projected: RunTotalsProjection | null;
72
+ observed_cost: number;
73
+ drift: DriftWarning[];
74
+ validate_error: string | null;
75
+ ceilings: {
76
+ max_run_tokens: number | undefined;
77
+ max_run_cost: number | undefined;
78
+ cutoff_phase: string | null | undefined;
79
+ };
80
+ fanout_n: number;
81
+ }
82
+ export declare function estimateCommand(argv: string[]): Promise<number>;
@@ -0,0 +1,317 @@
1
+ /**
2
+ * `spf estimate <chain> "<prompt>"` — a read-only projection of what a chain
3
+ * WOULD cost and WHICH models it would dispatch on, built only from real
4
+ * trace rows already on disk. Never mints an `adw_id`, never opens a
5
+ * session, never writes a row (design doc §5).
6
+ *
7
+ * Two questions, one command:
8
+ * - "what will this run on?" — the SAME `resolveTiering` (`core/tiering.ts`)
9
+ * a real run calls from `startRun`, so the printed routing cannot drift
10
+ * from what dispatch does.
11
+ * - "what will this cost?" — projected from real `sessions`/`phases`/
12
+ * `agent_end` rows for this exact chain name (never a joined session's),
13
+ * reported in TOKENS (never a fabricated per-token price table — see
14
+ * §5.3 for why cost is `$0.00` for exactly the rosters this feature is
15
+ * for) with an honest refusal — exit 3, no number — when there is
16
+ * nothing to project from.
17
+ */
18
+ import * as agents from "../../core/agents.js";
19
+ import { parseCli, resolvePrompt } from "../../core/utils.js";
20
+ import { findChain, resolveRequiredAgents, resolveRequiredSuites } from "../../chains/index.js";
21
+ import { probeServedOllamaTags, resolveTiering } from "../../core/tiering.js";
22
+ import { openTraceIfExists } from "./trace.js";
23
+ const KNOWN_OPTIONS = ["config", "cwd", "agent", "n"];
24
+ const KNOWN_FLAGS = ["json", "no-probe"];
25
+ const SAMPLE_CAP = 20;
26
+ export function usage() {
27
+ return 'usage: spf estimate <chain> "<prompt or path/to/prompt.md>" [--n N] [--json] [--config <path>] [--cwd <dir>] [--no-probe] [--agent <name>]';
28
+ }
29
+ // ── fanout-sibling recognition (design doc §5.2) ────────────────────────────
30
+ /**
31
+ * The base an attempt was derived from, per `fanout.ts`'s own naming scheme
32
+ * (`attemptAdwId`: `${baseAdwId}-${index}`, `fanout.ts:129-131`) — a
33
+ * heuristic, not a foreign key (there is no column recording it), which is
34
+ * why the design doc calls the base "recoverable from the suffix scheme"
35
+ * rather than "stored".
36
+ *
37
+ * The base itself is always `newId(8)` (`fanout.ts:277`) — 8 lowercase hex
38
+ * characters, never containing `-` — so the match requires exactly that
39
+ * shape rather than "any dash then digits". A bare `/^(.+)-\d+$/` is far too
40
+ * loose: `spf watch` mints `adw_id`s like `issue-41`/`issue-42`/`issue-43`
41
+ * (`watch.ts:357,469`, `Issue.id` a decimal string), and every one of those
42
+ * ends in `-<digits>` despite never having gone through `spf fanout` — the
43
+ * loose regex collapsed 41/42/43 into one shared "issue" base. Requiring the
44
+ * base to look like `newId(8)`'s actual output means `issue-41` doesn't
45
+ * match at all (`issue` isn't 8 hex chars) and is correctly treated as its
46
+ * own independent run.
47
+ */
48
+ function fanoutBaseOf(adwId) {
49
+ const match = /^([0-9a-f]{8})-\d+$/.exec(adwId);
50
+ return match ? match[1] : adwId;
51
+ }
52
+ /** At most one attempt per fanout base, most-recent kept (input is already recency-ordered). Does not cap; the caller decides how many of the result to keep. */
53
+ function collapseFanoutSiblings(sessions) {
54
+ const seenBases = new Set();
55
+ const deduped = [];
56
+ let collapsed = 0;
57
+ for (const session of sessions) {
58
+ const base = fanoutBaseOf(session.adw_id);
59
+ if (seenBases.has(base)) {
60
+ collapsed++;
61
+ continue;
62
+ }
63
+ seenBases.add(base);
64
+ deduped.push(session);
65
+ }
66
+ return { deduped, collapsed };
67
+ }
68
+ /**
69
+ * Two tiers, stopping at the first that qualifies, and an honest refusal.
70
+ * `sessions` is already most-recent-first (db.ts's own ORDER BY).
71
+ */
72
+ export function selectSample(sessions, joinedExcluded) {
73
+ const successes = sessions.filter((s) => s.status === "success");
74
+ const { deduped: successDeduped, collapsed: successCollapsed } = collapseFanoutSiblings(successes);
75
+ if (successDeduped.length >= 3) {
76
+ return { sessions: successDeduped.slice(0, SAMPLE_CAP), status: "success", fanoutCollapsed: successCollapsed, joinedExcluded };
77
+ }
78
+ const { deduped: anyDeduped, collapsed: anyCollapsed } = collapseFanoutSiblings(sessions);
79
+ if (anyDeduped.length >= 1) {
80
+ return { sessions: anyDeduped.slice(0, SAMPLE_CAP), status: "any", fanoutCollapsed: anyCollapsed, joinedExcluded };
81
+ }
82
+ return { sessions: [], status: "none", fanoutCollapsed: 0, joinedExcluded };
83
+ }
84
+ // ── projection (design doc §5.3) ────────────────────────────────────────────
85
+ /** p50, no interpolation — a 3-sample percentile is false precision (design doc §5.3). Even-length arrays average the two middle values, the ordinary definition. */
86
+ export function median(values) {
87
+ if (values.length === 0)
88
+ return 0;
89
+ const sorted = [...values].sort((a, b) => a - b);
90
+ const mid = Math.floor(sorted.length / 2);
91
+ return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
92
+ }
93
+ /** Strip a loop factory's trailing `_<digits>` (`steps.ts:572-579,597-600,670,683`) — `fix_1`/`fix_2` -> `fix`, `test_1` -> `test`. Reads the recorded name; never predicts it. */
94
+ export function normalizePhaseName(name) {
95
+ return name.replace(/_\d+$/, "");
96
+ }
97
+ /**
98
+ * Per run: sum every iteration of a normalized phase into one observation.
99
+ * Across runs: median (p50) and min/max of those per-run sums. A phase
100
+ * absent from a given run (a fix loop that never triggered) counts as a
101
+ * `0` observation for that run — not a skip — so its median reflects how
102
+ * often the phase actually ran, not just its size when it did.
103
+ */
104
+ export function aggregatePhases(sessions) {
105
+ const perRunSums = sessions.map(() => new Map());
106
+ const firstSeq = new Map();
107
+ sessions.forEach((session, i) => {
108
+ for (const phase of session.phases) {
109
+ const name = normalizePhaseName(phase.name);
110
+ perRunSums[i].set(name, (perRunSums[i].get(name) ?? 0) + phase.tokens);
111
+ if (!firstSeq.has(name) || phase.seq < firstSeq.get(name))
112
+ firstSeq.set(name, phase.seq);
113
+ }
114
+ });
115
+ const names = [...firstSeq.keys()].sort((a, b) => firstSeq.get(a) - firstSeq.get(b));
116
+ return names.map((name) => {
117
+ const perRun = perRunSums.map((m) => m.get(name) ?? 0);
118
+ return { name, p50: median(perRun), min: Math.min(...perRun), max: Math.max(...perRun) };
119
+ });
120
+ }
121
+ /** The HEADLINE: median of sampled runs' `sessions.total_tokens` — NOT the sum of the per-phase medians (those are different numbers; median of sums != sum of medians). */
122
+ export function projectRunTotals(sessions) {
123
+ if (sessions.length === 0)
124
+ return null;
125
+ const totals = sessions.map((s) => s.total_tokens);
126
+ return { p50: median(totals), min: Math.min(...totals), max: Math.max(...totals) };
127
+ }
128
+ /** Where a p50 run would be cut off by `max_run_tokens`, walking the phase breakdown cumulatively in the order phases first appeared. `undefined` means unset; `null` means set but never reached on the p50 path. */
129
+ export function findCutoffPhase(phases, maxRunTokens) {
130
+ if (maxRunTokens === undefined)
131
+ return undefined;
132
+ let cumulative = 0;
133
+ for (const phase of phases) {
134
+ cumulative += phase.p50;
135
+ if (cumulative >= maxRunTokens)
136
+ return phase.name;
137
+ }
138
+ return null;
139
+ }
140
+ /**
141
+ * Compares each sampled agent's HISTORICAL dispatch model against the
142
+ * resolution's EFFECTIVE model — `res.routing[agent]?.effective ?? agent's
143
+ * own configured model` — never against `roles`/`tiers` in isolation. That
144
+ * is what makes the warning fire correctly whether or not tiering is
145
+ * enabled: with tiering off, "effective" is just the agent's own `model:`,
146
+ * so an operator who edited the roster directly between runs (no tiering
147
+ * involved) still gets an accurate warning.
148
+ */
149
+ export function detectModelDrift(historicalModels, currentModel) {
150
+ const warnings = [];
151
+ for (const [agent, historical] of historicalModels) {
152
+ const current = currentModel(agent);
153
+ if (current !== undefined && current !== historical)
154
+ warnings.push({ agent, historical, current });
155
+ }
156
+ return warnings;
157
+ }
158
+ export async function estimateCommand(argv) {
159
+ const { positionals, options, flags } = parseCli(argv, KNOWN_OPTIONS, KNOWN_FLAGS);
160
+ if (positionals.length < 2) {
161
+ console.error(usage());
162
+ return 1;
163
+ }
164
+ const [chainName, promptArg] = positionals;
165
+ const chain = findChain(chainName);
166
+ if (!chain) {
167
+ console.error(`unknown chain: ${chainName} — run \`spf list\` to see every chain`);
168
+ return 1;
169
+ }
170
+ const n = options["n"] !== undefined ? Number.parseInt(options["n"], 10) : 1;
171
+ if (!Number.isInteger(n) || n < 1) {
172
+ console.error(`--n must be a positive integer, got ${JSON.stringify(options["n"])}`);
173
+ return 1;
174
+ }
175
+ const prompt = resolvePrompt(promptArg);
176
+ const trace = openTraceIfExists(options);
177
+ const cfg = trace.cfg;
178
+ // Same shape as run.ts:54-56 — conditional insertion, never `{agent: flags["agent"] ?? ""}`.
179
+ // `flags` never carries a value-taking option, and an empty-string sentinel
180
+ // would defeat steps.ts:492's own `options["agent"] ?? "builder"` default.
181
+ const chainOptions = {};
182
+ if (options["agent"] !== undefined)
183
+ chainOptions["agent"] = options["agent"];
184
+ const required = resolveRequiredAgents(chain, chainOptions);
185
+ const requiredSuites = resolveRequiredSuites(chain, chainOptions);
186
+ const servedOllamaTags = flags["no-probe"] ? null : await probeServedOllamaTags(cfg);
187
+ const res = resolveTiering({ cfg, chainName: chain.name, prompt, servedOllamaTags, required });
188
+ // Same detector `validate()` itself runs at every real `startRun` — never
189
+ // thrown here, only reported: `estimate` gates nothing (exit 3 is reserved
190
+ // for "no history"), it just tells the operator a real run would fail.
191
+ let validateError = null;
192
+ try {
193
+ agents.validate(cfg, required, requiredSuites, trace.anchor.cwd);
194
+ }
195
+ catch (error) {
196
+ validateError = error instanceof Error ? error.message : String(error);
197
+ }
198
+ const history = trace.db ? trace.db.chainPhaseHistory(chain.name) : { sessions: [], joinedExcluded: 0 };
199
+ const sample = selectSample(history.sessions, history.joinedExcluded);
200
+ const coldStart = sample.sessions.length === 0;
201
+ const phases = aggregatePhases(sample.sessions);
202
+ const projectedRaw = projectRunTotals(sample.sessions);
203
+ const projected = projectedRaw && n > 1 ? { p50: projectedRaw.p50 * n, min: projectedRaw.min * n, max: projectedRaw.max * n } : projectedRaw;
204
+ const observedCost = sample.sessions.length > 0 ? median(sample.sessions.map((s) => s.total_cost)) * n : 0;
205
+ const historicalModels = new Map();
206
+ if (trace.db) {
207
+ for (const session of sample.sessions) {
208
+ for (const row of trace.db.agentSessions(session.adw_id)) {
209
+ if (row.model !== null && !historicalModels.has(row.agent))
210
+ historicalModels.set(row.agent, row.model);
211
+ }
212
+ }
213
+ }
214
+ const currentModel = (agent) => res.routing[agent]?.effective ?? cfg.agents.find((a) => a.name === agent)?.model;
215
+ const drift = detectModelDrift(historicalModels, currentModel);
216
+ const maxTokens = cfg.defaults.max_run_tokens;
217
+ const maxCost = cfg.defaults.max_run_cost;
218
+ const cutoffPhase = findCutoffPhase(phases, maxTokens);
219
+ const report = {
220
+ chain: chain.name,
221
+ risk: res.risk,
222
+ signals: res.signals,
223
+ routing: res.routing,
224
+ notes: res.notes,
225
+ sample: { n: sample.sessions.length, status: sample.status, joined_excluded: sample.joinedExcluded, fanout_collapsed: sample.fanoutCollapsed },
226
+ phases,
227
+ projected,
228
+ observed_cost: observedCost,
229
+ drift,
230
+ validate_error: validateError,
231
+ ceilings: { max_run_tokens: maxTokens, max_run_cost: maxCost, cutoff_phase: cutoffPhase },
232
+ fanout_n: n,
233
+ };
234
+ if (flags["json"]) {
235
+ console.log(JSON.stringify(report, null, 2));
236
+ }
237
+ else {
238
+ printText(report, coldStart);
239
+ }
240
+ return coldStart ? 3 : 0;
241
+ }
242
+ function fmt(n) {
243
+ return Math.round(n).toLocaleString("en-US");
244
+ }
245
+ function printText(report, coldStart) {
246
+ const lines = [];
247
+ lines.push(`chain ${report.chain}`);
248
+ lines.push(`tier ${report.risk} (chain ${report.signals.chain} = ${report.signals.chain_weight}, ` +
249
+ `prompt ${report.signals.prompt_words}w = ${report.signals.prompt_weight}, sum ${report.signals.sum})`);
250
+ const routingEntries = Object.entries(report.routing);
251
+ if (routingEntries.length === 0) {
252
+ lines.push("routing (tiering.enabled is false, or nothing in this chain's roster is routed by tiering.roles)");
253
+ }
254
+ else {
255
+ for (const [agent, route] of routingEntries) {
256
+ const same = route.configured === route.effective;
257
+ lines.push(`routing ${agent} -> ${route.tier} tier (${route.effective}) ${same ? "[= configured]" : `[was ${route.configured}]`}`);
258
+ }
259
+ }
260
+ for (const note of report.notes)
261
+ lines.push(`note ${note}`);
262
+ if (report.validate_error) {
263
+ lines.push(`note a real run of this config would fail agents.validate() at start: ${report.validate_error}`);
264
+ }
265
+ if (coldStart) {
266
+ lines.push("");
267
+ lines.push(`sample no trace history for chain "${report.chain}" yet — run \`spf ${report.chain} "..."\` once; ` +
268
+ "this projects from real trace rows only");
269
+ if (report.sample.joined_excluded > 0) {
270
+ lines.push(` (${report.sample.joined_excluded} joined session(s) named this chain but cannot be attributed to it alone — excluded)`);
271
+ }
272
+ }
273
+ else {
274
+ const label = report.sample.status === "success" ? "successful" : "run(s), including failed/partial";
275
+ lines.push("");
276
+ lines.push(`sample ${report.sample.n} ${label} of "${report.chain}"`);
277
+ const extras = [];
278
+ if (report.sample.joined_excluded > 0)
279
+ extras.push(`${report.sample.joined_excluded} joined session(s) excluded`);
280
+ if (report.sample.fanout_collapsed > 0)
281
+ extras.push(`${report.sample.fanout_collapsed} fanout sibling(s) collapsed`);
282
+ if (report.sample.status === "any")
283
+ extras.push("a truncated run under-reports later phases");
284
+ if (extras.length > 0)
285
+ lines.push(` (${extras.join("; ")})`);
286
+ if (report.phases.length > 0) {
287
+ lines.push("");
288
+ lines.push("phase p50 tokens range");
289
+ for (const phase of report.phases) {
290
+ lines.push(`${phase.name.padEnd(20)} ${fmt(phase.p50).padStart(12)} ${fmt(phase.min)} - ${fmt(phase.max)}`);
291
+ }
292
+ }
293
+ if (report.projected) {
294
+ lines.push("");
295
+ lines.push(`projected ${fmt(report.projected.p50).padStart(12)} ${fmt(report.projected.min)} - ${fmt(report.projected.max)} ` +
296
+ "(median of RUN TOTALS, not the sum of the per-phase p50 column)");
297
+ if (report.fanout_n > 1) {
298
+ lines.push(` x${report.fanout_n} (spf fanout) already applied above`);
299
+ }
300
+ }
301
+ lines.push(`observed cost $${report.observed_cost.toFixed(3)} (cost is reported as observed in the sample, never modelled — $0.00 for a local roster)`);
302
+ for (const w of report.drift) {
303
+ lines.push(`drift ${w.agent}: sample ran ${w.historical}, current tiering picks ${w.current} — ` +
304
+ "token counts are not comparable across tokenizers; treat this projection as an order of magnitude, never rescaled");
305
+ }
306
+ }
307
+ lines.push("");
308
+ if (report.ceilings.max_run_tokens !== undefined) {
309
+ lines.push(`max_run_tokens ${fmt(report.ceilings.max_run_tokens)}` +
310
+ (report.ceilings.cutoff_phase ? ` -> would stop at "${report.ceilings.cutoff_phase}" on a p50 run` : ""));
311
+ lines.push(' (checked BEFORE each call: a run can overshoot by one call, and a one-dispatch chain can never trip it)');
312
+ }
313
+ if (report.ceilings.max_run_cost !== undefined) {
314
+ lines.push(`max_run_cost $${report.ceilings.max_run_cost.toFixed(3)}`);
315
+ }
316
+ console.log(lines.join("\n"));
317
+ }
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Wire `<worktreePath>/.spf/data` to the main repo's `dataDir` — same
3
+ * purpose as `cli/commands/watch.ts`'s own `linkDataDir`: without it an
4
+ * attempt's chain resolves its session/trace data relative to `cwd` (its
5
+ * worktree) and writes into a fresh `.spf/data` that dies with the
6
+ * worktree — invisible in `spf ui`, and gone before `readMetrics` could even
7
+ * read the gate rows that DECIDE the winner. Symlinking it to the main
8
+ * repo's data dir is what makes the shared WAL SQLite the one place all N
9
+ * attempts report to (`tracer.ts`'s WAL + `busy_timeout=5000` is exactly
10
+ * what concurrent writers through this symlink already rely on under `spf
11
+ * watch`). Idempotent. A plain function of its two inputs (not a closure
12
+ * over `dataPaths`) so it can be exercised directly in a test, against a
13
+ * real filesystem and a real git worktree, without a real chain or agent.
14
+ *
15
+ * `mkdirSync(dataDir, ...)` BEFORE the symlink is not optional: a fresh
16
+ * clone with `.spf/` committed (`.spf/data` gitignored), or a repo straight
17
+ * out of `spf init` (which seeds `.spf/` and `.spf/chains/` but never
18
+ * `.spf/data` — nothing has traced a run there yet) has no `.spf/data` to
19
+ * point at. Without this, the symlink is dangling and every attempt's `Run`
20
+ * constructor fails its own `ensureDir` on first use — `spf fanout` would be
21
+ * broken on a repo's first ever run.
22
+ */
23
+ export declare function linkFanoutDataDir(worktreePath: string, dataDir: string): void;
24
+ /**
25
+ * Keep the `.spf/data` symlink invisible to `git status`/`git add -A` in the
26
+ * attempt's own worktree. `.spf/data/` — a trailing-slash DIRECTORY pattern,
27
+ * the shape this repo's own gitignore uses — does NOT match a SYMLINK of
28
+ * the same name, so without this, `git status` shows `?? .spf/data` and a
29
+ * committing chain's `git_helper.commitAll` (`git add -A`) stages it.
30
+ * Following the printed `git merge <winner-branch>` instruction then
31
+ * fast-forwards the symlink straight into the MAIN repo, replacing its real
32
+ * `.spf/data` directory with a symlink pointing back at itself — destroying
33
+ * the trace db (`ls .spf/data` becomes `ELOOP`). `info/exclude` is
34
+ * per-worktree, local-only, and idempotent to append to — the opposite of
35
+ * adding a pattern to a tracked `.gitignore`, which would ship this
36
+ * workaround into every clone for a symlink only `spf fanout` itself ever
37
+ * creates.
38
+ */
39
+ export declare function excludeSpfDataFromGit(worktreePath: string): void;
40
+ export declare function fanoutCommand(argv: string[]): Promise<number>;