forge-orkes 0.64.5 → 0.68.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/create-forge.js +49 -27
- package/package.json +1 -1
- package/template/.claude/hooks/forge-model-outcome.sh +23 -6
- package/template/.claude/hooks/forge-slice-runner.sh +117 -13
- package/template/.claude/hooks/tests/forge-model-outcome.test.sh +24 -0
- package/template/.claude/settings.json +6 -1
- package/template/.claude/skills/planning/SKILL.md +1 -0
- package/template/.claude/skills/upgrading/SKILL.md +1 -1
- package/template/.forge/checks/forge-jarvis-dispatch.sh +16 -8
- package/template/.forge/checks/forge-jarvis-instruments.sh +67 -0
- package/template/.forge/checks/tests/forge-jarvis-dispatch.test.sh +35 -0
- package/template/.forge/checks/tests/forge-jarvis-instruments.test.sh +23 -0
- package/template/.forge/templates/plan.md +5 -0
- package/template/.forge/templates/slice-runner/config.yml +22 -13
package/bin/create-forge.js
CHANGED
|
@@ -52,31 +52,22 @@ const FORGE_END = '<!-- forge:end -->';
|
|
|
52
52
|
|
|
53
53
|
// --- File classification for upgrades ---
|
|
54
54
|
|
|
55
|
-
// Framework-owned: Forge controls these entirely
|
|
55
|
+
// Framework-owned: Forge controls these entirely. Auto-clean removes files that
|
|
56
|
+
// dropped out of the template — but ownership is decided per top-level UNIT
|
|
57
|
+
// (a skill DIR under .claude/skills/, an agent FILE under .claude/agents/), and
|
|
58
|
+
// only the template defines which units Forge shipped. A unit the base template
|
|
59
|
+
// never shipped — user-authored, vendored-parity, or an opt-in experimental
|
|
60
|
+
// skill installed separately (e.g. m10's orchestrating, the convention packs) —
|
|
61
|
+
// is NOT framework content and is preserved, never swept, even though it lives
|
|
62
|
+
// under a framework-owned dir. Deleting it would destroy work Forge never owned,
|
|
63
|
+
// and .claude/ is frequently gitignored, so the loss is unrecoverable. Files
|
|
64
|
+
// removed from WITHIN a still-shipped unit (e.g. a renamed helper inside a
|
|
65
|
+
// template skill dir) stay cleanable — their unit is still present in the
|
|
66
|
+
// template. See upgradeDir(). Experimental skills are additionally diffed
|
|
67
|
+
// against their shipped sources by detectExperimentalStaleness() (report-only;
|
|
68
|
+
// re-sync consent lives in the /upgrading skill layer).
|
|
56
69
|
const FRAMEWORK_OWNED_DIRS = ['.claude/agents', '.claude/skills'];
|
|
57
70
|
|
|
58
|
-
// Experimental / opt-in skills installed separately (e.g. from experimental/m10).
|
|
59
|
-
// They are NOT in the shipped base template, so the framework-owned auto-clean
|
|
60
|
-
// would otherwise delete them on upgrade. Preserve them instead.
|
|
61
|
-
//
|
|
62
|
-
// The npm tarball ships experimental/ alongside bin/ + template/, so this route
|
|
63
|
-
// CAN diff an installed experimental skill against its same-tarball source:
|
|
64
|
-
// detectExperimentalStaleness() reports unchanged|stale with the differing files.
|
|
65
|
-
// (Blind preservation is what silently froze these pre-0.33.0 — issue #5: a base
|
|
66
|
-
// migration that changed experimental-skill behavior became a no-op.) Detection
|
|
67
|
-
// only — consent for a re-sync lives in the /upgrading skill layer, and
|
|
68
|
-
// load-bearing conventions are still promoted to base (FORGE.md) rather than
|
|
69
|
-
// left to a frozen skill. This preserve-list gates auto-clean only; staleness
|
|
70
|
-
// discovery is by directory scan of the shipped experimental/ tree.
|
|
71
|
-
const EXPERIMENTAL_SKILL_PATHS = ['.claude/skills/orchestrating'];
|
|
72
|
-
|
|
73
|
-
function isExperimentalSkillPath(displayPath) {
|
|
74
|
-
const norm = displayPath.split(path.sep).join('/');
|
|
75
|
-
return EXPERIMENTAL_SKILL_PATHS.some(
|
|
76
|
-
(p) => norm === p || norm.startsWith(p + '/')
|
|
77
|
-
);
|
|
78
|
-
}
|
|
79
|
-
|
|
80
71
|
// Compare dotted numeric versions (e.g. "0.19.1"). Returns 1 if a>b, -1 if a<b, 0 if equal.
|
|
81
72
|
function compareVersions(a, b) {
|
|
82
73
|
const pa = String(a).split('.').map((n) => parseInt(n, 10) || 0);
|
|
@@ -259,9 +250,15 @@ function upgradeDir(relDir, { autoClean = false } = {}) {
|
|
|
259
250
|
const srcPath = path.join(srcDir, rel);
|
|
260
251
|
if (!fs.existsSync(srcPath)) {
|
|
261
252
|
const displayPath = path.join(relDir, rel);
|
|
262
|
-
//
|
|
263
|
-
|
|
264
|
-
|
|
253
|
+
// Ownership is per top-level unit: the first path segment of `rel` is the
|
|
254
|
+
// skill dir (or, for flat agents, the file itself). If the template no
|
|
255
|
+
// longer ships that unit at all, the file is user-authored, vendored, or an
|
|
256
|
+
// opt-in experimental skill — not Forge's to delete. Preserve it. Only
|
|
257
|
+
// files whose unit the template STILL ships (a stale file inside a shipped
|
|
258
|
+
// skill dir) remain removal candidates.
|
|
259
|
+
const topUnit = rel.split(path.sep)[0];
|
|
260
|
+
if (autoClean && !fs.existsSync(path.join(srcDir, topUnit))) {
|
|
261
|
+
result.preserved.push(`${displayPath} (kept — not shipped by the base template)`);
|
|
265
262
|
continue;
|
|
266
263
|
}
|
|
267
264
|
const destPath = path.join(destDir, rel);
|
|
@@ -374,6 +371,30 @@ function mergeManagedDeny(destSettings, srcSettings) {
|
|
|
374
371
|
}
|
|
375
372
|
}
|
|
376
373
|
|
|
374
|
+
/**
|
|
375
|
+
* Union the template's managed allow rules into the project's permissions.allow.
|
|
376
|
+
* Additive — a user's own allow/deny entries are untouched (mirrors
|
|
377
|
+
* mergeManagedDeny). Forge automates a worktree-heavy workflow (streams, the
|
|
378
|
+
* slice runner, orchestration/human-verified teardown), but the harness auto-mode
|
|
379
|
+
* classifier soft-denies `git worktree remove` for a worktree the session didn't
|
|
380
|
+
* create in-session, and a Remote-Control / detached session cannot enter
|
|
381
|
+
* bypassPermissions after launch. A NARROW allow rule survives auto mode and runs
|
|
382
|
+
* before the classifier, so the automating session can clean up its own worktrees.
|
|
383
|
+
* These are read/teardown of Forge-managed worktrees only — `remove` is git-guarded
|
|
384
|
+
* (refuses on a dirty/untracked tree without --force) and, in the Forge flow, only
|
|
385
|
+
* ever runs on a merged, human-verified worktree. It never touches commits or
|
|
386
|
+
* branches — worktree removal drops a checkout, not history.
|
|
387
|
+
*/
|
|
388
|
+
function mergeManagedAllow(destSettings, srcSettings) {
|
|
389
|
+
const srcAllow = (srcSettings.permissions && srcSettings.permissions.allow) || [];
|
|
390
|
+
if (!srcAllow.length) return;
|
|
391
|
+
destSettings.permissions = destSettings.permissions || {};
|
|
392
|
+
const destAllow = destSettings.permissions.allow || (destSettings.permissions.allow = []);
|
|
393
|
+
for (const rule of srcAllow) {
|
|
394
|
+
if (!destAllow.includes(rule)) destAllow.push(rule);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
377
398
|
/**
|
|
378
399
|
* Strip the legacy *inline* active-skill guard from PreToolUse (forge#12). It was
|
|
379
400
|
* an inline `if [ ! -f .../.forge/.active-skill ]; then ... exit 2; fi` one-liner;
|
|
@@ -437,6 +458,7 @@ function upgradeSettings() {
|
|
|
437
458
|
// Additively install framework-managed safety guardrails (ADR-017)
|
|
438
459
|
mergeManagedHooks(destSettings, srcSettings);
|
|
439
460
|
mergeManagedDeny(destSettings, srcSettings);
|
|
461
|
+
mergeManagedAllow(destSettings, srcSettings);
|
|
440
462
|
|
|
441
463
|
const after = JSON.stringify(destSettings);
|
|
442
464
|
if (before === after) return 'unchanged';
|
|
@@ -949,7 +971,7 @@ async function upgrade() {
|
|
|
949
971
|
if (results.preserved.length > 0) {
|
|
950
972
|
say(` Preserved (${results.preserved.length}):`);
|
|
951
973
|
for (const f of results.preserved) {
|
|
952
|
-
say(` ${f}
|
|
974
|
+
say(` ${f}`);
|
|
953
975
|
}
|
|
954
976
|
for (const e of experimental) {
|
|
955
977
|
say(
|
package/package.json
CHANGED
|
@@ -23,13 +23,17 @@
|
|
|
23
23
|
#
|
|
24
24
|
# Pinned field set (contract between record + recommend; keep names stable):
|
|
25
25
|
# ts repo milestone skill task_type complexity model first_try_pass check_exit
|
|
26
|
+
# cost_usd tokens (step-5 m-42 — durable per-phase spend; the ts field already
|
|
27
|
+
# dates every row, so cost_usd+tokens give spend-over-time stats
|
|
28
|
+
# from this ONE durable log, no new store. Both null when unset —
|
|
29
|
+
# lazy migration, existing rows/readers unaffected.)
|
|
26
30
|
set -u
|
|
27
31
|
|
|
28
32
|
usage() {
|
|
29
33
|
cat <<'EOF' >&2
|
|
30
34
|
usage: forge-model-outcome.sh record [--repo R] [--milestone M] [--skill S]
|
|
31
35
|
[--task-type T] [--complexity C] [--model MO] [--first-try-pass true|false]
|
|
32
|
-
[--check-exit N]
|
|
36
|
+
[--check-exit N] [--cost USD] [--tokens N]
|
|
33
37
|
forge-model-outcome.sh recommend
|
|
34
38
|
EOF
|
|
35
39
|
}
|
|
@@ -42,7 +46,7 @@ log_path() {
|
|
|
42
46
|
do_record() {
|
|
43
47
|
# --- parse flags (ALL optional — missing -> empty string / null) ---------
|
|
44
48
|
repo=""; milestone=""; skill=""; task_type=""; complexity=""; model=""
|
|
45
|
-
first_try_pass=""; check_exit=""
|
|
49
|
+
first_try_pass=""; check_exit=""; cost=""; tokens=""
|
|
46
50
|
while [ $# -gt 0 ]; do
|
|
47
51
|
case "$1" in
|
|
48
52
|
--repo) shift; repo="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
@@ -61,6 +65,10 @@ do_record() {
|
|
|
61
65
|
--first-try-pass=*) first_try_pass="${1#*=}"; shift ;;
|
|
62
66
|
--check-exit) shift; check_exit="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
63
67
|
--check-exit=*) check_exit="${1#*=}"; shift ;;
|
|
68
|
+
--cost) shift; cost="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
69
|
+
--cost=*) cost="${1#*=}"; shift ;;
|
|
70
|
+
--tokens) shift; tokens="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
71
|
+
--tokens=*) tokens="${1#*=}"; shift ;;
|
|
64
72
|
-*) printf 'forge-model-outcome: unknown option: %s (ignored)\n' "$1" >&2; shift ;;
|
|
65
73
|
*) shift ;;
|
|
66
74
|
esac
|
|
@@ -87,9 +95,16 @@ do_record() {
|
|
|
87
95
|
*) ce="$check_exit" ;;
|
|
88
96
|
esac
|
|
89
97
|
|
|
90
|
-
|
|
98
|
+
# cost_usd (float) + tokens (int) — numeric JSON, null when unset/malformed. A
|
|
99
|
+
# numeric literal (not a quoted string) keeps `jq` sums/averages trivial.
|
|
100
|
+
cost_j="null"
|
|
101
|
+
case "$cost" in ''|*[!0-9.]*) cost_j="null" ;; *) cost_j="$cost" ;; esac
|
|
102
|
+
tokens_j="null"
|
|
103
|
+
case "$tokens" in ''|*[!0-9]*) tokens_j="null" ;; *) tokens_j="$tokens" ;; esac
|
|
104
|
+
|
|
105
|
+
line=$(printf '{"ts":"%s","repo":"%s","milestone":"%s","skill":"%s","task_type":"%s","complexity":"%s","model":"%s","first_try_pass":%s,"check_exit":%s,"cost_usd":%s,"tokens":%s}' \
|
|
91
106
|
"$(esc "$ts")" "$(esc "$repo")" "$(esc "$milestone")" "$(esc "$skill")" \
|
|
92
|
-
"$(esc "$task_type")" "$(esc "$complexity")" "$(esc "$model")" "$fp" "$ce")
|
|
107
|
+
"$(esc "$task_type")" "$(esc "$complexity")" "$(esc "$model")" "$fp" "$ce" "$cost_j" "$tokens_j")
|
|
93
108
|
|
|
94
109
|
# The whole create+append is best-effort: swallow any failure (unwritable
|
|
95
110
|
# home, read-only fs, a ".forge" that's a regular file, ...) and exit 0
|
|
@@ -121,7 +136,8 @@ do_recommend() {
|
|
|
121
136
|
model: .[0].model,
|
|
122
137
|
task_type: .[0].task_type,
|
|
123
138
|
count: length,
|
|
124
|
-
pass_rate: (([ .[] | select(.first_try_pass == true) ] | length) / length)
|
|
139
|
+
pass_rate: (([ .[] | select(.first_try_pass == true) ] | length) / length),
|
|
140
|
+
total_cost: ([ .[] | (.cost_usd // 0) ] | add)
|
|
125
141
|
}
|
|
126
142
|
| . + {
|
|
127
143
|
status: (
|
|
@@ -149,7 +165,8 @@ do_recommend() {
|
|
|
149
165
|
count="$(printf '%s' "$b" | jq -r '.count')"
|
|
150
166
|
status="$(printf '%s' "$b" | jq -r '.status')"
|
|
151
167
|
pct="$(printf '%s' "$b" | jq -r '.pass_rate' | awk '{printf "%.0f", $1*100}')"
|
|
152
|
-
printf '
|
|
168
|
+
cost="$(printf '%s' "$b" | jq -r '.total_cost' | LC_NUMERIC=C awk '{printf "%.4f", $1+0}')"
|
|
169
|
+
printf ' model=%-10s task_type=%-14s status=%-10s count=%-4s first-try=%s%% spend=$%s\n' "$model" "$task_type" "$status" "$count" "$pct" "$cost"
|
|
153
170
|
done
|
|
154
171
|
|
|
155
172
|
printf '\nRouting recommendations:\n'
|
|
@@ -172,6 +172,35 @@ usage_num() { # usage_num <result.json> <input_tokens|output_tokens> — the AG
|
|
|
172
172
|
_um="$(sed 's/.*"usage":{//' "$1" 2>/dev/null | grep -o "\"$2\":[0-9]*" | head -1 | sed 's/.*://')"
|
|
173
173
|
[ -n "$_um" ] && printf '%s' "$_um" || printf '0'
|
|
174
174
|
}
|
|
175
|
+
actual_model_from_result() { # actual_model_from_result <result.json> — the PRIMARY model actually used
|
|
176
|
+
# The launcher's result envelope carries `modelUsage`, an object keyed by FULL model
|
|
177
|
+
# id (`claude-sonnet-5`, `claude-haiku-4-5-...`) with a per-model usage breakdown. A
|
|
178
|
+
# single phase legitimately touches MORE than one model (main work + background
|
|
179
|
+
# micro-calls), so "the actual model" is the PRIMARY: the entry with the max
|
|
180
|
+
# outputTokens. Step-5 attribution records reality from HERE, never from intent.
|
|
181
|
+
# Dependency-free awk (NFR-026); result.json is a single line.
|
|
182
|
+
# Anchor to the LAST `"modelUsage":{` (the launcher's real one, appended AFTER the
|
|
183
|
+
# model-controlled `result` string) — same spoof defense result_num uses with tail -1:
|
|
184
|
+
# a phase that embeds `"modelUsage":{…}` in its answer text can't win, because its copy
|
|
185
|
+
# precedes the launcher's and we take the last.
|
|
186
|
+
[ -f "$1" ] || { printf -- '-'; return 0; }
|
|
187
|
+
_am="$(awk '
|
|
188
|
+
{ ex = $0; seg = ""
|
|
189
|
+
while (match(ex, /"modelUsage":\{/)) { ex = substr(ex, RSTART+RLENGTH); seg = ex }
|
|
190
|
+
if (seg == "") next
|
|
191
|
+
best=""; bestn=-1
|
|
192
|
+
while (match(seg, /"[^"]+":\{[^}]*"outputTokens":[0-9]+/)) {
|
|
193
|
+
entry = substr(seg, RSTART, RLENGTH)
|
|
194
|
+
idm = entry; sub(/":\{.*/, "", idm); sub(/^"/, "", idm) # model id = leading quoted token
|
|
195
|
+
otm = entry; sub(/.*"outputTokens":/, "", otm); ot = otm + 0
|
|
196
|
+
if (ot > bestn) { bestn = ot; best = idm }
|
|
197
|
+
seg = substr(seg, RSTART+RLENGTH)
|
|
198
|
+
}
|
|
199
|
+
if (best != "") print best
|
|
200
|
+
exit
|
|
201
|
+
}' "$1" 2>/dev/null)"
|
|
202
|
+
[ -n "$_am" ] && printf '%s' "$_am" || printf -- '-'
|
|
203
|
+
}
|
|
175
204
|
|
|
176
205
|
# --- model-by-phase-type map (STEP-5 SEAM — read, NOT routed) ----------------
|
|
177
206
|
# The runner reads config.model_by_phase_type to RECORD each phase's intended model
|
|
@@ -201,12 +230,17 @@ phase_type_of() { # phase_type_of <phase> — optional `phase_type:` in the pha
|
|
|
201
230
|
| sed -E 's/^phase_type:[[:space:]]*//; s/[[:space:]]*(#.*)?$//')"
|
|
202
231
|
printf '%s' "$_pt"
|
|
203
232
|
}
|
|
204
|
-
model_for_phase() { # model_for_phase <phase>
|
|
233
|
+
model_for_phase() { # model_for_phase <phase> — the intended model, from the MAP ONLY
|
|
234
|
+
# The routing decision (and the recorded intended_model) come SOLELY from
|
|
235
|
+
# config.model_by_phase_type keyed by the phase's phase_type. It deliberately does
|
|
236
|
+
# NOT fall back to the phase's self-reported usage.model: a phase's own output must
|
|
237
|
+
# never influence its dispatch (that would break the absent-map byte-identical
|
|
238
|
+
# invariant on a parked-phase resume, where a prior report persists). No phase_type
|
|
239
|
+
# or no map entry → "-" (unrouted). The <report-file> arg is retained for call-site
|
|
240
|
+
# compatibility but no longer read.
|
|
205
241
|
_mp_type="$(phase_type_of "$1")"
|
|
206
242
|
_mp=""
|
|
207
243
|
[ -n "$_mp_type" ] && _mp="$(cfg_map_get model_by_phase_type "$_mp_type" "")"
|
|
208
|
-
# Fall back to what the phase itself recorded, then a placeholder.
|
|
209
|
-
[ -z "$_mp" ] && _mp="$(report_nested "$2" model)"
|
|
210
244
|
[ -z "$_mp" ] && _mp="-"
|
|
211
245
|
printf '%s' "$_mp"
|
|
212
246
|
}
|
|
@@ -469,6 +503,16 @@ for phase in $PHASES; do
|
|
|
469
503
|
# token furnace). A launch that writes no report at all is a CRASH, not a
|
|
470
504
|
# check failure — it halts immediately so crash-resume re-runs it next boot.
|
|
471
505
|
_pstart="$(date +%s 2>/dev/null || echo 0)" # phase wall-clock start (runner-measured)
|
|
506
|
+
# STEP 5 — per-phase model routing. Resolve the intended model ONCE (map value via
|
|
507
|
+
# model_for_phase, or "-"/"" when unmapped). `_route_model` is what THIS attempt
|
|
508
|
+
# launches with; on a routed STARTUP failure it is blanked for a single unrouted
|
|
509
|
+
# retry (the fallback), recorded `routed → fallback` in the receipt. All routing
|
|
510
|
+
# lives here + at the launch below — nowhere else (FR-198/NFR-044).
|
|
511
|
+
_intended_model="$(model_for_phase "$phase" "$report")"
|
|
512
|
+
_route_model="$_intended_model"
|
|
513
|
+
[ "$_route_model" = "-" ] && _route_model=""
|
|
514
|
+
_fallback=no
|
|
515
|
+
_routed_fallback_done=0
|
|
472
516
|
attempt=1
|
|
473
517
|
max_attempt=2
|
|
474
518
|
verdict=""
|
|
@@ -477,6 +521,11 @@ for phase in $PHASES; do
|
|
|
477
521
|
"$phase" "$idx" "$total" "$LAUNCHER" "$attempt" >&2
|
|
478
522
|
(
|
|
479
523
|
cd "$WORKTREE"
|
|
524
|
+
# STEP 5 routing: prepend `--model <value>` iff this phase is routed. Built via
|
|
525
|
+
# `set --` so an unrouted launch passes NO --model flag — the dispatch args are
|
|
526
|
+
# then byte-for-byte today's (the absent-map neg control). Empty `"$@"` expands
|
|
527
|
+
# to nothing; a set value is properly word-safe.
|
|
528
|
+
if [ -n "$_route_model" ]; then set -- --model "$_route_model"; else set --; fi
|
|
480
529
|
# A headless, zero-touch phase has no TTY to approve tool use, so the per-tool
|
|
481
530
|
# permission gate is bypassed — otherwise the very first Write (the work, or
|
|
482
531
|
# the mandatory phase-report.yml) blocks forever waiting for an approval that
|
|
@@ -489,7 +538,7 @@ for phase in $PHASES; do
|
|
|
489
538
|
FORGE_SLICE_PHASE="$phase" \
|
|
490
539
|
FORGE_SLICE_PHASE_INDEX="$idx" \
|
|
491
540
|
FORGE_SLICE_ATTEMPT="$attempt" \
|
|
492
|
-
"$LAUNCHER" -p \
|
|
541
|
+
"$LAUNCHER" -p "$@" \
|
|
493
542
|
--output-format json \
|
|
494
543
|
--max-turns "$MAX_TURNS" \
|
|
495
544
|
--max-budget-usd "$MAX_BUDGET" \
|
|
@@ -498,12 +547,28 @@ for phase in $PHASES; do
|
|
|
498
547
|
< "$prompt" \
|
|
499
548
|
> "$wdir/result.json" 2> "$wdir/launch.err"
|
|
500
549
|
) || {
|
|
501
|
-
printf '[runner] phase %s: launcher exited non-zero (attempt %s)\n'
|
|
550
|
+
printf '[runner] phase %s: launcher exited non-zero (attempt %s%s)\n' \
|
|
551
|
+
"$phase" "$attempt" "${_route_model:+, --model $_route_model}" >&2
|
|
502
552
|
}
|
|
503
553
|
|
|
504
554
|
# Record this attempt's session id — the fresh-context proof the retry asserts.
|
|
505
555
|
extract_session "$wdir/result.json" > "$wdir/attempt-$attempt.session"
|
|
506
556
|
|
|
557
|
+
# STEP 5 fallback — a ROUTED launch that wrote NO report is a startup failure
|
|
558
|
+
# (e.g. an unusable model). Retry ONCE unrouted (blank _route_model), recorded
|
|
559
|
+
# `routed → fallback` in the receipt. This is a DISTINCT axis from the
|
|
560
|
+
# checks-failed retry: it does not consume `attempt`, and a bad model degrades
|
|
561
|
+
# to the default rather than halting the slice (routing must be harmless). An
|
|
562
|
+
# unrouted launch that writes no report is a genuine crash → the halt below.
|
|
563
|
+
if [ ! -f "$report" ] && [ -n "$_route_model" ] && [ "$_routed_fallback_done" -eq 0 ]; then
|
|
564
|
+
printf '[runner] phase %s: routed launch (--model %s) wrote no report — falling back to default (unrouted), one retry\n' \
|
|
565
|
+
"$phase" "$_route_model" >&2
|
|
566
|
+
_routed_fallback_done=1
|
|
567
|
+
_fallback=yes
|
|
568
|
+
_route_model=""
|
|
569
|
+
continue
|
|
570
|
+
fi
|
|
571
|
+
|
|
507
572
|
# (4) REQUIRE the on-disk phase-report.yml. Absent = crash → halt now (do NOT
|
|
508
573
|
# spend the retry on a crash; the resume path re-runs it from files).
|
|
509
574
|
if [ ! -f "$report" ]; then
|
|
@@ -556,20 +621,59 @@ for phase in $PHASES; do
|
|
|
556
621
|
# spend seam), one row per phase. Sourced for FIDELITY (live walk found the
|
|
557
622
|
# phase-report's self-reported usage was all zeros — a phase can't know its
|
|
558
623
|
# own token count):
|
|
559
|
-
# tokens
|
|
560
|
-
# wall_clock_s
|
|
561
|
-
# retry_count
|
|
562
|
-
# model
|
|
563
|
-
#
|
|
624
|
+
# tokens = result.json input_tokens + output_tokens (launcher truth)
|
|
625
|
+
# wall_clock_s = runner-measured elapsed for the phase (incl. any retry)
|
|
626
|
+
# retry_count = runner-observed (attempt-1), not the phase's self-report
|
|
627
|
+
# model = intended model (kept for back-compat with the 5-col reader)
|
|
628
|
+
# intended_model = what routing ASKED for (map value, "-" when unmapped)
|
|
629
|
+
# actual_model = the PRIMARY model actually used (result.json modelUsage,
|
|
630
|
+
# max outputTokens) — reality, not intent
|
|
631
|
+
# fallback = yes when a routed launch fell back unrouted (routed → fallback)
|
|
632
|
+
# cost_usd = per-phase spend (result.json total_cost_usd) — the spend
|
|
633
|
+
# the by-phase-type-and-model attribution surface sums
|
|
564
634
|
_tok_in="$(usage_num "$wdir/result.json" input_tokens)"
|
|
565
635
|
_tok_out="$(usage_num "$wdir/result.json" output_tokens)"
|
|
566
636
|
_u_tokens="$(awk -v a="$_tok_in" -v b="$_tok_out" 'BEGIN{printf "%d", (a+0)+(b+0)}')"
|
|
567
637
|
_now="$(date +%s 2>/dev/null || echo 0)"
|
|
568
638
|
if [ "$_now" -ge "$_pstart" ] 2>/dev/null; then _u_wall=$(( _now - _pstart )); else _u_wall=0; fi
|
|
569
639
|
_u_retry=$(( attempt - 1 ))
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
640
|
+
_u_intended="${_intended_model:--}"
|
|
641
|
+
_u_actual="$(actual_model_from_result "$wdir/result.json")"
|
|
642
|
+
_u_model="$_u_intended" # 5-col `model` = intended (back-compat)
|
|
643
|
+
_u_cost="$_cost" # per-phase spend (result.json total_cost_usd)
|
|
644
|
+
[ -f "$USAGE_FILE" ] || printf 'phase_name\ttokens\twall_clock_s\tmodel\tretry_count\tintended_model\tactual_model\tfallback\tcost_usd\n' > "$USAGE_FILE"
|
|
645
|
+
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
|
646
|
+
"$phase" "$_u_tokens" "$_u_wall" "$_u_model" "$_u_retry" "$_u_intended" "$_u_actual" "$_fallback" "$_u_cost" >> "$USAGE_FILE"
|
|
647
|
+
|
|
648
|
+
# (4b) FEED THE M24 OUTCOME LADDER (step-5 B4). Append ONE outcome line per ROUTED
|
|
649
|
+
# phase so the ladder accumulates real per-(model, task_type) volume at last.
|
|
650
|
+
# The ladder answers "should I route task_type T to model X?", so it is keyed on
|
|
651
|
+
# the model the operator MAPPED — the intended model — in ONE consistent
|
|
652
|
+
# vocabulary for BOTH outcomes (a routing that held and one that fell back share
|
|
653
|
+
# a key, so success/failure of the same mapped model aggregate instead of
|
|
654
|
+
# fragmenting). Recorded only when there is a routing decision to judge:
|
|
655
|
+
# • unrouted (no phase_type / no map entry) → SKIP: the default model's baseline
|
|
656
|
+
# is not a routing choice.
|
|
657
|
+
# • park → SKIP: an operator interrupt is not a model-quality outcome.
|
|
658
|
+
# • routed → fallback → first_try_pass=false (the mapped model failed to run).
|
|
659
|
+
# • routed & ran → first_try_pass = attempt 1 AND terminal-good AND no checks failed.
|
|
660
|
+
# ADVISORY: the helper always exits 0; guard so a missing helper never touches the
|
|
661
|
+
# slice. The routing map is consumed READ-ONLY throughout the runner; only the
|
|
662
|
+
# operator ever edits it by hand (D-F) — no code path here modifies that config.
|
|
663
|
+
_mo_type="$(phase_type_of "$phase")"
|
|
664
|
+
if [ -n "$_mo_type" ] && [ -n "$_intended_model" ] && [ "$_intended_model" != "-" ] \
|
|
665
|
+
&& [ "$verdict" != park ] && [ -x "$SELF_DIR/forge-model-outcome.sh" ]; then
|
|
666
|
+
_mo_ftp=false
|
|
667
|
+
if [ "$_fallback" = no ] && [ "$attempt" -eq 1 ] \
|
|
668
|
+
&& { [ "$verdict" = proceed ] || [ "$verdict" = done ]; } && [ "$cfailed" -eq 0 ]; then
|
|
669
|
+
_mo_ftp=true
|
|
670
|
+
fi
|
|
671
|
+
"$SELF_DIR/forge-model-outcome.sh" record \
|
|
672
|
+
--repo "$(basename "$REPO_ROOT" 2>/dev/null || echo slice)" \
|
|
673
|
+
--skill executing --task-type "$_mo_type" \
|
|
674
|
+
--model "$_intended_model" --first-try-pass "$_mo_ftp" \
|
|
675
|
+
--check-exit "$cfailed" --cost "$_u_cost" --tokens "$_u_tokens" >/dev/null 2>&1 || true
|
|
676
|
+
fi
|
|
573
677
|
|
|
574
678
|
# (5) BRANCH on verdict. proceed advances; done ends with the done ping.
|
|
575
679
|
# park is STUBBED here (plan-48 wires the real §8 interrupt ping) — explicit,
|
|
@@ -103,6 +103,30 @@ else
|
|
|
103
103
|
fail "case6 recommend missing insufficient-data message: $out"
|
|
104
104
|
fi
|
|
105
105
|
|
|
106
|
+
# --- Case 7: cost_usd + tokens are stored numeric, null when omitted (m-42) --
|
|
107
|
+
d="$ROOT/case7"; mkdir -p "$d"
|
|
108
|
+
HOME="$d" "$HELPER" record --repo x --skill executing --task-type verifying \
|
|
109
|
+
--model haiku --first-try-pass true --check-exit 0 --cost 0.0333 --tokens 686 >/dev/null
|
|
110
|
+
HOME="$d" "$HELPER" record --repo x --skill executing --task-type docs \
|
|
111
|
+
--model sonnet --first-try-pass true >/dev/null # no cost/tokens → null
|
|
112
|
+
log="$d/.forge/model-outcomes.jsonl"
|
|
113
|
+
if grep -q '"cost_usd":0.0333' "$log" && grep -q '"tokens":686' "$log"; then
|
|
114
|
+
pass "case7 record stores numeric cost_usd + tokens"
|
|
115
|
+
else
|
|
116
|
+
fail "case7 cost/tokens not stored numeric: $(cat "$log")"
|
|
117
|
+
fi
|
|
118
|
+
if grep -q '"cost_usd":null' "$log" && grep -q '"tokens":null' "$log"; then
|
|
119
|
+
pass "case7 omitted cost/tokens store as null (lazy migration)"
|
|
120
|
+
else
|
|
121
|
+
fail "case7 omitted cost/tokens not null: $(cat "$log")"
|
|
122
|
+
fi
|
|
123
|
+
out="$(HOME="$d" "$HELPER" recommend)"
|
|
124
|
+
if printf '%s' "$out" | grep -qE 'spend=\$0\.0333'; then
|
|
125
|
+
pass "case7 recommend surfaces per-bucket spend"
|
|
126
|
+
else
|
|
127
|
+
fail "case7 recommend missing spend column: $out"
|
|
128
|
+
fi
|
|
129
|
+
|
|
106
130
|
# --- Report -----------------------------------------------------------------
|
|
107
131
|
printf '\n%d passed, %d failed\n' "$PASSED" "$FAILED"
|
|
108
132
|
[ "$FAILED" -eq 0 ]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"forge": {
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.68.0",
|
|
4
4
|
"default_tier": "standard",
|
|
5
5
|
"beads_integration": false,
|
|
6
6
|
"context_gates": {
|
|
@@ -48,6 +48,11 @@
|
|
|
48
48
|
"Edit(**/.secrets/**)",
|
|
49
49
|
"Edit(**/*.pem)",
|
|
50
50
|
"Edit(**/*.key)"
|
|
51
|
+
],
|
|
52
|
+
"allow": [
|
|
53
|
+
"Bash(git worktree remove *)",
|
|
54
|
+
"Bash(git worktree prune)",
|
|
55
|
+
"Bash(git worktree prune *)"
|
|
51
56
|
]
|
|
52
57
|
},
|
|
53
58
|
"hooks": {
|
|
@@ -193,6 +193,7 @@ Plan naming reflects the slice: `plan-01-user-signs-up.md`, not `plan-01-models.
|
|
|
193
193
|
- `{id}`=milestone, `{phase}`=phase# (preserved verbatim, NOT renumbered), `{name}`=kebab, `{NN}`=seq
|
|
194
194
|
- Ex: `.forge/phases/milestone-3/2-providers/plan-01.md`
|
|
195
195
|
2. Frontmatter: phase, plan#, wave, deps, `slice_exception:` (optional, see Core Principle), `integration_checkpoint:` (optional, see below)
|
|
196
|
+
- **`phase_type:` (per-phase model routing, m-42).** Stamp the workflow phase this plan SERVES — one of the `model_by_phase_type` keys (`researching | discussing | architecting | planning | executing | verifying | reviewing`). Build plans are almost always `executing`; a plan that authors a verify/review phase's work is `verifying`/`reviewing`. This is what lets the headless slice runner route each phase to the model mapped for its type (the map ships commented → routing is off until the operator activates it, so stamping this is always safe and never changes behavior on its own). Empty ⇒ that phase runs unrouted (inherits the default model). Emit it on every plan; when unsure, `executing`.
|
|
196
197
|
3. must_haves:
|
|
197
198
|
- **Truths:** User-observable outcomes (3-7). MUST be phrased as something the user can see, click, or receive -- not "model X exists" or "table Y created".
|
|
198
199
|
- Each truth SHOULD carry a `check:` command authored NOW, goal-backward, before any code exists -- a check written before the code cannot be retro-fitted to what got built (semantics in `.forge/templates/plan.md`).
|
|
@@ -139,7 +139,7 @@ Source version (`{source}/packages/create-forge/package.json`) **<** installed (
|
|
|
139
139
|
|
|
140
140
|
Dev mode syncs from a checkout, so it keeps file-classification semantics — **the npm bin's sync behavior is the spec**; do not re-derive it here. Categories:
|
|
141
141
|
|
|
142
|
-
- **Framework-owned** (`.claude/agents/*.md`, `.claude/skills/*/SKILL.md`, `.forge/FORGE.md`) → overwrite when different
|
|
142
|
+
- **Framework-owned** (`.claude/agents/*.md`, `.claude/skills/*/SKILL.md`, `.forge/FORGE.md`) → overwrite when different. Ownership is per top-level **unit** — a skill DIR under `.claude/skills/`, a flat agent FILE under `.claude/agents/`: a unit the template doesn't ship (user-authored, vendored-parity, or an opt-in experimental skill) is **preserved, never deleted**; only stale files *inside* a still-shipped skill dir are cleaned. Experimental skills additionally get the staleness pass below.
|
|
143
143
|
- **Template-only** (`.forge/templates/**`, `.forge/migrations/**`, `.forge/adapters/**`, `.forge/gitignore` → `.forge/.gitignore`) → overwrite; `.forge/FORGE.md` lands BEFORE the CLAUDE.md pass.
|
|
144
144
|
- **Additive** (`.claude/hooks/**` incl. `tests/`, `.claude/rules/**`, `.forge/checks/**`) → add + overwrite template-shipped files, never delete local extras, `chmod +x` synced `*.sh`.
|
|
145
145
|
- **Import-managed** `CLAUDE.md` → the bin's `ensureClaudeMdImport()` cases verbatim; user content outside a forge section is never modified.
|
|
@@ -91,17 +91,25 @@ materialize_work_order() {
|
|
|
91
91
|
# Prefer the worktree's OWN milestone — .forge/phases/ carries the repo's whole milestone
|
|
92
92
|
# history (32 dirs on the forge repo), so "the only milestone dir" is a fixture-world
|
|
93
93
|
# assumption. The Worktree Convention names the branch forge/{anchor} with anchor =
|
|
94
|
-
# {milestone-id}[-{session}]; derive the id and pick that milestone's dir.
|
|
94
|
+
# {milestone-id}[-{session}]; derive the id and pick that milestone's dir. The id's own
|
|
95
|
+
# m- may be a strippable prefix (m-40 → milestone-40) or PART of a labeled id kept
|
|
96
|
+
# verbatim (m-EVT01 → milestone-m-EVT01) — canvaz first run 2026-07-22: stripping it
|
|
97
|
+
# unconditionally missed milestone-m-EVT01 and fell through to a 38-dir refusal. Try the
|
|
98
|
+
# anchor verbatim before the m--stripped form, then both again minus one trailing
|
|
99
|
+
# -{session} token; first existing dir wins (exact/longer forms first).
|
|
95
100
|
_mdir=""
|
|
96
101
|
_branch="$(git -C "$WORKTREE" branch --show-current 2>/dev/null)"
|
|
97
102
|
case "$_branch" in
|
|
98
|
-
forge
|
|
99
|
-
_anchor="${_branch#forge/
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
forge/*)
|
|
104
|
+
_anchor="${_branch#forge/}" # m-40 | m-40-a1b2c3d4 | m-EVT01-e6063597
|
|
105
|
+
_short="${_anchor%-*}" # minus one trailing -token (session suffix)
|
|
106
|
+
for _cand in "$_anchor" "${_anchor#m-}" "$_short" "${_short#m-}"; do
|
|
107
|
+
[ -n "$_cand" ] || continue
|
|
108
|
+
if [ -d "$WORKTREE/.forge/phases/milestone-$_cand" ]; then
|
|
109
|
+
_mdir="$WORKTREE/.forge/phases/milestone-$_cand"
|
|
110
|
+
break
|
|
111
|
+
fi
|
|
112
|
+
done ;;
|
|
105
113
|
esac
|
|
106
114
|
# Fallback (non-git fixture dirs / unconventional branches): a SINGLE milestone dir is
|
|
107
115
|
# unambiguous; several without a branch-derived pick is a refusal, never a guess.
|
|
@@ -35,6 +35,7 @@ SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
|
35
35
|
REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)"
|
|
36
36
|
|
|
37
37
|
LOGS=""
|
|
38
|
+
USAGES=""
|
|
38
39
|
MARKER_DIR_OVERRIDE=""
|
|
39
40
|
LIVE_WINDOW=120
|
|
40
41
|
SINCE=""
|
|
@@ -43,6 +44,8 @@ DRY_RUN=0
|
|
|
43
44
|
while [ $# -gt 0 ]; do
|
|
44
45
|
case "$1" in
|
|
45
46
|
--log) LOGS="${LOGS}${LOGS:+
|
|
47
|
+
}${2:-}"; shift 2 ;;
|
|
48
|
+
--usage) USAGES="${USAGES}${USAGES:+
|
|
46
49
|
}${2:-}"; shift 2 ;;
|
|
47
50
|
--marker-dir) MARKER_DIR_OVERRIDE="${2:-}"; shift 2 ;;
|
|
48
51
|
--live-window) LIVE_WINDOW="${2:-120}"; shift 2 ;;
|
|
@@ -82,9 +85,36 @@ GLOBBED_EOF
|
|
|
82
85
|
fi
|
|
83
86
|
fi
|
|
84
87
|
|
|
88
|
+
# ── resolve the usage.tsv set (spend receipts): explicit --usage wins; else disk-glob ────────────
|
|
89
|
+
# Same slice worktrees as the logs, sibling file runner-work/usage.tsv. No new store —
|
|
90
|
+
# these ARE the per-phase receipts the runner already writes (step-5 spend attribution).
|
|
91
|
+
if [ -z "$USAGES" ]; then
|
|
92
|
+
if [ -n "$WT_ROOT" ]; then
|
|
93
|
+
for _wt in "$WT_ROOT"/*/; do
|
|
94
|
+
[ -d "$_wt" ] || continue
|
|
95
|
+
_u="${_wt}.forge/runner-work/usage.tsv"
|
|
96
|
+
[ -f "$_u" ] && USAGES="${USAGES}${USAGES:+
|
|
97
|
+
}$_u"
|
|
98
|
+
done
|
|
99
|
+
else
|
|
100
|
+
_uglobbed="$(git -C "$REPO_ROOT" worktree list --porcelain 2>/dev/null | awk '
|
|
101
|
+
/^worktree / { p = substr($0, 10) }
|
|
102
|
+
/^branch / { if ($2 ~ /refs\/heads\/forge\/m-/) print p "/.forge/runner-work/usage.tsv" }')"
|
|
103
|
+
while IFS= read -r _u; do
|
|
104
|
+
[ -n "$_u" ] || continue
|
|
105
|
+
[ -f "$_u" ] && USAGES="${USAGES}${USAGES:+
|
|
106
|
+
}$_u"
|
|
107
|
+
done <<UGLOBBED_EOF
|
|
108
|
+
$_uglobbed
|
|
109
|
+
UGLOBBED_EOF
|
|
110
|
+
fi
|
|
111
|
+
fi
|
|
112
|
+
|
|
85
113
|
if [ "$DRY_RUN" -eq 1 ]; then
|
|
86
114
|
printf 'forge-jarvis-instruments (--dry-run): resolved logs:\n'
|
|
87
115
|
if [ -n "$LOGS" ]; then printf '%s\n' "$LOGS" | sed 's/^/ /'; else printf ' (none)\n'; fi
|
|
116
|
+
printf 'forge-jarvis-instruments (--dry-run): resolved usage receipts:\n'
|
|
117
|
+
if [ -n "$USAGES" ]; then printf '%s\n' "$USAGES" | sed 's/^/ /'; else printf ' (none)\n'; fi
|
|
88
118
|
exit 0
|
|
89
119
|
fi
|
|
90
120
|
|
|
@@ -146,4 +176,41 @@ awk -F'\t' -v total="$_total" -v win="$LIVE_WINDOW" '
|
|
|
146
176
|
printf " (proxy — derived from relay latency, not an uptime store)\n"
|
|
147
177
|
}' "$TMP"
|
|
148
178
|
|
|
179
|
+
# ── 5. Spend by phase and model (step-5 attribution) ─────────────────────────────────────────────
|
|
180
|
+
# Answers "what did this cost, by phase and model?" WITHOUT opening a file — computed
|
|
181
|
+
# live from the runner's usage.tsv receipts (9-col step-5 schema: …actual_model fallback
|
|
182
|
+
# cost_usd). NO new store. Legacy 5-col receipts have no model/cost columns → skipped with
|
|
183
|
+
# a note. Groups by (phase, actual_model), sums cost, flags routed→fallback rows.
|
|
184
|
+
printf '\n5. Spend by phase and model (from usage.tsv receipts):\n'
|
|
185
|
+
printf '%s\n' "$USAGES" | while IFS= read -r U; do
|
|
186
|
+
[ -n "$U" ] && [ -f "$U" ] || continue
|
|
187
|
+
_hdr="$(head -1 "$U")"
|
|
188
|
+
case "$_hdr" in
|
|
189
|
+
*actual_model*cost_usd*)
|
|
190
|
+
# LC_NUMERIC=C so awk parses AND prints the `.`-decimal cost regardless of locale
|
|
191
|
+
# (a comma locale otherwise reads "0.63" as 0 and prints "0,63" — same fix the runner uses).
|
|
192
|
+
LC_NUMERIC=C awk -F'\t' -v src="$U" '
|
|
193
|
+
NR==1 { next }
|
|
194
|
+
NF>=9 && $1!="" {
|
|
195
|
+
key = $1 SUBSEP $7 # phase SUBSEP actual_model
|
|
196
|
+
cost[key] += ($9 + 0)
|
|
197
|
+
if ($8 == "yes") fb[key] += 1
|
|
198
|
+
seen[key] = 1
|
|
199
|
+
}
|
|
200
|
+
END {
|
|
201
|
+
for (k in seen) {
|
|
202
|
+
split(k, a, SUBSEP)
|
|
203
|
+
printf " %-16s %-26s $%.4f%s\n", a[1], a[2], cost[k], (fb[k] ? " (routed->fallback x" fb[k] ")" : "")
|
|
204
|
+
}
|
|
205
|
+
}' "$U"
|
|
206
|
+
;;
|
|
207
|
+
*)
|
|
208
|
+
printf ' (legacy 5-col receipt %s — no model/cost columns; re-run to populate)\n' "$U" ;;
|
|
209
|
+
esac
|
|
210
|
+
done
|
|
211
|
+
# Note when no receipts were found at all (a clean "nothing ran yet", not an error).
|
|
212
|
+
if [ -z "$USAGES" ]; then
|
|
213
|
+
printf ' (no usage.tsv receipts found — no slices instrumented yet)\n'
|
|
214
|
+
fi
|
|
215
|
+
|
|
149
216
|
exit 0
|
|
@@ -157,6 +157,41 @@ else
|
|
|
157
157
|
bad "dry-run side effects" "dry-run wrote work-order files"
|
|
158
158
|
fi
|
|
159
159
|
|
|
160
|
+
# --- 13-14. BRANCH-DERIVED MILESTONE PICK (canvaz first run 2026-07-22): with several
|
|
161
|
+
# milestone dirs, the forge/{anchor} branch names the one to build. The id's m- may be
|
|
162
|
+
# strippable (m-40 → milestone-40) or part of a labeled id (m-EVT01 → milestone-m-EVT01);
|
|
163
|
+
# unconditional stripping refused a clean labeled-id dispatch. These fixtures are real
|
|
164
|
+
# git repos — cases 9-12 (mktemp dirs) never reach the branch path.
|
|
165
|
+
mkbranch() { git -C "$1" init -q -b "$2" 2>/dev/null || { git -C "$1" init -q && git -C "$1" checkout -q -b "$2"; }; }
|
|
166
|
+
|
|
167
|
+
# 13. labeled id: forge/m-EVT01-<session> resolves milestone-m-EVT01 (m- is part of the id)
|
|
168
|
+
LSLICE="$ROOT/labeled-wt"
|
|
169
|
+
mkdir -p "$LSLICE/.forge/phases/milestone-m-EVT01/1-evt" "$LSLICE/.forge/phases/milestone-5/1-decoy"
|
|
170
|
+
printf 'evt plan\n' > "$LSLICE/.forge/phases/milestone-m-EVT01/1-evt/plan-01.md"
|
|
171
|
+
printf 'decoy\n' > "$LSLICE/.forge/phases/milestone-5/1-decoy/plan-01.md"
|
|
172
|
+
mkbranch "$LSLICE" forge/m-EVT01-e6063597
|
|
173
|
+
err13="$(sh "$DISPATCH" --slice "$LSLICE" --materialize-only 2>&1)"; rc=$?
|
|
174
|
+
if [ "$rc" = 0 ] && grep -q '^ - 1-evt$' "$LSLICE/slice.yml" 2>/dev/null \
|
|
175
|
+
&& grep -q 'evt plan' "$LSLICE/phases/1-evt/plan.md" 2>/dev/null && [ ! -d "$LSLICE/phases/1-decoy" ]; then
|
|
176
|
+
ok "labeled milestone id: branch picks milestone-m-EVT01 among several dirs (m- kept)"
|
|
177
|
+
else
|
|
178
|
+
bad "labeled-id branch pick" "rc=$rc err=$err13"
|
|
179
|
+
fi
|
|
180
|
+
|
|
181
|
+
# 14. numeric id + session suffix: forge/m-40-<session> still resolves milestone-40 (m- stripped)
|
|
182
|
+
QSLICE="$ROOT/numeric-wt"
|
|
183
|
+
mkdir -p "$QSLICE/.forge/phases/milestone-40/1-num" "$QSLICE/.forge/phases/milestone-39/1-decoy"
|
|
184
|
+
printf 'num plan\n' > "$QSLICE/.forge/phases/milestone-40/1-num/plan-01.md"
|
|
185
|
+
printf 'decoy\n' > "$QSLICE/.forge/phases/milestone-39/1-decoy/plan-01.md"
|
|
186
|
+
mkbranch "$QSLICE" forge/m-40-a1b2c3d4
|
|
187
|
+
err14="$(sh "$DISPATCH" --slice "$QSLICE" --materialize-only 2>&1)"; rc=$?
|
|
188
|
+
if [ "$rc" = 0 ] && grep -q '^ - 1-num$' "$QSLICE/slice.yml" 2>/dev/null \
|
|
189
|
+
&& grep -q 'num plan' "$QSLICE/phases/1-num/plan.md" 2>/dev/null && [ ! -d "$QSLICE/phases/1-decoy" ]; then
|
|
190
|
+
ok "numeric milestone id: branch picks milestone-40 among several dirs (m- stripped)"
|
|
191
|
+
else
|
|
192
|
+
bad "numeric-id branch pick" "rc=$rc err=$err14"
|
|
193
|
+
fi
|
|
194
|
+
|
|
160
195
|
# --- summary -----------------------------------------------------------------
|
|
161
196
|
printf '\n%s: %d passed, %d failed\n' "$(basename "$0")" "$PASSED" "$FAILED"
|
|
162
197
|
[ "$FAILED" = 0 ]
|
|
@@ -60,5 +60,28 @@ printf '%s\n' "$OUT" | grep -qE '2/3 relayed live' && ok "presence proxy:
|
|
|
60
60
|
# 5. the presence line labels itself a proxy (never a measured SLA)
|
|
61
61
|
printf '%s\n' "$OUT" | grep -qiE 'proxy' && ok "presence figure is labelled a proxy, not an SLA" || bad "proxy label" "$OUT"
|
|
62
62
|
|
|
63
|
+
# ── spend-by-phase-and-model section (step-5 attribution, from usage.tsv receipts) ──
|
|
64
|
+
# A hand-built 9-col usage.tsv: two models, one routed→fallback row. The section must
|
|
65
|
+
# answer "what did this cost, by phase and model" from the receipts — no new store.
|
|
66
|
+
USG="$ROOT/usage.tsv"
|
|
67
|
+
{
|
|
68
|
+
printf 'phase_name\ttokens\twall_clock_s\tmodel\tretry_count\tintended_model\tactual_model\tfallback\tcost_usd\n'
|
|
69
|
+
printf 'phase-1\t1000\t2\tsonnet\t0\tsonnet\tclaude-sonnet-5\tno\t0.6300\n'
|
|
70
|
+
printf 'phase-2\t500\t1\thaiku\t0\thaiku\tclaude-haiku-4-5-20251001\tno\t0.0040\n'
|
|
71
|
+
printf 'phase-3\t800\t3\topus\t0\topus\tclaude-sonnet-5\tyes\t0.4000\n'
|
|
72
|
+
} > "$USG"
|
|
73
|
+
|
|
74
|
+
SOUT="$(sh "$INSTR" --log "$LOG" --marker-dir "$MK" --usage "$USG" 2>/dev/null)"
|
|
75
|
+
|
|
76
|
+
printf '%s\n' "$SOUT" | grep -qE '5\. Spend by phase and model' && ok "spend section renders" || bad "spend section" "$SOUT"
|
|
77
|
+
printf '%s\n' "$SOUT" | grep -qE 'phase-1.*claude-sonnet-5.*\$0\.6300' && ok "spend: phase-1 on claude-sonnet-5 = \$0.6300" || bad "spend phase-1" "$SOUT"
|
|
78
|
+
printf '%s\n' "$SOUT" | grep -qE 'phase-2.*claude-haiku-4-5.*\$0\.0040' && ok "spend: phase-2 on claude-haiku = \$0.0040" || bad "spend phase-2" "$SOUT"
|
|
79
|
+
printf '%s\n' "$SOUT" | grep -qE 'phase-3.*routed->fallback' && ok "spend: phase-3 flagged routed->fallback" || bad "spend fallback flag" "$SOUT"
|
|
80
|
+
|
|
81
|
+
# neg control — the numbers are COMPUTED at render; no new stored aggregate file exists.
|
|
82
|
+
grep -rqiE 'spend[-_]?store|aggregate\.tsv|spend\.db|cost[-_]cache' "$SCRIPT_DIR/../forge-jarvis-instruments.sh" \
|
|
83
|
+
&& bad "no-new-store neg control" "found a spend-store reference" \
|
|
84
|
+
|| ok "no new spend store — computed from usage.tsv at render"
|
|
85
|
+
|
|
63
86
|
printf '\n%s: %d passed, %d failed\n' "$(basename "$0")" "$PASSED" "$FAILED"
|
|
64
87
|
[ "$FAILED" = 0 ]
|
|
@@ -13,6 +13,11 @@ phase: 1
|
|
|
13
13
|
phase_name: ""
|
|
14
14
|
plan: 1
|
|
15
15
|
type: execute # execute | tdd
|
|
16
|
+
phase_type: "" # the workflow phase this plan SERVES — one of the model_by_phase_type keys:
|
|
17
|
+
# researching | discussing | architecting | planning | executing | verifying | reviewing
|
|
18
|
+
# Drives per-phase model routing (m-42): the slice runner reads this to pick the phase's
|
|
19
|
+
# model from config.model_by_phase_type. Build plans are usually `executing`; a verify/review
|
|
20
|
+
# phase plan is `verifying`/`reviewing`. Empty = unrouted (inherits the default model).
|
|
16
21
|
wave: 1 # Execution wave (1 = no dependencies)
|
|
17
22
|
depends_on: [] # Plan IDs that must complete first
|
|
18
23
|
autonomous: true # false if contains checkpoints
|
|
@@ -45,18 +45,27 @@ divergence_budget_lines: 4000
|
|
|
45
45
|
# without touching the loop.
|
|
46
46
|
notify_channel: pushnotification
|
|
47
47
|
|
|
48
|
-
# --- model-by-phase-type map (
|
|
48
|
+
# --- model-by-phase-type map (per-phase model routing — step 5) --------------
|
|
49
49
|
|
|
50
|
-
#
|
|
51
|
-
# to
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
50
|
+
# Maps a phase's `phase_type:` (declared in its plan — the workflow phase it serves:
|
|
51
|
+
# researching|discussing|architecting|planning|executing|verifying|reviewing) to the
|
|
52
|
+
# model its headless `claude -p` launch runs on. `--model` accepts these aliases
|
|
53
|
+
# (or a full id like `claude-sonnet-5`).
|
|
54
|
+
#
|
|
55
|
+
# SHIPS COMMENTED = OFF BY DEFAULT. An absent/empty map reproduces today's behavior
|
|
56
|
+
# BYTE-FOR-BYTE: every phase inherits the default model, no `--model` flag is added.
|
|
57
|
+
# UNCOMMENT an entry to activate routing for that phase-type — that first uncomment
|
|
58
|
+
# is your moment to confirm it does what you expect (nothing chooses for you; the map
|
|
59
|
+
# is yours and only you edit it — no code ever writes it). A routed launch that fails
|
|
60
|
+
# at startup falls back once to the default and records `routed → fallback` in the
|
|
61
|
+
# usage receipt, so a bad value degrades loudly, never silently.
|
|
62
|
+
#
|
|
63
|
+
# Suggested starting point (uncomment the lines you want):
|
|
55
64
|
model_by_phase_type:
|
|
56
|
-
researching: sonnet
|
|
57
|
-
discussing: sonnet
|
|
58
|
-
architecting: opus
|
|
59
|
-
planning: opus
|
|
60
|
-
executing: sonnet
|
|
61
|
-
verifying: haiku
|
|
62
|
-
reviewing: sonnet
|
|
65
|
+
# researching: sonnet
|
|
66
|
+
# discussing: sonnet
|
|
67
|
+
# architecting: opus
|
|
68
|
+
# planning: opus
|
|
69
|
+
# executing: sonnet
|
|
70
|
+
# verifying: haiku
|
|
71
|
+
# reviewing: sonnet
|