forge-orkes 0.64.5 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/template/.claude/hooks/forge-model-outcome.sh +23 -6
- package/template/.claude/hooks/forge-slice-runner.sh +117 -13
- package/template/.claude/hooks/tests/forge-model-outcome.test.sh +24 -0
- package/template/.claude/skills/planning/SKILL.md +1 -0
- package/template/.forge/checks/forge-jarvis-instruments.sh +67 -0
- package/template/.forge/checks/tests/forge-jarvis-instruments.test.sh +23 -0
- package/template/.forge/templates/plan.md +5 -0
- package/template/.forge/templates/slice-runner/config.yml +22 -13
package/package.json
CHANGED
|
@@ -23,13 +23,17 @@
|
|
|
23
23
|
#
|
|
24
24
|
# Pinned field set (contract between record + recommend; keep names stable):
|
|
25
25
|
# ts repo milestone skill task_type complexity model first_try_pass check_exit
|
|
26
|
+
# cost_usd tokens (step-5 m-42 — durable per-phase spend; the ts field already
|
|
27
|
+
# dates every row, so cost_usd+tokens give spend-over-time stats
|
|
28
|
+
# from this ONE durable log, no new store. Both null when unset —
|
|
29
|
+
# lazy migration, existing rows/readers unaffected.)
|
|
26
30
|
set -u
|
|
27
31
|
|
|
28
32
|
usage() {
|
|
29
33
|
cat <<'EOF' >&2
|
|
30
34
|
usage: forge-model-outcome.sh record [--repo R] [--milestone M] [--skill S]
|
|
31
35
|
[--task-type T] [--complexity C] [--model MO] [--first-try-pass true|false]
|
|
32
|
-
[--check-exit N]
|
|
36
|
+
[--check-exit N] [--cost USD] [--tokens N]
|
|
33
37
|
forge-model-outcome.sh recommend
|
|
34
38
|
EOF
|
|
35
39
|
}
|
|
@@ -42,7 +46,7 @@ log_path() {
|
|
|
42
46
|
do_record() {
|
|
43
47
|
# --- parse flags (ALL optional — missing -> empty string / null) ---------
|
|
44
48
|
repo=""; milestone=""; skill=""; task_type=""; complexity=""; model=""
|
|
45
|
-
first_try_pass=""; check_exit=""
|
|
49
|
+
first_try_pass=""; check_exit=""; cost=""; tokens=""
|
|
46
50
|
while [ $# -gt 0 ]; do
|
|
47
51
|
case "$1" in
|
|
48
52
|
--repo) shift; repo="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
@@ -61,6 +65,10 @@ do_record() {
|
|
|
61
65
|
--first-try-pass=*) first_try_pass="${1#*=}"; shift ;;
|
|
62
66
|
--check-exit) shift; check_exit="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
63
67
|
--check-exit=*) check_exit="${1#*=}"; shift ;;
|
|
68
|
+
--cost) shift; cost="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
69
|
+
--cost=*) cost="${1#*=}"; shift ;;
|
|
70
|
+
--tokens) shift; tokens="${1:-}"; [ $# -gt 0 ] && shift || true ;;
|
|
71
|
+
--tokens=*) tokens="${1#*=}"; shift ;;
|
|
64
72
|
-*) printf 'forge-model-outcome: unknown option: %s (ignored)\n' "$1" >&2; shift ;;
|
|
65
73
|
*) shift ;;
|
|
66
74
|
esac
|
|
@@ -87,9 +95,16 @@ do_record() {
|
|
|
87
95
|
*) ce="$check_exit" ;;
|
|
88
96
|
esac
|
|
89
97
|
|
|
90
|
-
|
|
98
|
+
# cost_usd (float) + tokens (int) — numeric JSON, null when unset/malformed. A
|
|
99
|
+
# numeric literal (not a quoted string) keeps `jq` sums/averages trivial.
|
|
100
|
+
cost_j="null"
|
|
101
|
+
case "$cost" in ''|*[!0-9.]*) cost_j="null" ;; *) cost_j="$cost" ;; esac
|
|
102
|
+
tokens_j="null"
|
|
103
|
+
case "$tokens" in ''|*[!0-9]*) tokens_j="null" ;; *) tokens_j="$tokens" ;; esac
|
|
104
|
+
|
|
105
|
+
line=$(printf '{"ts":"%s","repo":"%s","milestone":"%s","skill":"%s","task_type":"%s","complexity":"%s","model":"%s","first_try_pass":%s,"check_exit":%s,"cost_usd":%s,"tokens":%s}' \
|
|
91
106
|
"$(esc "$ts")" "$(esc "$repo")" "$(esc "$milestone")" "$(esc "$skill")" \
|
|
92
|
-
"$(esc "$task_type")" "$(esc "$complexity")" "$(esc "$model")" "$fp" "$ce")
|
|
107
|
+
"$(esc "$task_type")" "$(esc "$complexity")" "$(esc "$model")" "$fp" "$ce" "$cost_j" "$tokens_j")
|
|
93
108
|
|
|
94
109
|
# The whole create+append is best-effort: swallow any failure (unwritable
|
|
95
110
|
# home, read-only fs, a ".forge" that's a regular file, ...) and exit 0
|
|
@@ -121,7 +136,8 @@ do_recommend() {
|
|
|
121
136
|
model: .[0].model,
|
|
122
137
|
task_type: .[0].task_type,
|
|
123
138
|
count: length,
|
|
124
|
-
pass_rate: (([ .[] | select(.first_try_pass == true) ] | length) / length)
|
|
139
|
+
pass_rate: (([ .[] | select(.first_try_pass == true) ] | length) / length),
|
|
140
|
+
total_cost: ([ .[] | (.cost_usd // 0) ] | add)
|
|
125
141
|
}
|
|
126
142
|
| . + {
|
|
127
143
|
status: (
|
|
@@ -149,7 +165,8 @@ do_recommend() {
|
|
|
149
165
|
count="$(printf '%s' "$b" | jq -r '.count')"
|
|
150
166
|
status="$(printf '%s' "$b" | jq -r '.status')"
|
|
151
167
|
pct="$(printf '%s' "$b" | jq -r '.pass_rate' | awk '{printf "%.0f", $1*100}')"
|
|
152
|
-
printf '
|
|
168
|
+
cost="$(printf '%s' "$b" | jq -r '.total_cost' | LC_NUMERIC=C awk '{printf "%.4f", $1+0}')"
|
|
169
|
+
printf ' model=%-10s task_type=%-14s status=%-10s count=%-4s first-try=%s%% spend=$%s\n' "$model" "$task_type" "$status" "$count" "$pct" "$cost"
|
|
153
170
|
done
|
|
154
171
|
|
|
155
172
|
printf '\nRouting recommendations:\n'
|
|
@@ -172,6 +172,35 @@ usage_num() { # usage_num <result.json> <input_tokens|output_tokens> — the AG
|
|
|
172
172
|
_um="$(sed 's/.*"usage":{//' "$1" 2>/dev/null | grep -o "\"$2\":[0-9]*" | head -1 | sed 's/.*://')"
|
|
173
173
|
[ -n "$_um" ] && printf '%s' "$_um" || printf '0'
|
|
174
174
|
}
|
|
175
|
+
actual_model_from_result() { # actual_model_from_result <result.json> — the PRIMARY model actually used
|
|
176
|
+
# The launcher's result envelope carries `modelUsage`, an object keyed by FULL model
|
|
177
|
+
# id (`claude-sonnet-5`, `claude-haiku-4-5-...`) with a per-model usage breakdown. A
|
|
178
|
+
# single phase legitimately touches MORE than one model (main work + background
|
|
179
|
+
# micro-calls), so "the actual model" is the PRIMARY: the entry with the max
|
|
180
|
+
# outputTokens. Step-5 attribution records reality from HERE, never from intent.
|
|
181
|
+
# Dependency-free awk (NFR-026); result.json is a single line.
|
|
182
|
+
# Anchor to the LAST `"modelUsage":{` (the launcher's real one, appended AFTER the
|
|
183
|
+
# model-controlled `result` string) — same spoof defense result_num uses with tail -1:
|
|
184
|
+
# a phase that embeds `"modelUsage":{…}` in its answer text can't win, because its copy
|
|
185
|
+
# precedes the launcher's and we take the last.
|
|
186
|
+
[ -f "$1" ] || { printf -- '-'; return 0; }
|
|
187
|
+
_am="$(awk '
|
|
188
|
+
{ ex = $0; seg = ""
|
|
189
|
+
while (match(ex, /"modelUsage":\{/)) { ex = substr(ex, RSTART+RLENGTH); seg = ex }
|
|
190
|
+
if (seg == "") next
|
|
191
|
+
best=""; bestn=-1
|
|
192
|
+
while (match(seg, /"[^"]+":\{[^}]*"outputTokens":[0-9]+/)) {
|
|
193
|
+
entry = substr(seg, RSTART, RLENGTH)
|
|
194
|
+
idm = entry; sub(/":\{.*/, "", idm); sub(/^"/, "", idm) # model id = leading quoted token
|
|
195
|
+
otm = entry; sub(/.*"outputTokens":/, "", otm); ot = otm + 0
|
|
196
|
+
if (ot > bestn) { bestn = ot; best = idm }
|
|
197
|
+
seg = substr(seg, RSTART+RLENGTH)
|
|
198
|
+
}
|
|
199
|
+
if (best != "") print best
|
|
200
|
+
exit
|
|
201
|
+
}' "$1" 2>/dev/null)"
|
|
202
|
+
[ -n "$_am" ] && printf '%s' "$_am" || printf -- '-'
|
|
203
|
+
}
|
|
175
204
|
|
|
176
205
|
# --- model-by-phase-type map (STEP-5 SEAM — read, NOT routed) ----------------
|
|
177
206
|
# The runner reads config.model_by_phase_type to RECORD each phase's intended model
|
|
@@ -201,12 +230,17 @@ phase_type_of() { # phase_type_of <phase> — optional `phase_type:` in the pha
|
|
|
201
230
|
| sed -E 's/^phase_type:[[:space:]]*//; s/[[:space:]]*(#.*)?$//')"
|
|
202
231
|
printf '%s' "$_pt"
|
|
203
232
|
}
|
|
204
|
-
model_for_phase() { # model_for_phase <phase>
|
|
233
|
+
model_for_phase() { # model_for_phase <phase> — the intended model, from the MAP ONLY
|
|
234
|
+
# The routing decision (and the recorded intended_model) come SOLELY from
|
|
235
|
+
# config.model_by_phase_type keyed by the phase's phase_type. It deliberately does
|
|
236
|
+
# NOT fall back to the phase's self-reported usage.model: a phase's own output must
|
|
237
|
+
# never influence its dispatch (that would break the absent-map byte-identical
|
|
238
|
+
# invariant on a parked-phase resume, where a prior report persists). No phase_type
|
|
239
|
+
# or no map entry → "-" (unrouted). The <report-file> arg is retained for call-site
|
|
240
|
+
# compatibility but no longer read.
|
|
205
241
|
_mp_type="$(phase_type_of "$1")"
|
|
206
242
|
_mp=""
|
|
207
243
|
[ -n "$_mp_type" ] && _mp="$(cfg_map_get model_by_phase_type "$_mp_type" "")"
|
|
208
|
-
# Fall back to what the phase itself recorded, then a placeholder.
|
|
209
|
-
[ -z "$_mp" ] && _mp="$(report_nested "$2" model)"
|
|
210
244
|
[ -z "$_mp" ] && _mp="-"
|
|
211
245
|
printf '%s' "$_mp"
|
|
212
246
|
}
|
|
@@ -469,6 +503,16 @@ for phase in $PHASES; do
|
|
|
469
503
|
# token furnace). A launch that writes no report at all is a CRASH, not a
|
|
470
504
|
# check failure — it halts immediately so crash-resume re-runs it next boot.
|
|
471
505
|
_pstart="$(date +%s 2>/dev/null || echo 0)" # phase wall-clock start (runner-measured)
|
|
506
|
+
# STEP 5 — per-phase model routing. Resolve the intended model ONCE (map value via
|
|
507
|
+
# model_for_phase, or "-"/"" when unmapped). `_route_model` is what THIS attempt
|
|
508
|
+
# launches with; on a routed STARTUP failure it is blanked for a single unrouted
|
|
509
|
+
# retry (the fallback), recorded `routed → fallback` in the receipt. All routing
|
|
510
|
+
# lives here + at the launch below — nowhere else (FR-198/NFR-044).
|
|
511
|
+
_intended_model="$(model_for_phase "$phase" "$report")"
|
|
512
|
+
_route_model="$_intended_model"
|
|
513
|
+
[ "$_route_model" = "-" ] && _route_model=""
|
|
514
|
+
_fallback=no
|
|
515
|
+
_routed_fallback_done=0
|
|
472
516
|
attempt=1
|
|
473
517
|
max_attempt=2
|
|
474
518
|
verdict=""
|
|
@@ -477,6 +521,11 @@ for phase in $PHASES; do
|
|
|
477
521
|
"$phase" "$idx" "$total" "$LAUNCHER" "$attempt" >&2
|
|
478
522
|
(
|
|
479
523
|
cd "$WORKTREE"
|
|
524
|
+
# STEP 5 routing: prepend `--model <value>` iff this phase is routed. Built via
|
|
525
|
+
# `set --` so an unrouted launch passes NO --model flag — the dispatch args are
|
|
526
|
+
# then byte-for-byte today's (the absent-map neg control). Empty `"$@"` expands
|
|
527
|
+
# to nothing; a set value is properly word-safe.
|
|
528
|
+
if [ -n "$_route_model" ]; then set -- --model "$_route_model"; else set --; fi
|
|
480
529
|
# A headless, zero-touch phase has no TTY to approve tool use, so the per-tool
|
|
481
530
|
# permission gate is bypassed — otherwise the very first Write (the work, or
|
|
482
531
|
# the mandatory phase-report.yml) blocks forever waiting for an approval that
|
|
@@ -489,7 +538,7 @@ for phase in $PHASES; do
|
|
|
489
538
|
FORGE_SLICE_PHASE="$phase" \
|
|
490
539
|
FORGE_SLICE_PHASE_INDEX="$idx" \
|
|
491
540
|
FORGE_SLICE_ATTEMPT="$attempt" \
|
|
492
|
-
"$LAUNCHER" -p \
|
|
541
|
+
"$LAUNCHER" -p "$@" \
|
|
493
542
|
--output-format json \
|
|
494
543
|
--max-turns "$MAX_TURNS" \
|
|
495
544
|
--max-budget-usd "$MAX_BUDGET" \
|
|
@@ -498,12 +547,28 @@ for phase in $PHASES; do
|
|
|
498
547
|
< "$prompt" \
|
|
499
548
|
> "$wdir/result.json" 2> "$wdir/launch.err"
|
|
500
549
|
) || {
|
|
501
|
-
printf '[runner] phase %s: launcher exited non-zero (attempt %s)\n'
|
|
550
|
+
printf '[runner] phase %s: launcher exited non-zero (attempt %s%s)\n' \
|
|
551
|
+
"$phase" "$attempt" "${_route_model:+, --model $_route_model}" >&2
|
|
502
552
|
}
|
|
503
553
|
|
|
504
554
|
# Record this attempt's session id — the fresh-context proof the retry asserts.
|
|
505
555
|
extract_session "$wdir/result.json" > "$wdir/attempt-$attempt.session"
|
|
506
556
|
|
|
557
|
+
# STEP 5 fallback — a ROUTED launch that wrote NO report is a startup failure
|
|
558
|
+
# (e.g. an unusable model). Retry ONCE unrouted (blank _route_model), recorded
|
|
559
|
+
# `routed → fallback` in the receipt. This is a DISTINCT axis from the
|
|
560
|
+
# checks-failed retry: it does not consume `attempt`, and a bad model degrades
|
|
561
|
+
# to the default rather than halting the slice (routing must be harmless). An
|
|
562
|
+
# unrouted launch that writes no report is a genuine crash → the halt below.
|
|
563
|
+
if [ ! -f "$report" ] && [ -n "$_route_model" ] && [ "$_routed_fallback_done" -eq 0 ]; then
|
|
564
|
+
printf '[runner] phase %s: routed launch (--model %s) wrote no report — falling back to default (unrouted), one retry\n' \
|
|
565
|
+
"$phase" "$_route_model" >&2
|
|
566
|
+
_routed_fallback_done=1
|
|
567
|
+
_fallback=yes
|
|
568
|
+
_route_model=""
|
|
569
|
+
continue
|
|
570
|
+
fi
|
|
571
|
+
|
|
507
572
|
# (4) REQUIRE the on-disk phase-report.yml. Absent = crash → halt now (do NOT
|
|
508
573
|
# spend the retry on a crash; the resume path re-runs it from files).
|
|
509
574
|
if [ ! -f "$report" ]; then
|
|
@@ -556,20 +621,59 @@ for phase in $PHASES; do
|
|
|
556
621
|
# spend seam), one row per phase. Sourced for FIDELITY (live walk found the
|
|
557
622
|
# phase-report's self-reported usage was all zeros — a phase can't know its
|
|
558
623
|
# own token count):
|
|
559
|
-
# tokens
|
|
560
|
-
# wall_clock_s
|
|
561
|
-
# retry_count
|
|
562
|
-
# model
|
|
563
|
-
#
|
|
624
|
+
# tokens = result.json input_tokens + output_tokens (launcher truth)
|
|
625
|
+
# wall_clock_s = runner-measured elapsed for the phase (incl. any retry)
|
|
626
|
+
# retry_count = runner-observed (attempt-1), not the phase's self-report
|
|
627
|
+
# model = intended model (kept for back-compat with the 5-col reader)
|
|
628
|
+
# intended_model = what routing ASKED for (map value, "-" when unmapped)
|
|
629
|
+
# actual_model = the PRIMARY model actually used (result.json modelUsage,
|
|
630
|
+
# max outputTokens) — reality, not intent
|
|
631
|
+
# fallback = yes when a routed launch fell back unrouted (routed → fallback)
|
|
632
|
+
# cost_usd = per-phase spend (result.json total_cost_usd) — the spend
|
|
633
|
+
# the by-phase-type-and-model attribution surface sums
|
|
564
634
|
_tok_in="$(usage_num "$wdir/result.json" input_tokens)"
|
|
565
635
|
_tok_out="$(usage_num "$wdir/result.json" output_tokens)"
|
|
566
636
|
_u_tokens="$(awk -v a="$_tok_in" -v b="$_tok_out" 'BEGIN{printf "%d", (a+0)+(b+0)}')"
|
|
567
637
|
_now="$(date +%s 2>/dev/null || echo 0)"
|
|
568
638
|
if [ "$_now" -ge "$_pstart" ] 2>/dev/null; then _u_wall=$(( _now - _pstart )); else _u_wall=0; fi
|
|
569
639
|
_u_retry=$(( attempt - 1 ))
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
640
|
+
_u_intended="${_intended_model:--}"
|
|
641
|
+
_u_actual="$(actual_model_from_result "$wdir/result.json")"
|
|
642
|
+
_u_model="$_u_intended" # 5-col `model` = intended (back-compat)
|
|
643
|
+
_u_cost="$_cost" # per-phase spend (result.json total_cost_usd)
|
|
644
|
+
[ -f "$USAGE_FILE" ] || printf 'phase_name\ttokens\twall_clock_s\tmodel\tretry_count\tintended_model\tactual_model\tfallback\tcost_usd\n' > "$USAGE_FILE"
|
|
645
|
+
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
|
646
|
+
"$phase" "$_u_tokens" "$_u_wall" "$_u_model" "$_u_retry" "$_u_intended" "$_u_actual" "$_fallback" "$_u_cost" >> "$USAGE_FILE"
|
|
647
|
+
|
|
648
|
+
# (4b) FEED THE M24 OUTCOME LADDER (step-5 B4). Append ONE outcome line per ROUTED
|
|
649
|
+
# phase so the ladder accumulates real per-(model, task_type) volume at last.
|
|
650
|
+
# The ladder answers "should I route task_type T to model X?", so it is keyed on
|
|
651
|
+
# the model the operator MAPPED — the intended model — in ONE consistent
|
|
652
|
+
# vocabulary for BOTH outcomes (a routing that held and one that fell back share
|
|
653
|
+
# a key, so success/failure of the same mapped model aggregate instead of
|
|
654
|
+
# fragmenting). Recorded only when there is a routing decision to judge:
|
|
655
|
+
# • unrouted (no phase_type / no map entry) → SKIP: the default model's baseline
|
|
656
|
+
# is not a routing choice.
|
|
657
|
+
# • park → SKIP: an operator interrupt is not a model-quality outcome.
|
|
658
|
+
# • routed → fallback → first_try_pass=false (the mapped model failed to run).
|
|
659
|
+
# • routed & ran → first_try_pass = attempt 1 AND terminal-good AND no checks failed.
|
|
660
|
+
# ADVISORY: the helper always exits 0; guard so a missing helper never touches the
|
|
661
|
+
# slice. The routing map is consumed READ-ONLY throughout the runner; only the
|
|
662
|
+
# operator ever edits it by hand (D-F) — no code path here modifies that config.
|
|
663
|
+
_mo_type="$(phase_type_of "$phase")"
|
|
664
|
+
if [ -n "$_mo_type" ] && [ -n "$_intended_model" ] && [ "$_intended_model" != "-" ] \
|
|
665
|
+
&& [ "$verdict" != park ] && [ -x "$SELF_DIR/forge-model-outcome.sh" ]; then
|
|
666
|
+
_mo_ftp=false
|
|
667
|
+
if [ "$_fallback" = no ] && [ "$attempt" -eq 1 ] \
|
|
668
|
+
&& { [ "$verdict" = proceed ] || [ "$verdict" = done ]; } && [ "$cfailed" -eq 0 ]; then
|
|
669
|
+
_mo_ftp=true
|
|
670
|
+
fi
|
|
671
|
+
"$SELF_DIR/forge-model-outcome.sh" record \
|
|
672
|
+
--repo "$(basename "$REPO_ROOT" 2>/dev/null || echo slice)" \
|
|
673
|
+
--skill executing --task-type "$_mo_type" \
|
|
674
|
+
--model "$_intended_model" --first-try-pass "$_mo_ftp" \
|
|
675
|
+
--check-exit "$cfailed" --cost "$_u_cost" --tokens "$_u_tokens" >/dev/null 2>&1 || true
|
|
676
|
+
fi
|
|
573
677
|
|
|
574
678
|
# (5) BRANCH on verdict. proceed advances; done ends with the done ping.
|
|
575
679
|
# park is STUBBED here (plan-48 wires the real §8 interrupt ping) — explicit,
|
|
@@ -103,6 +103,30 @@ else
|
|
|
103
103
|
fail "case6 recommend missing insufficient-data message: $out"
|
|
104
104
|
fi
|
|
105
105
|
|
|
106
|
+
# --- Case 7: cost_usd + tokens are stored numeric, null when omitted (m-42) --
|
|
107
|
+
d="$ROOT/case7"; mkdir -p "$d"
|
|
108
|
+
HOME="$d" "$HELPER" record --repo x --skill executing --task-type verifying \
|
|
109
|
+
--model haiku --first-try-pass true --check-exit 0 --cost 0.0333 --tokens 686 >/dev/null
|
|
110
|
+
HOME="$d" "$HELPER" record --repo x --skill executing --task-type docs \
|
|
111
|
+
--model sonnet --first-try-pass true >/dev/null # no cost/tokens → null
|
|
112
|
+
log="$d/.forge/model-outcomes.jsonl"
|
|
113
|
+
if grep -q '"cost_usd":0.0333' "$log" && grep -q '"tokens":686' "$log"; then
|
|
114
|
+
pass "case7 record stores numeric cost_usd + tokens"
|
|
115
|
+
else
|
|
116
|
+
fail "case7 cost/tokens not stored numeric: $(cat "$log")"
|
|
117
|
+
fi
|
|
118
|
+
if grep -q '"cost_usd":null' "$log" && grep -q '"tokens":null' "$log"; then
|
|
119
|
+
pass "case7 omitted cost/tokens store as null (lazy migration)"
|
|
120
|
+
else
|
|
121
|
+
fail "case7 omitted cost/tokens not null: $(cat "$log")"
|
|
122
|
+
fi
|
|
123
|
+
out="$(HOME="$d" "$HELPER" recommend)"
|
|
124
|
+
if printf '%s' "$out" | grep -qE 'spend=\$0\.0333'; then
|
|
125
|
+
pass "case7 recommend surfaces per-bucket spend"
|
|
126
|
+
else
|
|
127
|
+
fail "case7 recommend missing spend column: $out"
|
|
128
|
+
fi
|
|
129
|
+
|
|
106
130
|
# --- Report -----------------------------------------------------------------
|
|
107
131
|
printf '\n%d passed, %d failed\n' "$PASSED" "$FAILED"
|
|
108
132
|
[ "$FAILED" -eq 0 ]
|
|
@@ -193,6 +193,7 @@ Plan naming reflects the slice: `plan-01-user-signs-up.md`, not `plan-01-models.
|
|
|
193
193
|
- `{id}`=milestone, `{phase}`=phase# (preserved verbatim, NOT renumbered), `{name}`=kebab, `{NN}`=seq
|
|
194
194
|
- Ex: `.forge/phases/milestone-3/2-providers/plan-01.md`
|
|
195
195
|
2. Frontmatter: phase, plan#, wave, deps, `slice_exception:` (optional, see Core Principle), `integration_checkpoint:` (optional, see below)
|
|
196
|
+
- **`phase_type:` (per-phase model routing, m-42).** Stamp the workflow phase this plan SERVES — one of the `model_by_phase_type` keys (`researching | discussing | architecting | planning | executing | verifying | reviewing`). Build plans are almost always `executing`; a plan that authors a verify/review phase's work is `verifying`/`reviewing`. This is what lets the headless slice runner route each phase to the model mapped for its type (the map ships commented → routing is off until the operator activates it, so stamping this is always safe and never changes behavior on its own). Empty ⇒ that phase runs unrouted (inherits the default model). Emit it on every plan; when unsure, `executing`.
|
|
196
197
|
3. must_haves:
|
|
197
198
|
- **Truths:** User-observable outcomes (3-7). MUST be phrased as something the user can see, click, or receive -- not "model X exists" or "table Y created".
|
|
198
199
|
- Each truth SHOULD carry a `check:` command authored NOW, goal-backward, before any code exists -- a check written before the code cannot be retro-fitted to what got built (semantics in `.forge/templates/plan.md`).
|
|
@@ -35,6 +35,7 @@ SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
|
35
35
|
REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)"
|
|
36
36
|
|
|
37
37
|
LOGS=""
|
|
38
|
+
USAGES=""
|
|
38
39
|
MARKER_DIR_OVERRIDE=""
|
|
39
40
|
LIVE_WINDOW=120
|
|
40
41
|
SINCE=""
|
|
@@ -43,6 +44,8 @@ DRY_RUN=0
|
|
|
43
44
|
while [ $# -gt 0 ]; do
|
|
44
45
|
case "$1" in
|
|
45
46
|
--log) LOGS="${LOGS}${LOGS:+
|
|
47
|
+
}${2:-}"; shift 2 ;;
|
|
48
|
+
--usage) USAGES="${USAGES}${USAGES:+
|
|
46
49
|
}${2:-}"; shift 2 ;;
|
|
47
50
|
--marker-dir) MARKER_DIR_OVERRIDE="${2:-}"; shift 2 ;;
|
|
48
51
|
--live-window) LIVE_WINDOW="${2:-120}"; shift 2 ;;
|
|
@@ -82,9 +85,36 @@ GLOBBED_EOF
|
|
|
82
85
|
fi
|
|
83
86
|
fi
|
|
84
87
|
|
|
88
|
+
# ── resolve the usage.tsv set (spend receipts): explicit --usage wins; else disk-glob ────────────
|
|
89
|
+
# Same slice worktrees as the logs, sibling file runner-work/usage.tsv. No new store —
|
|
90
|
+
# these ARE the per-phase receipts the runner already writes (step-5 spend attribution).
|
|
91
|
+
if [ -z "$USAGES" ]; then
|
|
92
|
+
if [ -n "$WT_ROOT" ]; then
|
|
93
|
+
for _wt in "$WT_ROOT"/*/; do
|
|
94
|
+
[ -d "$_wt" ] || continue
|
|
95
|
+
_u="${_wt}.forge/runner-work/usage.tsv"
|
|
96
|
+
[ -f "$_u" ] && USAGES="${USAGES}${USAGES:+
|
|
97
|
+
}$_u"
|
|
98
|
+
done
|
|
99
|
+
else
|
|
100
|
+
_uglobbed="$(git -C "$REPO_ROOT" worktree list --porcelain 2>/dev/null | awk '
|
|
101
|
+
/^worktree / { p = substr($0, 10) }
|
|
102
|
+
/^branch / { if ($2 ~ /refs\/heads\/forge\/m-/) print p "/.forge/runner-work/usage.tsv" }')"
|
|
103
|
+
while IFS= read -r _u; do
|
|
104
|
+
[ -n "$_u" ] || continue
|
|
105
|
+
[ -f "$_u" ] && USAGES="${USAGES}${USAGES:+
|
|
106
|
+
}$_u"
|
|
107
|
+
done <<UGLOBBED_EOF
|
|
108
|
+
$_uglobbed
|
|
109
|
+
UGLOBBED_EOF
|
|
110
|
+
fi
|
|
111
|
+
fi
|
|
112
|
+
|
|
85
113
|
if [ "$DRY_RUN" -eq 1 ]; then
|
|
86
114
|
printf 'forge-jarvis-instruments (--dry-run): resolved logs:\n'
|
|
87
115
|
if [ -n "$LOGS" ]; then printf '%s\n' "$LOGS" | sed 's/^/ /'; else printf ' (none)\n'; fi
|
|
116
|
+
printf 'forge-jarvis-instruments (--dry-run): resolved usage receipts:\n'
|
|
117
|
+
if [ -n "$USAGES" ]; then printf '%s\n' "$USAGES" | sed 's/^/ /'; else printf ' (none)\n'; fi
|
|
88
118
|
exit 0
|
|
89
119
|
fi
|
|
90
120
|
|
|
@@ -146,4 +176,41 @@ awk -F'\t' -v total="$_total" -v win="$LIVE_WINDOW" '
|
|
|
146
176
|
printf " (proxy — derived from relay latency, not an uptime store)\n"
|
|
147
177
|
}' "$TMP"
|
|
148
178
|
|
|
179
|
+
# ── 5. Spend by phase and model (step-5 attribution) ─────────────────────────────────────────────
|
|
180
|
+
# Answers "what did this cost, by phase and model?" WITHOUT opening a file — computed
|
|
181
|
+
# live from the runner's usage.tsv receipts (9-col step-5 schema: …actual_model fallback
|
|
182
|
+
# cost_usd). NO new store. Legacy 5-col receipts have no model/cost columns → skipped with
|
|
183
|
+
# a note. Groups by (phase, actual_model), sums cost, flags routed→fallback rows.
|
|
184
|
+
printf '\n5. Spend by phase and model (from usage.tsv receipts):\n'
|
|
185
|
+
printf '%s\n' "$USAGES" | while IFS= read -r U; do
|
|
186
|
+
[ -n "$U" ] && [ -f "$U" ] || continue
|
|
187
|
+
_hdr="$(head -1 "$U")"
|
|
188
|
+
case "$_hdr" in
|
|
189
|
+
*actual_model*cost_usd*)
|
|
190
|
+
# LC_NUMERIC=C so awk parses AND prints the `.`-decimal cost regardless of locale
|
|
191
|
+
# (a comma locale otherwise reads "0.63" as 0 and prints "0,63" — same fix the runner uses).
|
|
192
|
+
LC_NUMERIC=C awk -F'\t' -v src="$U" '
|
|
193
|
+
NR==1 { next }
|
|
194
|
+
NF>=9 && $1!="" {
|
|
195
|
+
key = $1 SUBSEP $7 # phase SUBSEP actual_model
|
|
196
|
+
cost[key] += ($9 + 0)
|
|
197
|
+
if ($8 == "yes") fb[key] += 1
|
|
198
|
+
seen[key] = 1
|
|
199
|
+
}
|
|
200
|
+
END {
|
|
201
|
+
for (k in seen) {
|
|
202
|
+
split(k, a, SUBSEP)
|
|
203
|
+
printf " %-16s %-26s $%.4f%s\n", a[1], a[2], cost[k], (fb[k] ? " (routed->fallback x" fb[k] ")" : "")
|
|
204
|
+
}
|
|
205
|
+
}' "$U"
|
|
206
|
+
;;
|
|
207
|
+
*)
|
|
208
|
+
printf ' (legacy 5-col receipt %s — no model/cost columns; re-run to populate)\n' "$U" ;;
|
|
209
|
+
esac
|
|
210
|
+
done
|
|
211
|
+
# Note when no receipts were found at all (a clean "nothing ran yet", not an error).
|
|
212
|
+
if [ -z "$USAGES" ]; then
|
|
213
|
+
printf ' (no usage.tsv receipts found — no slices instrumented yet)\n'
|
|
214
|
+
fi
|
|
215
|
+
|
|
149
216
|
exit 0
|
|
@@ -60,5 +60,28 @@ printf '%s\n' "$OUT" | grep -qE '2/3 relayed live' && ok "presence proxy:
|
|
|
60
60
|
# 5. the presence line labels itself a proxy (never a measured SLA)
|
|
61
61
|
printf '%s\n' "$OUT" | grep -qiE 'proxy' && ok "presence figure is labelled a proxy, not an SLA" || bad "proxy label" "$OUT"
|
|
62
62
|
|
|
63
|
+
# ── spend-by-phase-and-model section (step-5 attribution, from usage.tsv receipts) ──
|
|
64
|
+
# A hand-built 9-col usage.tsv: two models, one routed→fallback row. The section must
|
|
65
|
+
# answer "what did this cost, by phase and model" from the receipts — no new store.
|
|
66
|
+
USG="$ROOT/usage.tsv"
|
|
67
|
+
{
|
|
68
|
+
printf 'phase_name\ttokens\twall_clock_s\tmodel\tretry_count\tintended_model\tactual_model\tfallback\tcost_usd\n'
|
|
69
|
+
printf 'phase-1\t1000\t2\tsonnet\t0\tsonnet\tclaude-sonnet-5\tno\t0.6300\n'
|
|
70
|
+
printf 'phase-2\t500\t1\thaiku\t0\thaiku\tclaude-haiku-4-5-20251001\tno\t0.0040\n'
|
|
71
|
+
printf 'phase-3\t800\t3\topus\t0\topus\tclaude-sonnet-5\tyes\t0.4000\n'
|
|
72
|
+
} > "$USG"
|
|
73
|
+
|
|
74
|
+
SOUT="$(sh "$INSTR" --log "$LOG" --marker-dir "$MK" --usage "$USG" 2>/dev/null)"
|
|
75
|
+
|
|
76
|
+
printf '%s\n' "$SOUT" | grep -qE '5\. Spend by phase and model' && ok "spend section renders" || bad "spend section" "$SOUT"
|
|
77
|
+
printf '%s\n' "$SOUT" | grep -qE 'phase-1.*claude-sonnet-5.*\$0\.6300' && ok "spend: phase-1 on claude-sonnet-5 = \$0.6300" || bad "spend phase-1" "$SOUT"
|
|
78
|
+
printf '%s\n' "$SOUT" | grep -qE 'phase-2.*claude-haiku-4-5.*\$0\.0040' && ok "spend: phase-2 on claude-haiku = \$0.0040" || bad "spend phase-2" "$SOUT"
|
|
79
|
+
printf '%s\n' "$SOUT" | grep -qE 'phase-3.*routed->fallback' && ok "spend: phase-3 flagged routed->fallback" || bad "spend fallback flag" "$SOUT"
|
|
80
|
+
|
|
81
|
+
# neg control — the numbers are COMPUTED at render; no new stored aggregate file exists.
|
|
82
|
+
grep -rqiE 'spend[-_]?store|aggregate\.tsv|spend\.db|cost[-_]cache' "$SCRIPT_DIR/../forge-jarvis-instruments.sh" \
|
|
83
|
+
&& bad "no-new-store neg control" "found a spend-store reference" \
|
|
84
|
+
|| ok "no new spend store — computed from usage.tsv at render"
|
|
85
|
+
|
|
63
86
|
printf '\n%s: %d passed, %d failed\n' "$(basename "$0")" "$PASSED" "$FAILED"
|
|
64
87
|
[ "$FAILED" = 0 ]
|
|
@@ -13,6 +13,11 @@ phase: 1
|
|
|
13
13
|
phase_name: ""
|
|
14
14
|
plan: 1
|
|
15
15
|
type: execute # execute | tdd
|
|
16
|
+
phase_type: "" # the workflow phase this plan SERVES — one of the model_by_phase_type keys:
|
|
17
|
+
# researching | discussing | architecting | planning | executing | verifying | reviewing
|
|
18
|
+
# Drives per-phase model routing (m-42): the slice runner reads this to pick the phase's
|
|
19
|
+
# model from config.model_by_phase_type. Build plans are usually `executing`; a verify/review
|
|
20
|
+
# phase plan is `verifying`/`reviewing`. Empty = unrouted (inherits the default model).
|
|
16
21
|
wave: 1 # Execution wave (1 = no dependencies)
|
|
17
22
|
depends_on: [] # Plan IDs that must complete first
|
|
18
23
|
autonomous: true # false if contains checkpoints
|
|
@@ -45,18 +45,27 @@ divergence_budget_lines: 4000
|
|
|
45
45
|
# without touching the loop.
|
|
46
46
|
notify_channel: pushnotification
|
|
47
47
|
|
|
48
|
-
# --- model-by-phase-type map (
|
|
48
|
+
# --- model-by-phase-type map (per-phase model routing — step 5) --------------
|
|
49
49
|
|
|
50
|
-
#
|
|
51
|
-
# to
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
50
|
+
# Maps a phase's `phase_type:` (declared in its plan — the workflow phase it serves:
|
|
51
|
+
# researching|discussing|architecting|planning|executing|verifying|reviewing) to the
|
|
52
|
+
# model its headless `claude -p` launch runs on. `--model` accepts these aliases
|
|
53
|
+
# (or a full id like `claude-sonnet-5`).
|
|
54
|
+
#
|
|
55
|
+
# SHIPS COMMENTED = OFF BY DEFAULT. An absent/empty map reproduces today's behavior
|
|
56
|
+
# BYTE-FOR-BYTE: every phase inherits the default model, no `--model` flag is added.
|
|
57
|
+
# UNCOMMENT an entry to activate routing for that phase-type — that first uncomment
|
|
58
|
+
# is your moment to confirm it does what you expect (nothing chooses for you; the map
|
|
59
|
+
# is yours and only you edit it — no code ever writes it). A routed launch that fails
|
|
60
|
+
# at startup falls back once to the default and records `routed → fallback` in the
|
|
61
|
+
# usage receipt, so a bad value degrades loudly, never silently.
|
|
62
|
+
#
|
|
63
|
+
# Suggested starting point (uncomment the lines you want):
|
|
55
64
|
model_by_phase_type:
|
|
56
|
-
researching: sonnet
|
|
57
|
-
discussing: sonnet
|
|
58
|
-
architecting: opus
|
|
59
|
-
planning: opus
|
|
60
|
-
executing: sonnet
|
|
61
|
-
verifying: haiku
|
|
62
|
-
reviewing: sonnet
|
|
65
|
+
# researching: sonnet
|
|
66
|
+
# discussing: sonnet
|
|
67
|
+
# architecting: opus
|
|
68
|
+
# planning: opus
|
|
69
|
+
# executing: sonnet
|
|
70
|
+
# verifying: haiku
|
|
71
|
+
# reviewing: sonnet
|