ai-consultants 2.19.2 → 2.20.0

package/scripts/lib/routing.sh

@@ -35,6 +35,10 @@ _AFFINITY_DATA=""
 # portable workaround. At ~135 entries x ~25 chars = ~3KB, memory is a
 # non-issue.
 _AFFINITY_RESULT_CACHE=" "
+# Parallel per-key cache for capability lookups: " CONSULTANT|AXIS=score".
+# Same leading-space bracketing trick as _AFFINITY_RESULT_CACHE to avoid
+# substring collisions. Reset alongside it when the affinity file changes.
+_CAPABILITY_RESULT_CACHE=" "
 
 # Resolve the affinity file path using the same search precedence as user
 # config (v2.12+): explicit AFFINITY_FILE > user config dir > bundled default.
@@ -66,6 +70,7 @@ _load_affinity_data() {
         fi
         _AFFINITY_LOADED_FILE="$file"
         _AFFINITY_RESULT_CACHE=" "
+        _CAPABILITY_RESULT_CACHE=" "
     fi
 }
 
@@ -116,6 +121,60 @@ get_affinity() {
     echo "$score"
 }
 
+# Get a consultant's capability score on a quality/efficiency axis.
+# Usage: get_capability <consultant> <axis>   (axis: intelligence|taste|cost)
+# Scores are 1-10 (higher = better). A missing consultant/axis, missing
+# 'capabilities' block, or missing jq falls back to capability_default (5).
+# Cached per (consultant, axis), mirroring get_affinity.
+get_capability() {
+    local consultant="$1"
+    local axis="$2"
+
+    _load_affinity_data
+
+    local key=" ${consultant}|${axis}="
+    case "$_CAPABILITY_RESULT_CACHE" in
+        *"${key}"*)
+            local cached="${_CAPABILITY_RESULT_CACHE#*"${key}"}"
+            echo "${cached%% *}"
+            return 0
+            ;;
+    esac
+
+    if [[ -z "$_AFFINITY_DATA" ]] || ! command -v jq >/dev/null 2>&1; then
+        echo "${CAPABILITY_DEFAULT:-5}"
+        return 0
+    fi
+
+    local score
+    score=$(jq -r --arg c "$consultant" --arg ax "$axis" '
+        (.capabilities[$c][$ax]) // (.capability_default // 5)
+    ' <<<"$_AFFINITY_DATA" 2>/dev/null)
+
+    [[ -z "$score" || "$score" == "null" ]] && score="${CAPABILITY_DEFAULT:-5}"
+    _CAPABILITY_RESULT_CACHE+="${key#" "}${score} "
+    echo "$score"
+}
+
+# Get the quality axis a question category stresses (intelligence|taste).
+# Usage: get_category_axis <category>
+# Unmapped categories (and missing JSON/jq) default to "intelligence".
+get_category_axis() {
+    local category="$1"
+
+    _load_affinity_data
+
+    if [[ -z "$_AFFINITY_DATA" ]] || ! command -v jq >/dev/null 2>&1; then
+        echo "intelligence"
+        return 0
+    fi
+
+    local axis
+    axis=$(jq -r --arg cat "$category" '(.category_axis[$cat]) // "intelligence"' <<<"$_AFFINITY_DATA" 2>/dev/null)
+    [[ -z "$axis" || "$axis" == "null" ]] && axis="intelligence"
+    echo "$axis"
+}
+
 # Select the best consultants for a category
 # Usage: select_consultants <category> [min_affinity] [max_consultants]
 # Returns: list of consultants sorted by affinity
@@ -129,12 +188,26 @@ select_consultants() {
     local selected=()
     local scores=()
 
-    # Collect affinities
+    # Collect affinities. When capability-aware routing is on, the ELIGIBILITY
+    # filter still uses raw category affinity (same panel as before), but the
+    # SORT rank is nudged by the consultant's capability on the axis this
+    # category stresses — so under a max/limit the quality-axis reorders which
+    # eligible consultants make the cut. Default (flag off): rank == affinity.
+    local _cap_axis=""
+    if [[ "${ENABLE_CAPABILITY_ROUTING:-false}" == "true" ]]; then
+        _cap_axis=$(get_category_axis "$category")
+    fi
     for c in "${consultants[@]}"; do
         local score=$(get_affinity "$category" "$c")
         if [[ $score -ge $min_affinity ]]; then
+            local rank=$score
+            if [[ -n "$_cap_axis" ]]; then
+                local cap
+                cap=$(get_capability "$c" "$_cap_axis")
+                rank=$(( score + cap - ${CAPABILITY_DEFAULT:-5} ))
+            fi
             selected+=("$c")
-            scores+=("$score")
+            scores+=("$rank")
         fi
     done
 

package/scripts/lib/voting.sh

@@ -8,6 +8,44 @@
 # CONFIDENCE-WEIGHTED VOTING
 # =============================================================================
 
+# Resolve the capability-weighting axis for the current run, or "" when
+# capability weighting is off or the routing helpers are unavailable (e.g.
+# voting.sh sourced standalone). Reads the run category from QUESTION_CATEGORY.
+# Centralized so each voting function computes the axis once, not per response.
+_voting_capability_axis() {
+    if [[ "${ENABLE_CAPABILITY_WEIGHTING:-false}" != "true" ]]; then
+        echo ""
+        return 0
+    fi
+    if ! declare -f get_capability >/dev/null 2>&1 || ! declare -f get_category_axis >/dev/null 2>&1; then
+        echo ""
+        return 0
+    fi
+    get_category_axis "${QUESTION_CATEGORY:-GENERAL}"
+}
+
+# Effective vote weight for a consultant: its self-reported confidence,
+# optionally modulated by how capable it is on the axis this question's
+# category stresses (capability-aware voting). Degrades to plain confidence
+# when no axis is supplied (feature off / helpers unavailable).
+# Usage: _effective_vote_weight <consultant> <confidence> <axis-or-empty>
+#   weight = confidence * (STRENGTH + capability) / STRENGTH   (integer math)
+#   STRENGTH = CAPABILITY_WEIGHT_STRENGTH (default 10): higher = gentler nudge.
+_effective_vote_weight() {
+    local consultant="$1" confidence="$2" axis="$3"
+    if [[ -z "$axis" ]]; then
+        echo "$confidence"
+        return 0
+    fi
+    local cap strength
+    cap=$(get_capability "$consultant" "$axis")
+    [[ "$cap" =~ ^[0-9]+$ ]] || cap="${CAPABILITY_DEFAULT:-5}"
+    strength="${CAPABILITY_WEIGHT_STRENGTH:-10}"
+    [[ "$strength" =~ ^[0-9]+$ ]] || strength=10
+    [[ "$strength" -le 0 ]] && strength=10
+    echo $(( confidence * (strength + cap) / strength ))
+}
+
 # Calculate the weighted average of confidence scores
 # Usage: calculate_weighted_average <json_responses_dir>
 calculate_weighted_average() {
@@ -178,17 +216,22 @@ calculate_weighted_recommendation() {
     map_clear "APPROACH_WEIGHTS"
     map_clear "APPROACH_SUPPORTERS"
 
+    # Resolve the capability-weighting axis once (empty when weighting is off)
+    local _cap_axis
+    _cap_axis=$(_voting_capability_axis)
+
     # Collect weights for each approach
     for f in "$responses_dir"/*.json; do
         if [[ -f "$f" && -s "$f" ]]; then
-            local consultant approach confidence current_weight current_supporters
+            local consultant approach confidence current_weight current_supporters weight
             consultant=$(jq -r '.consultant // "unknown"' "$f" 2>/dev/null)
             approach=$(jq -r '.response.approach // "unknown"' "$f" 2>/dev/null)
             confidence=$(jq -r '.confidence.score // 5' "$f" 2>/dev/null)
 
-            # Add weight
+            # Add weight (capability-modulated when ENABLE_CAPABILITY_WEIGHTING)
+            weight=$(_effective_vote_weight "$consultant" "$confidence" "$_cap_axis")
             current_weight=$(map_get "APPROACH_WEIGHTS" "$approach")
-            map_set "APPROACH_WEIGHTS" "$approach" "$((${current_weight:-0} + confidence))"
+            map_set "APPROACH_WEIGHTS" "$approach" "$((${current_weight:-0} + weight))"
 
             # Add supporter
             current_supporters=$(map_get "APPROACH_SUPPORTERS" "$approach")
@@ -257,19 +300,27 @@ calculate_final_score() {
 
     local weighted_sum=0
     local total_confidence=0
+    local _cap_axis
+    _cap_axis=$(_voting_capability_axis)
 
     for f in "$responses_dir"/*.json; do
         if [[ -f "$f" && -s "$f" ]]; then
+            local consultant=$(jq -r '.consultant // "unknown"' "$f" 2>/dev/null)
             local approach=$(jq -r '.response.approach // "unknown"' "$f" 2>/dev/null)
             local confidence=$(jq -r '.confidence.score // 5' "$f" 2>/dev/null)
+            local weight
+            weight=$(_effective_vote_weight "$consultant" "$confidence" "$_cap_axis")
 
-            total_confidence=$((total_confidence + confidence))
+            # Normalize by the same weight used in the numerator so the score
+            # stays on the 1-10 scale. weight == confidence unless capability
+            # weighting is enabled.
+            total_confidence=$((total_confidence + weight))
 
-            # Full weight if supports, half if neutral, zero if dissents
+            # Full weight if supports, reduced if dissents (still considered)
             if [[ "$approach" == "$winning_approach" ]]; then
-                weighted_sum=$((weighted_sum + confidence * 10))  # Full support
+                weighted_sum=$((weighted_sum + weight * 10))  # Full support
             else
-                weighted_sum=$((weighted_sum + confidence * 2))   # Dissent but consider
+                weighted_sum=$((weighted_sum + weight * 2))   # Dissent but consider
             fi
         fi
     done

package/scripts/roster_audit.sh

@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# roster_audit.sh — "uncorrelated value" audit for the consultant roster.
+#
+# A consultant earns its seat only if it contributes signal the others DON'T —
+# a distinct approach, not an echo of the panel. This is the diversity-not-
+# redundancy bar from the model-routing "before a new model earns a row" test
+# (catch >=1 thing both incumbents miss), applied to a consultation panel
+# instead of a review lane.
+#
+# For each past consultation, a consultant's approach is DISTINCT if its keyword
+# set has low Jaccard overlap (< threshold) with EVERY other consultant that
+# round. Aggregated across consultations, a consultant that is rarely distinct
+# is correlated with the rest — a candidate to drop or down-weight; one that is
+# often distinct is pulling its weight on diversity.
+#
+# Reuses the voting.sh keyword/Jaccard machinery. Read-only; no live API calls.
+#
+# Usage: roster_audit.sh [--recent N] [--threshold PCT] [--json] [dir ...]
+#   dir...         consultation output dirs (each holds <consultant>.json files)
+#   --recent N     if no dirs given, audit the N most recent under the base (20)
+#   --threshold P  Jaccard %% at/above which two approaches count as correlated (20)
+#   --json         emit JSON instead of a table
+#   -h, --help     show this header
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/voting.sh
+source "$SCRIPT_DIR/lib/voting.sh"
+
+MIN_SAMPLE=3        # participations below which a verdict is "insufficient-data"
+acc=""              # temp accumulator file (cleaned on EXIT)
+trap '[[ -n "$acc" ]] && rm -f "$acc"' EXIT
+
+# Verdict from participation count + distinctiveness percentage.
+# Usage: _verdict <participated> <pct>
+_verdict() {
+    local p="$1" pct="$2"
+    if [[ "$p" -lt "$MIN_SAMPLE" ]]; then echo "insufficient-data"; return; fi
+    if [[ "$pct" -ge 50 ]]; then echo "unique-value"
+    elif [[ "$pct" -ge 20 ]]; then echo "some-value"
+    else echo "redundant?"; fi
+}
+
+_render_table() { # <audited> <threshold> <rows>
+    local audited="$1" threshold="$2" rows="$3" redundant="" c p d pct v
+    printf 'Roster uncorrelated-value audit  (%d consultations, correlated >= %d%%)\n\n' "$audited" "$threshold"
+    printf '%-12s  %12s  %8s  %9s  %s\n' CONSULTANT PARTICIPATED DISTINCT "DISTINCT%" VERDICT
+    while read -r c p d pct; do
+        [[ -z "$c" ]] && continue
+        v=$(_verdict "$p" "$pct")
+        printf '%-12s  %12d  %8d  %8d%%  %s\n' "$c" "$p" "$d" "$pct" "$v"
+        [[ "$v" == "redundant?" ]] && redundant="${redundant:+$redundant, }$c"
+    done <<< "$rows"
+    echo
+    if [[ -n "$redundant" ]]; then
+        printf 'Possibly redundant (>=%d samples, <20%% distinct): %s\n' "$MIN_SAMPLE" "$redundant"
+        echo '  -> consider dropping or down-weighting these in references/affinity.json.'
+    else
+        echo 'No consultant flagged redundant on this sample.'
+    fi
+}
+
+_render_json() { # <audited> <threshold> <rows>
+    local audited="$1" threshold="$2" rows="$3" items="[]" c p d pct v
+    while read -r c p d pct; do
+        [[ -z "$c" ]] && continue
+        v=$(_verdict "$p" "$pct")
+        items=$(jq -c --arg c "$c" --argjson p "$p" --argjson d "$d" --argjson pct "$pct" --arg v "$v" \
+            '. + [{consultant:$c, participated:$p, distinct:$d, distinct_pct:$pct, verdict:$v}]' <<<"$items")
+    done <<< "$rows"
+    jq -n --argjson audited "$audited" --argjson threshold "$threshold" --argjson roster "$items" \
+        '{consultations_audited:$audited, correlated_threshold_pct:$threshold, roster:$roster}'
+}
+
+main() {
+    local recent=20 threshold=20 as_json=0
+    local dirs=()
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --recent)    recent="${2:-20}"; shift 2 ;;
+            --threshold) threshold="${2:-20}"; shift 2 ;;
+            --json)      as_json=1; shift ;;
+            -h|--help)   grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+            -*)          echo "unknown option: $1" >&2; exit 2 ;;
+            *)           dirs+=("$1"); shift ;;
+        esac
+    done
+
+    command -v jq >/dev/null 2>&1 || { echo "roster_audit: jq is required" >&2; exit 2; }
+
+    # No dirs given -> the N most recent consultation dirs under the base.
+    if [[ ${#dirs[@]} -eq 0 ]]; then
+        # shellcheck source=config.sh
+        source "$SCRIPT_DIR/config.sh" >/dev/null 2>&1 || true
+        local base="${DEFAULT_OUTPUT_DIR_BASE:-/tmp/ai_consultations}"
+        if [[ ! -d "$base" ]]; then
+            echo "roster_audit: no dirs given and base '$base' not found" >&2
+            exit 1
+        fi
+        while IFS= read -r d; do dirs+=("$d"); done < <(
+            find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -r | head -n "$recent"
+        )
+    fi
+
+    acc="$(mktemp "${TMPDIR:-/tmp}/roster_audit.XXXXXX")"
+
+    local audited=0 dir f
+    for dir in ${dirs[@]+"${dirs[@]}"}; do
+        [[ -d "$dir" ]] || continue
+        local names=() kw=()
+        for f in "$dir"/*.json; do
+            [[ -f "$f" && -s "$f" ]] || continue
+            local c a al
+            c=$(jq -r '.consultant // empty' "$f" 2>/dev/null)
+            [[ -z "$c" ]] && continue                       # not a consultant response file
+            a=$(jq -r '.response.approach? // "unknown"' "$f" 2>/dev/null)
+            al=$(printf '%s' "$a" | tr '[:upper:]' '[:lower:]')
+            case "$al" in unknown|""|n/a|none|"not available") continue ;; esac
+            names+=("$c"); kw+=("$(_extract_keywords "$a")")
+        done
+        local n=${#names[@]}
+        [[ $n -lt 2 ]] && continue                          # need >=2 to assess correlation
+        audited=$((audited+1))
+        local i j
+        for ((i=0; i<n; i++)); do
+            local maxsim=0 s
+            for ((j=0; j<n; j++)); do
+                [[ $i -eq $j ]] && continue
+                s=$(_jaccard_similarity "${kw[$i]}" "${kw[$j]}")
+                [[ $s -gt $maxsim ]] && maxsim=$s
+            done
+            local distinct=0
+            [[ $maxsim -lt $threshold ]] && distinct=1
+            printf '%s %d\n' "${names[$i]}" "$distinct" >> "$acc"
+        done
+    done
+
+    if [[ $audited -eq 0 ]]; then
+        echo "roster_audit: no consultations with >=2 responders found (nothing to audit)" >&2
+        exit 1
+    fi
+
+    # Aggregate: "consultant participated distinct pct", sorted by pct then distinct desc.
+    local rows
+    rows=$(awk '
+        { part[$1]++; dist[$1]+=$2 }
+        END { for (c in part){ p=part[c]; d=dist[c]; pct=(p>0)?int(d*100/p):0; printf "%s %d %d %d\n", c, p, d, pct } }
+    ' "$acc" | sort -k4,4nr -k3,3nr)
+
+    if [[ "$as_json" -eq 1 ]]; then
+        _render_json "$audited" "$threshold" "$rows"
+    else
+        _render_table "$audited" "$threshold" "$rows"
+    fi
+}
+
+main "$@"

package/scripts/roster_calibrate.sh

@@ -0,0 +1,217 @@
+#!/usr/bin/env bash
+# roster_calibrate.sh — MEASURE capability scores from consultation history,
+# replacing the heuristic seeds in references/affinity.json (Tier A).
+#
+# The three axes are derived, not guessed — each from a signal the tool already
+# records, and sliced by the SAME category_axis that consumes them:
+#
+#   intelligence = mean blind PEER-REVIEW score on intelligence-axis categories
+#   taste        = mean blind PEER-REVIEW score on taste-axis categories
+#   cost         = mean observed $/response (tokens_used x catalog rate),
+#                  rank-normalized (cheapest = 10). This is cost-per-*task*
+#                  (verbosity + completion), not per-token sticker price.
+#
+# Requires consultations run with ENABLE_PEER_REVIEW=true (peer scores) and the
+# default cost tracking (tokens_used in metadata). Read-only unless --write.
+#
+# Usage: roster_calibrate.sh [--recent N] [--json] [--write] [dir ...]
+#   dir...       consultation output dirs (each: <consultant>.json + peer_review/
+#                + optimization_metrics.json). Default: N most recent under base.
+#   --recent N   number of recent consultations to use (default 30)
+#   --json       emit {measured_capabilities, samples} JSON
+#   --write      merge measured values into references/affinity.json (backup first)
+#   -h, --help   show this header
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=lib/routing.sh
+source "$SCRIPT_DIR/lib/routing.sh"        # get_category_axis
+COST_RATES="$SCRIPT_DIR/../docs/cost_rates.json"
+AFFINITY_JSON="$SCRIPT_DIR/../references/affinity.json"
+
+# Input/output token split convention (mirrors lib/costs.sh's 60/40 estimate)
+# and the fallback rate for a model absent from the catalog (per-1K).
+SPLIT_IN=0.6; SPLIT_OUT=0.4
+FALLBACK_IN=0.005; FALLBACK_OUT=0.015
+
+qacc=""; cacc=""
+trap '[[ -n "$qacc" ]] && rm -f "$qacc"; [[ -n "$cacc" ]] && rm -f "$cacc"' EXIT
+
+# Per-response cost in dollars from a model + total token count. Rate looked up
+# case-insensitively (callers/consultants vary case); falls back if absent.
+_response_cost() {
+    local model="$1" tokens="$2" rin rout
+    read -r rin rout < <(jq -r --arg m "$model" '
+        (.models | to_entries
+         | map(select(.key | ascii_downcase == ($m | ascii_downcase)))
+         | .[0].value) as $r
+        | if $r then "\($r.input) \($r.output)" else "NA NA" end
+    ' "$COST_RATES" 2>/dev/null)
+    if [[ "$rin" == "NA" || -z "${rin:-}" ]]; then rin=$FALLBACK_IN; rout=$FALLBACK_OUT; fi
+    awk -v t="$tokens" -v i="$rin" -v o="$rout" -v si="$SPLIT_IN" -v so="$SPLIT_OUT" \
+        'BEGIN{ printf "%.8f", (t/1000)*(si*i + so*o) }'
+}
+
+# Locate a peer-review aggregate file in <dir>/peer_review by SHAPE (an array of
+# {consultant, average_peer_score}) — robust to its exact filename.
+_find_peer_file() {
+    local pdir="$1/peer_review" pf
+    [[ -d "$pdir" ]] || return 1
+    for pf in "$pdir"/*.json; do
+        [[ -f "$pf" ]] || continue
+        if jq -e 'type=="array" and length>0 and (.[0]|has("average_peer_score") and has("consultant"))' \
+            "$pf" >/dev/null 2>&1; then
+            echo "$pf"; return 0
+        fi
+    done
+    return 1
+}
+
+main() {
+    local recent=30 as_json=0 do_write=0
+    local dirs=()
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --recent) recent="${2:-30}"; shift 2 ;;
+            --json)   as_json=1; shift ;;
+            --write)  do_write=1; shift ;;
+            -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+            -*)       echo "unknown option: $1" >&2; exit 2 ;;
+            *)        dirs+=("$1"); shift ;;
+        esac
+    done
+
+    command -v jq >/dev/null 2>&1 || { echo "roster_calibrate: jq is required" >&2; exit 2; }
+    command -v awk >/dev/null 2>&1 || { echo "roster_calibrate: awk is required" >&2; exit 2; }
+
+    if [[ ${#dirs[@]} -eq 0 ]]; then
+        # shellcheck source=config.sh
+        source "$SCRIPT_DIR/config.sh" >/dev/null 2>&1 || true
+        local base="${DEFAULT_OUTPUT_DIR_BASE:-/tmp/ai_consultations}"
+        [[ -d "$base" ]] || { echo "roster_calibrate: no dirs given and base '$base' not found" >&2; exit 1; }
+        while IFS= read -r d; do dirs+=("$d"); done < <(
+            find "$base" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -r | head -n "$recent"
+        )
+    fi
+
+    qacc="$(mktemp "${TMPDIR:-/tmp}/rc_quality.XXXXXX")"   # lines: "consultant axis score"
+    cacc="$(mktemp "${TMPDIR:-/tmp}/rc_cost.XXXXXX")"      # lines: "consultant cost"
+
+    local used=0 dir
+    for dir in ${dirs[@]+"${dirs[@]}"}; do
+        [[ -d "$dir" ]] || continue
+
+        # Category -> quality axis (intelligence|taste) for this consultation.
+        local cat axis
+        cat=$(jq -r '.quality_metrics.category // empty' "$dir/optimization_metrics.json" 2>/dev/null)
+        [[ -z "$cat" ]] && cat="GENERAL"
+        axis=$(get_category_axis "$cat")
+
+        # Peer-review quality scores (blind) -> quality accumulator.
+        local peer_file
+        if peer_file=$(_find_peer_file "$dir"); then
+            while read -r c s; do
+                [[ -z "$c" || "$c" == "unknown" ]] && continue
+                echo "$c $axis $s" >> "$qacc"
+            done < <(jq -r '.[] | "\(.consultant) \(.average_peer_score)"' "$peer_file" 2>/dev/null)
+            used=$((used+1))
+        fi
+
+        # Observed cost per response -> cost accumulator.
+        local f
+        for f in "$dir"/*.json; do
+            [[ -f "$f" && -s "$f" ]] || continue
+            local c model tokens
+            c=$(jq -r '.consultant // empty' "$f" 2>/dev/null)
+            [[ -z "$c" ]] && continue
+            model=$(jq -r '.model // "unknown"' "$f" 2>/dev/null)
+            tokens=$(jq -r '.metadata.tokens_used // empty' "$f" 2>/dev/null)
+            [[ -z "$tokens" || ! "$tokens" =~ ^[0-9]+$ ]] && continue
+            echo "$c $(_response_cost "$model" "$tokens")" >> "$cacc"
+        done
+    done
+
+    if [[ $used -eq 0 ]]; then
+        echo "roster_calibrate: no consultations with peer-review data found (run with ENABLE_PEER_REVIEW=true)" >&2
+        exit 1
+    fi
+
+    # --- Aggregate quality per (consultant, axis): mean, rounded, clamped 1-10 ---
+    # Emits: "consultant intelligence <score|-> <n>  taste <score|-> <n>"
+    local quality_rows
+    quality_rows=$(awk '
+        { key=$1 SUBSEP $2; sum[key]+=$3; n[key]++; seen[$1]=1 }
+        END {
+            for (c in seen) {
+                ik=c SUBSEP "intelligence"; tk=c SUBSEP "taste"
+                iv=(ik in sum)?int(sum[ik]/n[ik]+0.5):-1; ic=(ik in n)?n[ik]:0
+                tv=(tk in sum)?int(sum[tk]/n[tk]+0.5):-1; tc=(tk in n)?n[tk]:0
+                if (iv>10) iv=10; if (iv!=-1 && iv<1) iv=1
+                if (tv>10) tv=10; if (tv!=-1 && tv<1) tv=1
+                printf "%s %s %d %s %d\n", c, (iv==-1?"-":iv), ic, (tv==-1?"-":tv), tc
+            }
+        }' "$qacc" | sort)
+
+    # --- Aggregate cost per consultant: mean, then rank-normalize (cheapest=10) ---
+    # Emits: "consultant <costscore> <n>"
+    local cost_rows
+    cost_rows=$(awk '
+        { sum[$1]+=$2; n[$1]++ }
+        END { for (c in sum) printf "%s %.10f %d\n", c, sum[c]/n[c], n[c] }
+    ' "$cacc" | sort -k2,2g | awk '
+        { c[NR]=$1; mean[NR]=$2; cnt[NR]=$3 }
+        END {
+            N=NR
+            for (r=1; r<=N; r++) {
+                if (N==1) score=5
+                else score=int(1 + 9*(N-r)/(N-1) + 0.5)   # rank 1 (cheapest) -> 10
+                print c[r], score, cnt[r]
+            }
+        }' | sort)
+
+    # --- Assemble measured_capabilities + samples JSON ---
+    local caps='{}' samples='{}' c iv ic tv tc
+    while read -r c iv ic tv tc; do
+        [[ -z "$c" ]] && continue
+        caps=$(jq -c --arg c "$c" \
+            --argjson iv "$([[ "$iv" == "-" ]] && echo null || echo "$iv")" \
+            --argjson tv "$([[ "$tv" == "-" ]] && echo null || echo "$tv")" \
+            '.[$c] = ((.[$c] // {}) + (if $iv==null then {} else {intelligence:$iv} end)
+                                   + (if $tv==null then {} else {taste:$tv} end))' <<<"$caps")
+        samples=$(jq -c --arg c "$c" --argjson ic "$ic" --argjson tc "$tc" \
+            '.[$c] = ((.[$c] // {}) + {intelligence_n:$ic, taste_n:$tc})' <<<"$samples")
+    done <<< "$quality_rows"
+
+    while read -r c score cnt; do
+        [[ -z "$c" ]] && continue
+        caps=$(jq -c --arg c "$c" --argjson v "$score" '.[$c] = ((.[$c] // {}) + {cost:$v})' <<<"$caps")
+        samples=$(jq -c --arg c "$c" --argjson n "$cnt" '.[$c] = ((.[$c] // {}) + {cost_n:$n})' <<<"$samples")
+    done <<< "$cost_rows"
+
+    local result
+    result=$(jq -n --argjson caps "$caps" --argjson samples "$samples" \
+        '{measured_capabilities:$caps, samples:$samples}')
+
+    if [[ "$do_write" -eq 1 ]]; then
+        cp "$AFFINITY_JSON" "${AFFINITY_JSON}.bak"
+        jq --argjson m "$caps" '.capabilities = ((.capabilities // {}) * $m)' "$AFFINITY_JSON" \
+            > "${AFFINITY_JSON}.tmp" && mv "${AFFINITY_JSON}.tmp" "$AFFINITY_JSON"
+        echo "Merged measured capabilities into $AFFINITY_JSON (backup: ${AFFINITY_JSON}.bak)"
+        echo "Measured $(echo "$caps" | jq 'length') consultant(s) from $used consultation(s)."
+    elif [[ "$as_json" -eq 1 ]]; then
+        echo "$result"
+    else
+        echo "Measured capabilities from $used consultation(s) with peer-review data:"
+        echo "(intelligence/taste = mean blind peer score on that axis's categories; cost = rank, cheapest=10)"
+        echo
+        echo "$result" | jq -r '
+            .measured_capabilities as $c | .samples as $s
+            | ($c | keys[]) as $k
+            | "  \($k): intelligence=\($c[$k].intelligence // "n/a") taste=\($c[$k].taste // "n/a") cost=\($c[$k].cost // "n/a")"
+              + "   (n: int=\($s[$k].intelligence_n // 0) taste=\($s[$k].taste_n // 0) cost=\($s[$k].cost_n // 0))"'
+        echo
+        echo "Apply with --write (merges into affinity.json, keeping unmeasured cells), or --json for raw."
+    fi
+}
+
+main "$@"

package/scripts/run_calibration.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# run_calibration.sh — data-collection harness for roster_calibrate.sh.
+#
+# Runs each benchmark question (references/calibration_benchmark.json) through
+# the full panel with ENABLE_PEER_REVIEW=true, collects the output dirs, then
+# calibrates measured intelligence/taste/cost scores.
+#
+# WARNING: this makes real consultant calls for EVERY question — it costs money
+# and takes time (dozens of consultations). Use --limit to sample first and
+# --dry-run to preview without running anything.
+#
+# Usage: run_calibration.sh [--limit N] [--dry-run] [--write]
+#   --limit N    only run the first N benchmark questions
+#   --dry-run    print the plan; execute nothing
+#   --write      after collecting, merge measured scores into affinity.json
+#   -h, --help   show this header
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BENCH="$SCRIPT_DIR/../references/calibration_benchmark.json"
+
+main() {
+    local limit=0 dry=0 do_write=0
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --limit)   limit="${2:-0}"; shift 2 ;;
+            --dry-run) dry=1; shift ;;
+            --write)   do_write=1; shift ;;
+            -h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+            *)         echo "unknown option: $1" >&2; exit 2 ;;
+        esac
+    done
+
+    command -v jq >/dev/null 2>&1 || { echo "run_calibration: jq is required" >&2; exit 2; }
+    [[ -f "$BENCH" ]] || { echo "run_calibration: benchmark not found: $BENCH" >&2; exit 1; }
+
+    local total
+    total=$(jq '.questions | length' "$BENCH")
+    [[ "$limit" -gt 0 && "$limit" -lt "$total" ]] && total="$limit"
+
+    echo "Calibration run: $total question(s) x full panel, ENABLE_PEER_REVIEW=true."
+    [[ "$dry" -eq 1 ]] && echo "(dry run — nothing will be executed)"
+    echo
+
+    local dirs=() i=0 q qcat outdir
+    while IFS= read -r q; do
+        [[ "$limit" -gt 0 && "$i" -ge "$limit" ]] && break
+        i=$((i+1))
+        qcat=$(jq -r ".questions[$((i-1))].category" "$BENCH")
+        printf '[%d/%d] %-12s %.70s\n' "$i" "$total" "$qcat" "$q"
+        [[ "$dry" -eq 1 ]] && continue
+        outdir=$(ENABLE_PEER_REVIEW=true "$SCRIPT_DIR/consult_all.sh" "$q" 2>/dev/null | tail -1)
+        if [[ -n "$outdir" && -d "$outdir" ]]; then
+            dirs+=("$outdir")
+        else
+            echo "     ! consultation produced no output dir — skipped"
+        fi
+    done < <(jq -r '.questions[].q' "$BENCH")
+
+    if [[ "$dry" -eq 1 ]]; then
+        echo; echo "Dry run complete. Re-run without --dry-run to collect data."
+        exit 0
+    fi
+
+    if [[ ${#dirs[@]} -eq 0 ]]; then
+        echo "No consultations succeeded — nothing to calibrate. Check ./scripts/doctor.sh --live." >&2
+        exit 1
+    fi
+
+    echo; echo "Collected ${#dirs[@]} consultation(s). Calibrating..."
+    local calib_args=()
+    [[ "$do_write" -eq 1 ]] && calib_args+=(--write)
+    "$SCRIPT_DIR/roster_calibrate.sh" ${calib_args[@]+"${calib_args[@]}"} "${dirs[@]}"
+}
+
+main "$@"