@salesforce/afv-skills 1.67.0 → 1.68.0

package/skills/agentforce-offline-experimentation-run/scripts/btlib/results.py

@@ -0,0 +1,288 @@
+"""Run-dir artifacts — the stored traces of one replay run.
+
+    units.jsonl              replayed units: request + golden (bt.unit/1); replay again with --from-units
+    results.jsonl            one line per unit x repeat: response, scores, status (bt.result/1)
+    summary.json             aggregates, run config, code fingerprint (bt.summary/1)
+    replay_results_sfap.csv  the legacy CSV columns, one row per unit x repeat, plus unit columns
+
+Run dirs hold user utterances: keep them out of git. Every writer goes through redact_obj and
+creates its file readable by its owner only.
+"""
+
+import csv
+import hashlib
+import json
+import os
+import statistics
+import subprocess
+from collections import Counter, defaultdict
+
+from . import SCHEMA_RESULT, SCHEMA_SUMMARY
+from .redact import redact_obj
+from .scoring import MATCH_BUCKETS
+from .sfap import TIMEOUT_KINDS
+
+UNIT_FIELDS = ("unit_id", "session_id", "turn", "step", "step_id", "interaction_id", "org", "agent",
+               "agent_version", "compatible_versions", "node", "step_kind", "split", "channel", "topic",
+               "user_utterance", "production")
+SCORE_FIELDS = ("decision", "names", "names_strict", "args", "text", "score", "strict", "bucket",
+                "detail", "args_detail", "judge", "schema_violations", "topic_match", "confusion",
+                "proxies")
+LEGACY_CSV_FIELDS = ["session_id", "utterance", "original_model", "replay_model", "actual_topic",
+                     "replayed_topic", "topic_match", "actual_action", "replayed_action",
+                     "function_match_score", "function_args_score", "actual_outcome",
+                     "replayed_outcome", "text_resp_score"]
+CSV_FIELDS = LEGACY_CSV_FIELDS + ["unit_id", "node", "step_kind", "repeat", "status", "bucket",
+                                  "unit_score"]
+
+
+def unit_kind(unit):
+    """router_tool | router_text | subagent_tool | subagent_text."""
+    return f"{unit.get('step_kind')}_{unit['golden']['kind']}"
+
+
+def make_record(unit, repeat, model, seed, request_sha, call, scores=None, patch=None):
+    """One results.jsonl line. A failed judge call turns the record into an infra error.
+    patch: {"base_request_sha", "ops"} of a patched run (--patch): the unpatched request's
+    sha and the ops that matched this unit."""
+    rec = {"schema": SCHEMA_RESULT}
+    rec.update({k: unit.get(k) for k in UNIT_FIELDS})
+    patch = patch or {}
+    rec.update(kind=unit_kind(unit), repeat=repeat, model=model,
+               golden_model=unit["request"].get("model"), seed=seed, request_sha=request_sha,
+               base_request_sha=patch.get("base_request_sha"), patch_ops=patch.get("ops"),
+               status=call["status"], error=call.get("error"), error_kind=call.get("error_kind"),
+               http_status=call.get("http_status"), attempts=call.get("attempts"),
+               latency_ms=call.get("latency_ms"), cached=bool(call.get("cached")))
+    resp = call.get("response")
+    rec["response"] = {k: resp.get(k) for k in ("text", "tool_calls", "finish_reason")} if resp else None
+    rec["usage"] = resp.get("usage") if resp else None
+    rec["golden"] = unit["golden"]
+    rec.update({k: None for k in SCORE_FIELDS})
+    if scores is not None:
+        if scores.get("judge_error"):
+            rec.update(status="infra_error", error_kind="judge", error=scores["judge_error"])
+        else:
+            rec.update({k: scores.get(k) for k in SCORE_FIELDS})
+    return rec
+
+
+def open_private(path, **kwargs):
+    """A file opened for writing that only its owner can read (mode 0600), before anything is written."""
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        os.chmod(path, 0o600)  # an existing file keeps its mode through os.open
+        return os.fdopen(fd, "w", encoding="utf-8", **kwargs)
+    except BaseException:
+        os.close(fd)
+        raise
+
+
+def write_jsonl(path, rows):
+    with open_private(path) as f:
+        for row in rows:
+            f.write(json.dumps(redact_obj(row), ensure_ascii=False) + "\n")
+
+
+def append_jsonl(f, row):
+    f.write(json.dumps(redact_obj(row), ensure_ascii=False) + "\n")
+    f.flush()
+
+
+def read_jsonl(path):
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def write_json(path, obj):
+    with open_private(path) as f:
+        json.dump(redact_obj(obj), f, indent=2, ensure_ascii=False)
+        f.write("\n")
+
+
+# Modules that neither replay nor score: editing them must not make two runs incomparable.
+NOT_FINGERPRINTED = ("agentpatch.py", "compare.py", "fetch.py")
+
+
+def code_fingerprint():
+    """Hash of the btlib sources that replay and score, plus the git commit, so runs scored by
+    different code are told apart."""
+    here = os.path.dirname(os.path.abspath(__file__))
+    h = hashlib.sha256()
+    for name in sorted(os.listdir(here)):
+        if name.endswith(".py") and name not in NOT_FINGERPRINTED:
+            with open(os.path.join(here, name), "rb") as f:
+                h.update(name.encode() + b"\0" + f.read() + b"\0")
+    out = {"btlib_sha256": h.hexdigest()[:16], "git": None, "git_dirty": None}
+    try:
+        git = subprocess.run(["git", "-C", here, "rev-parse", "--short=12", "HEAD"],
+                             capture_output=True, text=True, timeout=10)
+        if git.returncode == 0:
+            out["git"] = git.stdout.strip()
+            dirty = subprocess.run(["git", "-C", here, "status", "--porcelain", "--", "."],
+                                   capture_output=True, text=True, timeout=10)
+            out["git_dirty"] = bool(dirty.stdout.strip())
+    except (OSError, subprocess.SubprocessError):
+        pass
+    return out
+
+
+def aggregate(rows):
+    n = len(rows)
+    if not n:
+        return {"n": 0, "agreement": None, "strict": None, "match": 0}
+    return {"n": n,
+            "agreement": round(sum(r["score"] for r in rows) / n, 4),
+            "strict": round(sum(r["strict"] for r in rows) / n, 4),
+            "match": sum(1 for r in rows if r["bucket"] in MATCH_BUCKETS)}
+
+
+def _group(rows, keys_of):
+    groups = defaultdict(list)
+    for r in rows:
+        for k in keys_of(r):
+            groups[k].append(r)
+    out = {}
+    for k in sorted(groups):
+        agg = aggregate(groups[k])
+        agg["buckets"] = dict(Counter(r["bucket"] for r in groups[k]).most_common())
+        out[k] = agg
+    return out
+
+
+def golden_tools(rec):
+    """Tools the golden called; '(text)' for a text golden."""
+    return sorted({c["name"] for c in rec["golden"]["tool_calls"]}) or ["(text)"]
+
+
+def groundedness(rec):
+    """The category production's groundedness check gave the record's turn; '(none)' without one."""
+    return (rec.get("production") or {}).get("groundedness") or "(none)"
+
+
+def production_summary(records):
+    """Production's own checks over the turns and sessions of these records (report-only)."""
+    turns, sessions = {}, {}
+    for r in records:
+        p = r.get("production")
+        if p:
+            turns.setdefault((r["session_id"], r["turn"]), p)
+            sessions.setdefault(r["session_id"], p)
+    guardrails = defaultdict(Counter)
+    for p in turns.values():
+        for name, value in (p.get("guardrails") or {}).items():
+            guardrails[name][value] += 1
+    return {"turns": len(turns),
+            "groundedness": dict(Counter(p.get("groundedness") or "(none)" for p in turns.values()).most_common()),
+            "guardrails": {name: dict(c.most_common()) for name, c in sorted(guardrails.items())},
+            "turns_with_step_errors": sum(1 for p in turns.values() if p.get("step_errors")),
+            "sessions": len(sessions),
+            "end_type": dict(Counter(p.get("session_end_type") or "(none)" for p in sessions.values()).most_common()),
+            "sessions_with_action_errors": sum(1 for p in sessions.values() if p.get("session_action_errors"))}
+
+
+def summarize(records, config=None, skipped=None, unit_count=None):
+    ok = [r for r in records if r["status"] == "ok"]
+    infra = [r for r in records if r["status"] != "ok"]
+    n = len(records)
+    routed = [r for r in ok if r["topic_match"] is not None]
+    texts = [r for r in ok if r["proxies"]]
+    confusion = Counter(tuple(r["confusion"]) for r in ok if r["confusion"])
+    violations = Counter(v for r in ok for v in r["schema_violations"] or [])
+    return {
+        "schema": SCHEMA_SUMMARY,
+        "config": config or {},
+        "counts": {
+            "units": unit_count if unit_count is not None else len({r["unit_id"] for r in records}),
+            "records": n, "scored": len(ok), "infra_error": len(infra),
+            "cached": sum(1 for r in records if r.get("cached")),
+            "infra_rate": round(len(infra) / n, 4) if n else 0.0,
+            "infra_by_kind": dict(Counter(r["error_kind"] for r in infra).most_common()),
+            "timed_out_units": timed_out_units(records),
+            "skipped_steps": dict(skipped or {}),
+        },
+        "overall": aggregate(ok),
+        "by_kind": _group(ok, lambda r: [r["kind"]]),
+        "by_node": _group(ok, lambda r: [r["node"]]),
+        "by_tool": _group(ok, golden_tools),
+        "by_bucket": dict(Counter(r["bucket"] for r in ok).most_common()),
+        "by_split": _group(ok, lambda r: [r["split"]] if r.get("split") else []),
+        "by_groundedness": _group(ok, lambda r: [groundedness(r)]),
+        "production": production_summary(records),
+        "topic": {"n": len(routed), "match": sum(1 for r in routed if r["topic_match"]),
+                  "rate": round(sum(1 for r in routed if r["topic_match"]) / len(routed), 4) if routed else None},
+        "confusion": [{"golden": g, "candidate": c, "n": k} for (g, c), k in confusion.most_common()],
+        "schema_violations": {"records": sum(1 for r in ok if r["schema_violations"]),
+                              "notes": dict(violations.most_common())},
+        "text": {
+            "n": len(texts),
+            "median_len_ratio": round(statistics.median(r["proxies"]["len_ratio"] for r in texts), 3) if texts else None,
+            "question_ending_agrees": sum(1 for r in texts if r["proxies"]["golden_ends_with_question"]
+                                          == r["proxies"]["candidate_ends_with_question"]),
+            "raw_id_leaks": sum(1 for r in texts if r["proxies"]["candidate_raw_ids"] > r["proxies"]["golden_raw_ids"]),
+        },
+        "per_repeat": {str(rep): aggregate([r for r in ok if r["repeat"] == rep])
+                       for rep in sorted({r["repeat"] for r in records})},
+        "judge_calls": sum(len(r["judge"] or []) for r in ok),
+        "usage": _usage_totals(records),
+    }
+
+
+def timed_out_units(records):
+    """The units whose every record timed out: the model may not finish them (a runaway answer)."""
+    timed_out = defaultdict(list)
+    for r in records:
+        timed_out[r["unit_id"]].append(r["status"] != "ok" and r.get("error_kind") in TIMEOUT_KINDS)
+    return sum(1 for flags in timed_out.values() if all(flags))
+
+
+def _usage_totals(records):
+    totals = Counter()
+    for r in records:
+        for k, v in (r.get("usage") or {}).items():
+            if isinstance(v, (int, float)) and not isinstance(v, bool):
+                totals[k] += v
+    return dict(totals)
+
+
+def _names(calls):
+    return "+".join(c["name"] or "" for c in calls or [])
+
+
+def csv_row(rec):
+    """Legacy CSV columns for one record (topic columns only on router steps)."""
+    g_calls = rec["golden"]["tool_calls"]
+    resp = rec["response"] or {}
+    c_calls = resp.get("tool_calls") or []
+    ok = rec["status"] == "ok"
+    router = rec["step_kind"] == "router"
+    row = {
+        "session_id": rec["session_id"], "utterance": rec["user_utterance"],
+        "original_model": rec["golden_model"], "replay_model": rec["model"],
+        "actual_topic": _names(g_calls) if router else (rec["topic"] or ""),
+        "replayed_topic": _names(c_calls) if router and ok else "",
+        "topic_match": int(bool(rec["topic_match"])) if router and ok else "",
+        "actual_action": _names(g_calls), "replayed_action": _names(c_calls),
+        "function_match_score": "", "function_args_score": "",
+        "actual_outcome": rec["golden"]["text"], "replayed_outcome": resp.get("text") or "",
+        "text_resp_score": "",
+        "unit_id": rec["unit_id"], "node": rec["node"], "step_kind": rec["step_kind"],
+        "repeat": rec["repeat"], "status": rec["status"], "bucket": rec["bucket"] or "",
+        "unit_score": rec["score"] if ok else "",
+    }
+    if ok:
+        if g_calls or c_calls:
+            row["function_match_score"] = int(bool(g_calls and c_calls and rec["names_strict"]))
+        if rec["names"]:
+            row["function_args_score"] = int(bool(rec["args"]))
+        if not g_calls and not c_calls:
+            row["text_resp_score"] = int(bool(rec["text"]))
+    return row
+
+
+def write_csv(path, records):
+    with open_private(path, newline="") as f:
+        w = csv.DictWriter(f, fieldnames=CSV_FIELDS)
+        w.writeheader()
+        for rec in records:
+            w.writerow(redact_obj(csv_row(rec)))

package/skills/agentforce-offline-experimentation-run/scripts/btlib/scoring.py

@@ -0,0 +1,340 @@
+"""Score one replayed step against its golden (the production answer to the same request).
+
+Golden is a tool call:
+    decision  candidate also called a tool (not text)
+    names     same multiset of tool names (optionally lenient via configured equivalence groups)
+    args      every argument the same: IDs, codes, enums and numbers compared exactly,
+              missing == empty, free-text strings sent to the judge. A key one call leaves out
+              takes its default (the agent config's arg_defaults, else the schema's "default").
+              Not compared on a transition that declares no parameters (it takes none)
+    unit score 1 (all match) | 0.5 (decision + names, args differ) | 0
+Golden is text:
+    candidate must reply with text the judge finds equivalent: 1 | 0. A tool call scores 0.
+Strict score: 1 only for exact names (no equivalence groups), matching args with no default
+taken, or equivalent text.
+
+Buckets: match, transition_args (a match only because arguments on such a transition were not
+compared), wrong_args, wrong_tool, missed_call, extra_call, tool_to_text, text_to_tool,
+text_divergence.
+"""
+
+import json
+import re
+from collections import Counter
+
+from .agentsrc import TRANSITION_SUFFIX
+
+MATCH_BUCKETS = ("match", "transition_args")  # the buckets that score 1
+# Last word of a parameter name that marks an identifier (productCode, order_id, SKU, email).
+_ID_WORDS = {"id", "ids", "code", "codes", "number", "num", "sku", "email", "phone", "uuid", "url"}
+_KEY_WORDS = re.compile(r"[A-Z]?[a-z0-9]+|[A-Z]+(?![a-z])")
+_ID_VALUE = re.compile(r"^(?=\S*\d)\S+$")          # one token containing a digit: SHOE-M-088, 006Rt00000AbCdE
+_SF_ID = re.compile(r"\b(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{15}(?:[A-Za-z0-9]{3})?\b")
+_SF_ID_FULL = re.compile(r"^(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{15}(?:[A-Za-z0-9]{3})?$")
+_UUID = re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b")
+
+
+def _id_key(key):
+    words = _KEY_WORDS.findall(key or "")
+    return bool(words) and words[-1].lower() in _ID_WORDS
+
+
+def _empty(v):
+    return v is None or v == "" or v == [] or v == {}
+
+
+def _num(v):
+    if isinstance(v, bool):
+        return None
+    if isinstance(v, (int, float)):
+        return float(v)
+    if isinstance(v, str):
+        try:
+            return float(v.strip())
+        except ValueError:
+            return None
+    return None
+
+
+def _bool(v):
+    if isinstance(v, bool):
+        return v
+    if isinstance(v, str) and v.strip().lower() in ("true", "false"):
+        return v.strip().lower() == "true"
+    return None
+
+
+def values_equal(a, b):
+    """Deterministic equality with light normalisation (case/space for strings, '2' == 2).
+
+    Salesforce record ids stay case-sensitive: 15-character ids differing only in case
+    are different records.
+    """
+    if _empty(a) and _empty(b):
+        return True
+    if isinstance(a, str) and isinstance(b, str):
+        a, b = a.strip(), b.strip()
+        if _SF_ID_FULL.match(a) or _SF_ID_FULL.match(b):
+            return a == b
+        return " ".join(a.split()).lower() == " ".join(b.split()).lower()
+    na, nb = _num(a), _num(b)
+    if na is not None and nb is not None:
+        return na == nb
+    ba, bb = _bool(a), _bool(b)
+    if ba is not None and bb is not None:
+        return ba == bb
+    return json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True)
+
+
+def is_free_text(key, golden_value, candidate_value, prop_schema):
+    """Free text may be equivalent without being equal (a search phrase); IDs and enums may not."""
+    if not (isinstance(golden_value, str) and isinstance(candidate_value, str)):
+        return False
+    if (prop_schema or {}).get("enum") or (prop_schema or {}).get("pattern"):
+        return False
+    if _id_key(key):
+        return False
+    if _ID_VALUE.match(golden_value.strip()) or _ID_VALUE.match(candidate_value.strip()):
+        return False
+    return True
+
+
+def compare_args(golden_args, candidate_args, schema=None, defaults=None):
+    """{"equal": [...], "differ": [...], "free_text": [...], "defaulted": [...], "omitted": {...}}
+    over the union of keys.
+
+    A key one call leaves out (or sends empty) takes its default: defaults[key] (the agent
+    config's arg_defaults for the tool), else the schema's "default". "defaulted" lists the
+    equal keys that are equal only so; "omitted" maps each differing key one call left out to
+    that call, "golden" or "candidate".
+    """
+    props = (schema or {}).get("properties") or {}
+    known = {k: p["default"] for k, p in props.items() if isinstance(p, dict) and "default" in p}
+    known.update(defaults or {})
+    out = {"equal": [], "differ": [], "free_text": [], "defaulted": [], "omitted": {}}
+    for key in sorted(set(golden_args) | set(candidate_args)):
+        g, c = golden_args.get(key), candidate_args.get(key)
+        left_out = "golden" if _empty(g) else "candidate" if _empty(c) else None
+        if values_equal(g, c):
+            out["equal"].append(key)
+        elif left_out and key in known and values_equal(known[key], c if left_out == "golden" else g):
+            out["equal"].append(key)
+            out["defaulted"].append(key)
+        elif is_free_text(key, g, c, props.get(key)):
+            out["free_text"].append(key)
+        else:
+            out["differ"].append(key)
+            if left_out:
+                out["omitted"][key] = left_out
+    return out
+
+
+def schema_violations(calls, schemas):
+    """Problems of candidate calls against the offered tools' parameter schemas."""
+    notes = []
+    types = {"string": str, "boolean": bool, "object": dict, "array": list}
+    for call in calls:
+        name, args = call.get("name"), call.get("args") or {}
+        if "args_raw" in call:
+            notes.append(f"{name}: arguments are not a JSON object")
+        if name not in schemas:
+            notes.append(f"{name}: tool was not offered")
+            continue
+        schema = schemas[name] or {}
+        props = schema.get("properties") or {}
+        for req in schema.get("required") or []:
+            if _empty(args.get(req)):
+                notes.append(f"{name}: missing required '{req}'")
+        for key, val in args.items():
+            if props and key not in props:
+                notes.append(f"{name}: unknown parameter '{key}'")
+                continue
+            ps = props.get(key) or {}
+            t = ps.get("type")
+            if t in ("number", "integer"):
+                n = _num(val)
+                if n is None or (t == "integer" and not n.is_integer()):
+                    notes.append(f"{name}.{key}: expected {t}")
+            elif t in types and val is not None and not isinstance(val, types[t]):
+                notes.append(f"{name}.{key}: expected {t}")
+            if ps.get("enum") and val not in ps["enum"]:
+                notes.append(f"{name}.{key}: not one of the allowed values")
+            if isinstance(val, str):
+                if ps.get("pattern"):
+                    try:
+                        if not re.search(ps["pattern"], val):
+                            notes.append(f"{name}.{key}: does not match the pattern")
+                    except re.error:
+                        pass
+                if ps.get("maxLength") is not None and len(val) > ps["maxLength"]:
+                    notes.append(f"{name}.{key}: longer than maxLength {ps['maxLength']}")
+                if ps.get("minLength") is not None and len(val) < ps["minLength"]:
+                    notes.append(f"{name}.{key}: shorter than minLength {ps['minLength']}")
+    return notes
+
+
+def text_proxies(golden_text, candidate_text):
+    """Report-only text signals; not part of the unit score."""
+    def ids(t):
+        return len(_SF_ID.findall(t or "")) + len(_UUID.findall(t or ""))
+    g, c = golden_text or "", candidate_text or ""
+    return {
+        "len_ratio": round(len(c) / max(1, len(g)), 3),
+        "golden_ends_with_question": g.rstrip().endswith("?"),
+        "candidate_ends_with_question": c.rstrip().endswith("?"),
+        "golden_raw_ids": ids(g),
+        "candidate_raw_ids": ids(c),
+    }
+
+
+class Equivalence:
+    """Tool-name equivalence groups from the agent config; empty (exact names) by default."""
+
+    def __init__(self, groups=None):
+        self.canon = {}
+        for i, group in enumerate(groups or []):
+            for name in group:
+                self.canon[name] = f"group{i}:" + "|".join(sorted(group))
+
+    def key(self, name):
+        return self.canon.get(name, name)
+
+    def same(self, a, b):
+        return a == b or self.key(a) == self.key(b)
+
+
+def takes_no_args(fn):
+    """Is this offered function a transition (the platform's suffix ends its description) that
+    declares no parameters?"""
+    fn = fn or {}
+    return ((fn.get("description") or "").endswith(TRANSITION_SUFFIX)
+            and not (fn.get("parameters") or {}).get("properties"))
+
+
+def _pair_calls(golden_calls, candidate_calls, eq):
+    """Pair golden and candidate calls with equivalent names; prefer pairs whose args are equal."""
+    pairs, used = [], set()
+    for g in golden_calls:
+        options = [i for i, c in enumerate(candidate_calls)
+                   if i not in used and eq.same(g["name"], c["name"])]
+        if not options:
+            continue
+        best = next((i for i in options if values_equal(g.get("args") or {}, candidate_calls[i].get("args") or {})),
+                    options[0])
+        used.add(best)
+        pairs.append((g, candidate_calls[best]))
+    return pairs
+
+
+def score_unit(unit, response, judge=None, equivalence=None, arg_defaults=None):
+    """Score one ok response. Returns a dict; 'judge_error' is set when a needed judge call failed.
+    arg_defaults: {tool: {param: value}}, the agent config's defaults of the params a call may leave out."""
+    eq = equivalence or Equivalence()
+    arg_defaults = arg_defaults or {}
+    golden = unit["golden"]
+    msgs = unit["request"]["messages"]
+    offered = {(t.get("function") or {}).get("name"): t.get("function") or {} for t in unit["request"]["tools"]}
+    schemas = {name: fn.get("parameters") or {} for name, fn in offered.items()}
+    g_calls = golden.get("tool_calls") or []
+    c_calls = response.get("tool_calls") or []
+    c_text = response.get("text") or ""
+    out = {"golden_kind": golden["kind"], "decision": None, "names": None, "names_strict": None,
+           "args": None, "text": None, "score": 0.0, "strict": 0.0, "bucket": None,
+           "detail": "", "args_detail": [], "judge": [], "judge_error": None,
+           "schema_violations": schema_violations(c_calls, schemas) if c_calls else [],
+           "topic_match": None, "confusion": None, "proxies": None}
+
+    if unit.get("step_kind") == "router":
+        # Topic = where the router sent the turn (its go_to_* calls); text = stayed put.
+        out["topic_match"] = sorted(c["name"] for c in g_calls) == sorted(c["name"] for c in c_calls)
+
+    if golden["kind"] == "tool":
+        out["decision"] = bool(c_calls)
+        if not c_calls:
+            out["bucket"] = "tool_to_text"
+            out["detail"] = f"expected {', '.join(c['name'] for c in g_calls)}, replied with text"
+            out["confusion"] = ["+".join(sorted(c["name"] for c in g_calls)), "(text)"]
+            return out
+        g_names = Counter(eq.key(c["name"]) for c in g_calls)
+        c_names = Counter(eq.key(c["name"]) for c in c_calls)
+        out["names"] = g_names == c_names
+        out["names_strict"] = Counter(c["name"] for c in g_calls) == Counter(c["name"] for c in c_calls)
+        if not out["names"]:
+            if not (g_names - c_names) and (c_names - g_names):
+                out["bucket"] = "extra_call"
+            elif (g_names - c_names) and not (c_names - g_names):
+                out["bucket"] = "missed_call"
+            else:
+                out["bucket"] = "wrong_tool"
+            g_list = "+".join(sorted(c["name"] for c in g_calls))
+            c_list = "+".join(sorted(c["name"] for c in c_calls))
+            out["confusion"] = [g_list, c_list]
+            out["detail"] = f"expected {g_list}, got {c_list}"
+            return out
+        all_equal, not_compared, defaulted = True, [], False
+        for g, c in _pair_calls(g_calls, c_calls, eq):
+            cmp = compare_args(g.get("args") or {}, c.get("args") or {}, schemas.get(g["name"]),
+                               arg_defaults.get(g["name"]))
+            entry = {"name": g["name"], "candidate_name": c["name"], **cmp, "equivalent": None}
+            if ((cmp["differ"] or cmp["free_text"]) and "args_raw" not in c
+                    and takes_no_args(offered.get(g["name"])) and takes_no_args(offered.get(c["name"]))):
+                entry["equivalent"], entry["not_compared"] = True, True
+                not_compared += [f"{g['name']}.{k}" for k in cmp["differ"] + cmp["free_text"]]
+            elif cmp["differ"] or "args_raw" in c:
+                entry["equivalent"] = False
+            elif cmp["free_text"]:
+                if judge is None:
+                    entry["equivalent"] = False
+                    entry["judge_skipped"] = True
+                else:
+                    verdict, err = judge.args(g["name"], schemas.get(g["name"]), g.get("args") or {},
+                                              c.get("args") or {}, msgs)
+                    if verdict is None:
+                        out["judge_error"] = err
+                        return out
+                    out["judge"].append({"for": f"args:{g['name']}", **verdict})
+                    entry["equivalent"] = verdict["equivalent"]
+            else:
+                entry["equivalent"] = True
+            all_equal = all_equal and entry["equivalent"]
+            defaulted = defaulted or bool(cmp["defaulted"])
+            out["args_detail"].append(entry)
+        out["args"] = all_equal
+        if all_equal and not_compared:
+            out["bucket"], out["score"] = "transition_args", 1.0
+            out["detail"] = "arguments on a transition that takes none, not compared: " + ", ".join(not_compared)
+        elif all_equal:
+            out["bucket"], out["score"] = "match", 1.0
+            out["strict"] = 1.0 if out["names_strict"] and not defaulted else 0.0
+        else:
+            out["bucket"], out["score"] = "wrong_args", 0.5
+            differ = [f"{e['name']}.{k}" + (f" (left out by the {e['omitted'][k]})" if k in e["omitted"] else "")
+                      for e in out["args_detail"] if not e["equivalent"]
+                      for k in (e["differ"] + e["free_text"])]
+            out["detail"] = "arguments differ: " + ", ".join(differ) if differ else "arguments differ"
+        return out
+
+    # golden is text
+    out["decision"] = not c_calls
+    if c_calls:
+        out["bucket"] = "text_to_tool"
+        out["detail"] = f"expected a text reply, called {', '.join(c['name'] for c in c_calls)}"
+        out["confusion"] = ["(text)", "+".join(sorted(c["name"] for c in c_calls))]
+        return out
+    out["proxies"] = text_proxies(golden.get("text"), c_text)
+    if " ".join((golden.get("text") or "").split()) == " ".join(c_text.split()):
+        out["text"] = True
+    elif judge is None:
+        out["text"] = False
+        out["detail"] = "no judge configured"
+    else:
+        verdict, err = judge.text(golden.get("text") or "", c_text, msgs)
+        if verdict is None:
+            out["judge_error"] = err
+            return out
+        out["judge"].append({"for": "text", **verdict})
+        out["text"] = verdict["equivalent"]
+        out["detail"] = verdict.get("reasoning", "")
+    out["bucket"] = "match" if out["text"] else "text_divergence"
+    out["score"] = out["strict"] = 1.0 if out["text"] else 0.0
+    return out