@heytherevibin/skillforge 0.8.0 → 0.10.1

package/python/app/routing_signals.py

@@ -7,6 +7,8 @@ from typing import Any, Protocol
 
 import numpy as np
 
+from app.route_quality import coerce_route_float
+
 _TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9_\-./]{2,}", re.I)
 
 
@@ -93,3 +95,61 @@ def keyword_overlap_scores(route_query: str, skill_cards: list[str]) -> np.ndarr
         ct = set(tokenize_skills_query(card))
         out.append(float(len(qt & ct)))
     return np.array(out, dtype=np.float64)
+
+
+def host_pick_shortlist_lines(
+    *,
+    prompt: str,
+    route_query: str,
+    facet_rows: list[dict[str, Any]],
+    max_candidates: int | None = None,
+    line_chars: int | None = None,
+) -> tuple[str, list[dict[str, Any]]]:
+    """Tight numbered list + structured rows for MCP host-pick phase (no in-process LLM)."""
+    mc = max_candidates
+    if mc is None:
+        mc = max(3, int(os.getenv("SKILLFORGE_HOST_PICK_MAX", "12")))
+    lc = line_chars if line_chars is not None else int(os.getenv("SKILLFORGE_HOST_PICK_LINE_CHARS", "120"))
+    prompt_one = (prompt or "").strip().replace("\n", " ")
+    if len(prompt_one) > 160:
+        prompt_one = prompt_one[:157] + "…"
+    rows_out: list[dict[str, Any]] = []
+    lines: list[str] = [
+        "# Host pick — choose skill names only from this list",
+        "",
+        f"Task: {prompt_one}",
+        "",
+        "Reply with JSON only:",
+        '{"picked": ["exact-skill-id", ...], "reasoning": "one line"}',
+        f"Use 0–{mc} names from the numbered lines only (empty picked is allowed). Copy names exactly.",
+        "",
+        "```",
+    ]
+    for i, f in enumerate(facet_rows[:mc], start=1):
+        name = str(f.get("name") or "")
+        cos = coerce_route_float(f.get("cosine_similarity"))
+        card = f"{f.get('title') or name}: {(f.get('description_preview') or '')[:lc]}".replace("\n", " ").strip()
+        if len(card) > lc:
+            card = card[: lc - 1] + "…"
+        line = f"{i:>2}. {name} | cos={cos:.3f} | {card}"
+        lines.append(line)
+        rows_out.append({
+            "id": name,
+            "rank": i,
+            "name": name,
+            "cosine_similarity": round(cos, 6),
+            "routing_score": f.get("routing_score"),
+            "sparse_signal": f.get("sparse_signal"),
+            "learned_weight": f.get("learned_weight"),
+            "router_hybrid": f.get("router_hybrid"),
+            "source": f.get("source"),
+            "one_liner": card,
+            "rationale_one_liner": card,
+        })
+    lines.append("```")
+    rq = (route_query or "").strip()
+    if len(rq) > 400:
+        rq = rq[:397] + "…"
+    if rq:
+        lines.extend(["", f"_Retrieval query:_ {rq}"])
+    return "\n".join(lines), rows_out

package/python/app/weights_cli.py

@@ -0,0 +1,152 @@
+"""Export / import per-user skill_weights rows (JSON snapshot)."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from app.db_paths import resolve_orchestrator_db
+from app.main import init_db
+
+
+def _parse_args(argv: list[str] | None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Export or import learned skill_weights (uses, thumbs, routing bias).")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    ex = sub.add_parser("export", help="Dump skill_weights rows to JSON (stdout unless -o).")
+    ex.add_argument("-o", "--output", type=Path, default=None, help="Output file (default: stdout).")
+    ex.add_argument("--user-id", default="", help="Logical user id (default '' = global row set).")
+    ex.add_argument(
+        "--project-root",
+        default="",
+        help="Resolve DB from <root>/.skillforge/orchestrator.db (else env / global).",
+    )
+
+    im = sub.add_parser("import", help="Load JSON snapshot into skill_weights.")
+    im.add_argument("file", type=Path, help="JSON file from skillforge weights export.")
+    im.add_argument(
+        "--user-id",
+        default=None,
+        help="Override user_id for all imported rows (default: use file's user_id).",
+    )
+    im.add_argument(
+        "--project-root",
+        default="",
+        help="Target DB path (same as export).",
+    )
+    im.add_argument(
+        "--replace-user",
+        action="store_true",
+        help="Delete existing rows for the target user_id before import.",
+    )
+    return p.parse_args(argv)
+
+
+def export_weights(con, user_id: str) -> dict:
+    cur = con.execute(
+        """
+        SELECT skill_name, weight, uses, referenced, thumbs_up, thumbs_down, disabled, updated_at
+        FROM skill_weights WHERE user_id = ? ORDER BY skill_name
+        """,
+        (user_id,),
+    )
+    rows = []
+    for r in cur.fetchall():
+        rows.append({
+            "skill_name": r[0],
+            "weight": float(r[1]),
+            "uses": int(r[2]),
+            "referenced": int(r[3]),
+            "thumbs_up": int(r[4]),
+            "thumbs_down": int(r[5]),
+            "disabled": int(r[6]),
+            "updated_at": float(r[7]) if r[7] is not None else None,
+        })
+    return {"version": 1, "user_id": user_id, "exported_at": time.time(), "rows": rows}
+
+
+def import_weights(con, data: dict, *, user_id_override: str | None, replace_user: bool) -> int:
+    if not isinstance(data, dict):
+        raise ValueError("root must be object")
+    rows = data.get("rows")
+    if not isinstance(rows, list):
+        raise ValueError("rows must be array")
+    uid = user_id_override if user_id_override is not None else str(data.get("user_id") or "")
+    if replace_user:
+        con.execute("DELETE FROM skill_weights WHERE user_id = ?", (uid,))
+    n = 0
+    now = time.time()
+    for raw in rows:
+        if not isinstance(raw, dict):
+            continue
+        name = raw.get("skill_name")
+        if not name or not isinstance(name, str):
+            continue
+        con.execute(
+            """
+            INSERT INTO skill_weights
+            (user_id, skill_name, weight, uses, referenced, thumbs_up, thumbs_down, disabled, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(user_id, skill_name) DO UPDATE SET
+                weight = excluded.weight,
+                uses = excluded.uses,
+                referenced = excluded.referenced,
+                thumbs_up = excluded.thumbs_up,
+                thumbs_down = excluded.thumbs_down,
+                disabled = excluded.disabled,
+                updated_at = excluded.updated_at
+            """,
+            (
+                uid,
+                name,
+                float(raw.get("weight", 0.0)),
+                int(raw.get("uses", 0)),
+                int(raw.get("referenced", 0)),
+                int(raw.get("thumbs_up", 0)),
+                int(raw.get("thumbs_down", 0)),
+                int(raw.get("disabled", 0)),
+                float(raw.get("updated_at") or now),
+            ),
+        )
+        n += 1
+    con.commit()
+    return n
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = _parse_args(argv)
+    pr = (getattr(args, "project_root", "") or "").strip() or None
+    db_path = resolve_orchestrator_db(pr)
+    con = init_db(db_path)
+    try:
+        if args.cmd == "export":
+            payload = export_weights(con, args.user_id)
+            text = json.dumps(payload, indent=2)
+            if args.output:
+                args.output.write_text(text + "\n", encoding="utf-8")
+                print(f"Wrote {len(payload['rows'])} rows → {args.output}", file=sys.stderr)
+            else:
+                print(text)
+            raise SystemExit(0)
+        if args.cmd == "import":
+            path = args.file.expanduser().resolve()
+            if not path.is_file():
+                print(f"skillforge weights import: not found {path}", file=sys.stderr)
+                raise SystemExit(2)
+            data = json.loads(path.read_text(encoding="utf-8"))
+            n = import_weights(
+                con,
+                data,
+                user_id_override=args.user_id,
+                replace_user=bool(args.replace_user),
+            )
+            print(f"Imported {n} row(s) into {db_path}", file=sys.stderr)
+            raise SystemExit(0)
+    finally:
+        con.close()
+
+
+if __name__ == "__main__":
+    main()

package/python/fixtures/route_eval/smoke.json

@@ -0,0 +1,18 @@
+{
+  "version": 1,
+  "defaults": {
+    "candidate_window": 22
+  },
+  "cases": [
+    {
+      "id": "python-testing",
+      "prompt": "pytest fixtures caplog and monkeypatch for an API integration test",
+      "expect_in_candidates": ["python-testing"]
+    },
+    {
+      "id": "docker-patterns",
+      "prompt": "docker compose healthcheck restart policy and rollout",
+      "expect_in_candidates": ["docker-patterns"]
+    }
+  ]
+}

package/python/tests/test_feedback_weights.py

@@ -0,0 +1,77 @@
+"""Tests for feedback_effect snapshot and weights export/import."""
+from __future__ import annotations
+
+from app.feedback_meta import build_feedback_effect, get_skill_weight_detail
+from app.main import init_db, update_skill_stat
+from app.weights_cli import export_weights, import_weights
+
+
+def test_get_skill_weight_detail_missing(tmp_path) -> None:
+    con = init_db(tmp_path / "a.db")
+    assert get_skill_weight_detail(con, "nope", "") is None
+
+
+def test_build_feedback_effect_after_use(tmp_path) -> None:
+    con = init_db(tmp_path / "b.db")
+    update_skill_stat(con, "alpha", "uses", 1, user_id="")
+    update_skill_stat(con, "alpha", "thumbs_up", 1, user_id="")
+    fe = build_feedback_effect(con, ["alpha", "beta"], user_id="")
+    assert fe["schema"] == "feedback_effect/1"
+    assert len(fe["picked"]) == 2
+    alpha = next(p for p in fe["picked"] if p["skill"] == "alpha")
+    assert alpha["has_db_row"] is True
+    assert alpha["uses"] >= 1
+    beta = next(p for p in fe["picked"] if p["skill"] == "beta")
+    assert beta["has_db_row"] is False
+
+
+def test_weights_export_import_roundtrip(tmp_path) -> None:
+    db1 = tmp_path / "w1.db"
+    db2 = tmp_path / "w2.db"
+    con = init_db(db1)
+    update_skill_stat(con, "x-skill", "uses", 2, user_id="u1")
+    update_skill_stat(con, "x-skill", "referenced", 1, user_id="u1")
+    con.close()
+
+    con = init_db(db1)
+    blob = export_weights(con, "u1")
+    con.close()
+    assert any(r["skill_name"] == "x-skill" for r in blob["rows"])
+
+    con2 = init_db(db2)
+    n = import_weights(con2, blob, user_id_override=None, replace_user=False)
+    con2.close()
+    assert n >= 1
+
+    con2 = init_db(db2)
+    cur = con2.execute(
+        "SELECT uses, referenced FROM skill_weights WHERE user_id = ? AND skill_name = ?",
+        ("u1", "x-skill"),
+    )
+    row = cur.fetchone()
+    con2.close()
+    assert row is not None
+    assert int(row[0]) == 2
+
+
+def test_weights_import_replace_user(tmp_path) -> None:
+    db = tmp_path / "w3.db"
+    con = init_db(db)
+    update_skill_stat(con, "a", "uses", 1, user_id="")
+    update_skill_stat(con, "b", "uses", 1, user_id="")
+    con.close()
+
+    payload = {
+        "version": 1,
+        "user_id": "",
+        "rows": [{"skill_name": "only", "weight": 0.0, "uses": 1, "referenced": 0, "thumbs_up": 0, "thumbs_down": 0, "disabled": 0}],
+    }
+    con = init_db(db)
+    import_weights(con, payload, user_id_override="", replace_user=True)
+    con.close()
+
+    con = init_db(db)
+    cur = con.execute("SELECT skill_name FROM skill_weights WHERE user_id = '' ORDER BY skill_name")
+    names = [r[0] for r in cur.fetchall()]
+    con.close()
+    assert names == ["only"]

package/python/tests/test_materialize.py

@@ -0,0 +1,51 @@
+"""Tests for project bootstrap file writes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.materialize import materialize_project_files
+
+
+def test_materialize_writes_cursor_command(tmp_path: Path) -> None:
+    root = tmp_path / "proj"
+    root.mkdir()
+    out = materialize_project_files(
+        str(root),
+        ["alpha"],
+        {"alpha": "desc"},
+        merge=True,
+    )
+    rel = {Path(p).as_posix() for p in out["written"]}
+    assert ".cursor/commands/skillforge.md" in rel
+    assert ".claude/commands/skillforge.md" in rel
+    cc = root / ".claude" / "commands" / "skillforge.md"
+    assert cc.is_file()
+    cct = cc.read_text(encoding="utf-8")
+    assert "route_skills" in cct
+    assert "alpha" in cct
+    cur = root / ".cursor" / "commands" / "skillforge.md"
+    assert "alpha" in cur.read_text(encoding="utf-8")
+
+
+def test_materialize_merge_false_skips_existing_command(tmp_path: Path) -> None:
+    root = tmp_path / "proj"
+    root.mkdir()
+    cmd = root / ".cursor" / "commands" / "skillforge.md"
+    cmd.parent.mkdir(parents=True)
+    cmd.write_text("keep-me", encoding="utf-8")
+    rule = root / ".cursor" / "rules" / "skillforge.mdc"
+    rule.parent.mkdir(parents=True, exist_ok=True)
+    rule.write_text("keep-rule", encoding="utf-8")
+    ccmd = root / ".claude" / "commands" / "skillforge.md"
+    ccmd.parent.mkdir(parents=True, exist_ok=True)
+    ccmd.write_text("keep-cc", encoding="utf-8")
+    materialize_project_files(
+        str(root),
+        ["b"],
+        {},
+        merge=False,
+    )
+    assert cmd.read_text(encoding="utf-8") == "keep-me"
+    assert rule.read_text(encoding="utf-8") == "keep-rule"
+    assert ccmd.read_text(encoding="utf-8") == "keep-cc"

package/python/tests/test_mcp_contract.py

@@ -135,3 +135,120 @@ def test_build_route_skills_meta_error_field() -> None:
     )
     assert meta["error"] == "empty_prompt"
     assert meta["sources"] == []
+
+
+def test_build_route_skills_meta_includes_route_quality() -> None:
+    rq = {"schema": "route_quality/1", "picked_count": 1}
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "route_quality": rq,
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert meta.get("route_quality") == rq
+
+
+def test_build_route_skills_meta_ignores_non_dict_route_quality() -> None:
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "route_quality": "not-a-dict",
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert "route_quality" not in meta
+
+
+def test_build_route_skills_meta_includes_feedback_effect() -> None:
+    fb = {"schema": "feedback_effect/1", "picked": []}
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "feedback_effect": fb,
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert meta.get("feedback_effect") == fb
+
+
+def test_build_route_skills_meta_ignores_non_dict_feedback_effect() -> None:
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "feedback_effect": [1, 2],
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert "feedback_effect" not in meta
+
+
+def test_build_route_skills_meta_includes_routing_overlay() -> None:
+    ro = {"schema": "routing_overlay/1", "exclude_skills": ["x"]}
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "routing_overlay": ro,
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert meta.get("routing_overlay") == ro
+
+
+def test_build_route_skills_meta_ignores_non_dict_routing_overlay() -> None:
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "routing_overlay": "bad",
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert "routing_overlay" not in meta

package/python/tests/test_route_eval_harness.py

@@ -0,0 +1,45 @@
+"""Unit tests for route eval fixture matcher (no embedding load)."""
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from app.route_eval_harness import evaluate_case_result, load_eval_fixture
+
+
+def _cands(names: list[str]) -> list:
+    return [(SimpleNamespace(name=n), 0.9) for n in names]
+
+
+def test_evaluate_case_expect_in_candidates() -> None:
+    r = {"candidates": _cands(["a", "b", "python-testing"]), "picked_names": ["a"]}
+    case = {"id": "t", "prompt": "x", "expect_in_candidates": ["python-testing"]}
+    assert evaluate_case_result(r, case, defaults={"candidate_window": 10}) == []
+
+
+def test_evaluate_case_missing_candidate() -> None:
+    r = {"candidates": _cands(["x", "y"]), "picked_names": ["x"]}
+    case = {"id": "t", "prompt": "x", "expect_in_candidates": ["python-testing"]}
+    err = evaluate_case_result(r, case, defaults={"candidate_window": 5})
+    assert err and "python-testing" in err[0]
+
+
+def test_evaluate_picked_any() -> None:
+    r = {"candidates": _cands(["a", "b"]), "picked_names": ["b"]}
+    case = {"id": "t", "expect_picked_any": ["b"]}
+    assert evaluate_case_result(r, case) == []
+
+
+def test_host_shortlist_fails() -> None:
+    r = {"host_pick_shortlist": True, "candidates": [], "picked_names": []}
+    err = evaluate_case_result(r, {"id": "h"}, defaults={})
+    assert any("host shortlist" in e for e in err)
+
+
+def test_load_fixture(tmp_path) -> None:
+    p = tmp_path / "f.json"
+    p.write_text(
+        '{"version":1,"cases":[{"prompt":"hi","expect_in_candidates":["z"]}]}',
+        encoding="utf-8",
+    )
+    data = load_eval_fixture(p)
+    assert len(data["cases"]) == 1

package/python/tests/test_route_quality.py

@@ -0,0 +1,120 @@
+"""Tests for route_quality telemetry (robust coercion, hybrid diagnostics)."""
+from __future__ import annotations
+
+from app.route_quality import (
+    build_route_quality,
+    coerce_route_float,
+    policy_includes_added_count,
+    top1_cosine_vs_routing_agreement,
+)
+
+
+def test_coerce_route_float() -> None:
+    assert coerce_route_float("0.5") == 0.5
+    assert coerce_route_float(None) == 0.0
+    assert coerce_route_float("nope") == 0.0
+    assert coerce_route_float(float("nan")) == 0.0
+    assert coerce_route_float(float("inf")) == 0.0
+    assert coerce_route_float(1.0, default=-1.0) == 1.0
+
+
+def test_policy_includes_added_count() -> None:
+    assert policy_includes_added_count(None) == 0
+    assert policy_includes_added_count([{"effect": "added"}, {"effect": "skip"}, "bad", {}]) == 1
+
+
+def test_top1_agreement() -> None:
+    assert top1_cosine_vs_routing_agreement([]) is None
+    assert top1_cosine_vs_routing_agreement([{"name": "a"}]) is None
+    facets = [
+        {"name": "high_route", "cosine_similarity": 0.1, "routing_score": 1.0},
+        {"name": "high_cos", "cosine_similarity": 0.9, "routing_score": 0.2},
+    ]
+    assert top1_cosine_vs_routing_agreement(facets) is False
+    facets2 = [
+        {"name": "winner", "cosine_similarity": 0.9, "routing_score": 1.0},
+        {"name": "lose", "cosine_similarity": 0.1, "routing_score": 0.2},
+    ]
+    assert top1_cosine_vs_routing_agreement(facets2) is True
+
+
+def test_build_route_quality_empty_facets() -> None:
+    rq = build_route_quality(
+        facet_list=[],
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=float("nan"),
+        policy_rules_loaded="bogus",
+        policy_audit=None,
+        host_picked=False,
+        pick_path="embedding_top",
+    )
+    assert rq["shortlist"]["size"] == 0
+    assert rq["shortlist"]["top_cosine_similarity"] is None
+    assert rq["session"]["change_jaccard"] == 0.0
+    assert rq["policy"]["rules_loaded"] == 0
+
+
+def test_build_route_quality_malformed_metrics() -> None:
+    facets = [
+        {"name": "a", "cosine_similarity": "bad", "routing_score": float("nan")},
+        {"name": "b", "cosine_similarity": 0.5, "routing_score": 0.2},
+    ]
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="host",
+        router_hybrid="weighted",
+        picked_names=["a"],
+        rerouted=True,
+        change=0.25,
+        policy_rules_loaded=-3,
+        policy_audit=[{"effect": "added"}, {"effect": "added"}],
+        host_picked=False,
+        pick_path="haiku_pick",
+    )
+    assert rq["shortlist"]["top_cosine_similarity"] == 0.0
+    assert rq["shortlist"]["top_routing_score"] == 0.0
+    assert rq["shortlist"]["second_cosine_similarity"] == 0.5
+    assert rq["shortlist"]["cosine_margin"] == round(-0.5, 6)
+    assert rq["shortlist"]["top1_dense_and_fused_agree"] is False
+    assert rq["policy"]["rules_loaded"] == 0
+    assert rq["policy"]["includes_added"] == 2
+    assert rq["picked_count"] == 1
+
+
+def test_build_route_quality_hybrid_off_skips_agree() -> None:
+    facets = [
+        {"name": "x", "cosine_similarity": 0.1},
+        {"name": "y", "cosine_similarity": 0.9},
+    ]
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=0,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="embedding_top",
+    )
+    assert rq["shortlist"]["top1_dense_and_fused_agree"] is None
+
+
+def test_build_route_quality_rules_loaded_ok() -> None:
+    rq = build_route_quality(
+        facet_list=[],
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=12,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="host_shortlist",
+    )
+    assert rq["policy"]["rules_loaded"] == 12