@heytherevibin/skillforge 0.7.0 → 0.10.0

package/python/app/weights_cli.py

@@ -0,0 +1,152 @@
+"""Export / import per-user skill_weights rows (JSON snapshot)."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from pathlib import Path
+
+from app.db_paths import resolve_orchestrator_db
+from app.main import init_db
+
+
+def _parse_args(argv: list[str] | None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Export or import learned skill_weights (uses, thumbs, routing bias).")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    ex = sub.add_parser("export", help="Dump skill_weights rows to JSON (stdout unless -o).")
+    ex.add_argument("-o", "--output", type=Path, default=None, help="Output file (default: stdout).")
+    ex.add_argument("--user-id", default="", help="Logical user id (default '' = global row set).")
+    ex.add_argument(
+        "--project-root",
+        default="",
+        help="Resolve DB from <root>/.skillforge/orchestrator.db (else env / global).",
+    )
+
+    im = sub.add_parser("import", help="Load JSON snapshot into skill_weights.")
+    im.add_argument("file", type=Path, help="JSON file from skillforge weights export.")
+    im.add_argument(
+        "--user-id",
+        default=None,
+        help="Override user_id for all imported rows (default: use file's user_id).",
+    )
+    im.add_argument(
+        "--project-root",
+        default="",
+        help="Target DB path (same as export).",
+    )
+    im.add_argument(
+        "--replace-user",
+        action="store_true",
+        help="Delete existing rows for the target user_id before import.",
+    )
+    return p.parse_args(argv)
+
+
+def export_weights(con, user_id: str) -> dict:
+    cur = con.execute(
+        """
+        SELECT skill_name, weight, uses, referenced, thumbs_up, thumbs_down, disabled, updated_at
+        FROM skill_weights WHERE user_id = ? ORDER BY skill_name
+        """,
+        (user_id,),
+    )
+    rows = []
+    for r in cur.fetchall():
+        rows.append({
+            "skill_name": r[0],
+            "weight": float(r[1]),
+            "uses": int(r[2]),
+            "referenced": int(r[3]),
+            "thumbs_up": int(r[4]),
+            "thumbs_down": int(r[5]),
+            "disabled": int(r[6]),
+            "updated_at": float(r[7]) if r[7] is not None else None,
+        })
+    return {"version": 1, "user_id": user_id, "exported_at": time.time(), "rows": rows}
+
+
+def import_weights(con, data: dict, *, user_id_override: str | None, replace_user: bool) -> int:
+    if not isinstance(data, dict):
+        raise ValueError("root must be object")
+    rows = data.get("rows")
+    if not isinstance(rows, list):
+        raise ValueError("rows must be array")
+    uid = user_id_override if user_id_override is not None else str(data.get("user_id") or "")
+    if replace_user:
+        con.execute("DELETE FROM skill_weights WHERE user_id = ?", (uid,))
+    n = 0
+    now = time.time()
+    for raw in rows:
+        if not isinstance(raw, dict):
+            continue
+        name = raw.get("skill_name")
+        if not name or not isinstance(name, str):
+            continue
+        con.execute(
+            """
+            INSERT INTO skill_weights
+            (user_id, skill_name, weight, uses, referenced, thumbs_up, thumbs_down, disabled, updated_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(user_id, skill_name) DO UPDATE SET
+                weight = excluded.weight,
+                uses = excluded.uses,
+                referenced = excluded.referenced,
+                thumbs_up = excluded.thumbs_up,
+                thumbs_down = excluded.thumbs_down,
+                disabled = excluded.disabled,
+                updated_at = excluded.updated_at
+            """,
+            (
+                uid,
+                name,
+                float(raw.get("weight", 0.0)),
+                int(raw.get("uses", 0)),
+                int(raw.get("referenced", 0)),
+                int(raw.get("thumbs_up", 0)),
+                int(raw.get("thumbs_down", 0)),
+                int(raw.get("disabled", 0)),
+                float(raw.get("updated_at") or now),
+            ),
+        )
+        n += 1
+    con.commit()
+    return n
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = _parse_args(argv)
+    pr = (getattr(args, "project_root", "") or "").strip() or None
+    db_path = resolve_orchestrator_db(pr)
+    con = init_db(db_path)
+    try:
+        if args.cmd == "export":
+            payload = export_weights(con, args.user_id)
+            text = json.dumps(payload, indent=2)
+            if args.output:
+                args.output.write_text(text + "\n", encoding="utf-8")
+                print(f"Wrote {len(payload['rows'])} rows → {args.output}", file=sys.stderr)
+            else:
+                print(text)
+            raise SystemExit(0)
+        if args.cmd == "import":
+            path = args.file.expanduser().resolve()
+            if not path.is_file():
+                print(f"skillforge weights import: not found {path}", file=sys.stderr)
+                raise SystemExit(2)
+            data = json.loads(path.read_text(encoding="utf-8"))
+            n = import_weights(
+                con,
+                data,
+                user_id_override=args.user_id,
+                replace_user=bool(args.replace_user),
+            )
+            print(f"Imported {n} row(s) into {db_path}", file=sys.stderr)
+            raise SystemExit(0)
+    finally:
+        con.close()
+
+
+if __name__ == "__main__":
+    main()

package/python/fixtures/route_eval/smoke.json

@@ -0,0 +1,18 @@
+{
+  "version": 1,
+  "defaults": {
+    "candidate_window": 22
+  },
+  "cases": [
+    {
+      "id": "python-testing",
+      "prompt": "pytest fixtures caplog and monkeypatch for an API integration test",
+      "expect_in_candidates": ["python-testing"]
+    },
+    {
+      "id": "docker-patterns",
+      "prompt": "docker compose healthcheck restart policy and rollout",
+      "expect_in_candidates": ["docker-patterns"]
+    }
+  ]
+}

package/python/requirements.txt

@@ -1,3 +1,4 @@
 anthropic>=0.39
 sentence-transformers>=2.7
 numpy>=1.26
+rank-bm25>=0.2.2

package/python/tests/test_feedback_weights.py

@@ -0,0 +1,77 @@
+"""Tests for feedback_effect snapshot and weights export/import."""
+from __future__ import annotations
+
+from app.feedback_meta import build_feedback_effect, get_skill_weight_detail
+from app.main import init_db, update_skill_stat
+from app.weights_cli import export_weights, import_weights
+
+
+def test_get_skill_weight_detail_missing(tmp_path) -> None:
+    con = init_db(tmp_path / "a.db")
+    assert get_skill_weight_detail(con, "nope", "") is None
+
+
+def test_build_feedback_effect_after_use(tmp_path) -> None:
+    con = init_db(tmp_path / "b.db")
+    update_skill_stat(con, "alpha", "uses", 1, user_id="")
+    update_skill_stat(con, "alpha", "thumbs_up", 1, user_id="")
+    fe = build_feedback_effect(con, ["alpha", "beta"], user_id="")
+    assert fe["schema"] == "feedback_effect/1"
+    assert len(fe["picked"]) == 2
+    alpha = next(p for p in fe["picked"] if p["skill"] == "alpha")
+    assert alpha["has_db_row"] is True
+    assert alpha["uses"] >= 1
+    beta = next(p for p in fe["picked"] if p["skill"] == "beta")
+    assert beta["has_db_row"] is False
+
+
+def test_weights_export_import_roundtrip(tmp_path) -> None:
+    db1 = tmp_path / "w1.db"
+    db2 = tmp_path / "w2.db"
+    con = init_db(db1)
+    update_skill_stat(con, "x-skill", "uses", 2, user_id="u1")
+    update_skill_stat(con, "x-skill", "referenced", 1, user_id="u1")
+    con.close()
+
+    con = init_db(db1)
+    blob = export_weights(con, "u1")
+    con.close()
+    assert any(r["skill_name"] == "x-skill" for r in blob["rows"])
+
+    con2 = init_db(db2)
+    n = import_weights(con2, blob, user_id_override=None, replace_user=False)
+    con2.close()
+    assert n >= 1
+
+    con2 = init_db(db2)
+    cur = con2.execute(
+        "SELECT uses, referenced FROM skill_weights WHERE user_id = ? AND skill_name = ?",
+        ("u1", "x-skill"),
+    )
+    row = cur.fetchone()
+    con2.close()
+    assert row is not None
+    assert int(row[0]) == 2
+
+
+def test_weights_import_replace_user(tmp_path) -> None:
+    db = tmp_path / "w3.db"
+    con = init_db(db)
+    update_skill_stat(con, "a", "uses", 1, user_id="")
+    update_skill_stat(con, "b", "uses", 1, user_id="")
+    con.close()
+
+    payload = {
+        "version": 1,
+        "user_id": "",
+        "rows": [{"skill_name": "only", "weight": 0.0, "uses": 1, "referenced": 0, "thumbs_up": 0, "thumbs_down": 0, "disabled": 0}],
+    }
+    con = init_db(db)
+    import_weights(con, payload, user_id_override="", replace_user=True)
+    con.close()
+
+    con = init_db(db)
+    cur = con.execute("SELECT skill_name FROM skill_weights WHERE user_id = '' ORDER BY skill_name")
+    names = [r[0] for r in cur.fetchall()]
+    con.close()
+    assert names == ["only"]

package/python/tests/test_materialize.py

@@ -0,0 +1,51 @@
+"""Tests for project bootstrap file writes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.materialize import materialize_project_files
+
+
+def test_materialize_writes_cursor_command(tmp_path: Path) -> None:
+    root = tmp_path / "proj"
+    root.mkdir()
+    out = materialize_project_files(
+        str(root),
+        ["alpha"],
+        {"alpha": "desc"},
+        merge=True,
+    )
+    rel = {Path(p).as_posix() for p in out["written"]}
+    assert ".cursor/commands/skillforge.md" in rel
+    assert ".claude/commands/skillforge.md" in rel
+    cc = root / ".claude" / "commands" / "skillforge.md"
+    assert cc.is_file()
+    cct = cc.read_text(encoding="utf-8")
+    assert "route_skills" in cct
+    assert "alpha" in cct
+    cur = root / ".cursor" / "commands" / "skillforge.md"
+    assert "alpha" in cur.read_text(encoding="utf-8")
+
+
+def test_materialize_merge_false_skips_existing_command(tmp_path: Path) -> None:
+    root = tmp_path / "proj"
+    root.mkdir()
+    cmd = root / ".cursor" / "commands" / "skillforge.md"
+    cmd.parent.mkdir(parents=True)
+    cmd.write_text("keep-me", encoding="utf-8")
+    rule = root / ".cursor" / "rules" / "skillforge.mdc"
+    rule.parent.mkdir(parents=True, exist_ok=True)
+    rule.write_text("keep-rule", encoding="utf-8")
+    ccmd = root / ".claude" / "commands" / "skillforge.md"
+    ccmd.parent.mkdir(parents=True, exist_ok=True)
+    ccmd.write_text("keep-cc", encoding="utf-8")
+    materialize_project_files(
+        str(root),
+        ["b"],
+        {},
+        merge=False,
+    )
+    assert cmd.read_text(encoding="utf-8") == "keep-me"
+    assert rule.read_text(encoding="utf-8") == "keep-rule"
+    assert ccmd.read_text(encoding="utf-8") == "keep-cc"

package/python/tests/test_mcp_contract.py

@@ -135,3 +135,120 @@ def test_build_route_skills_meta_error_field() -> None:
     )
     assert meta["error"] == "empty_prompt"
     assert meta["sources"] == []
+
+
+def test_build_route_skills_meta_includes_route_quality() -> None:
+    rq = {"schema": "route_quality/1", "picked_count": 1}
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "route_quality": rq,
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert meta.get("route_quality") == rq
+
+
+def test_build_route_skills_meta_ignores_non_dict_route_quality() -> None:
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "route_quality": "not-a-dict",
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert "route_quality" not in meta
+
+
+def test_build_route_skills_meta_includes_feedback_effect() -> None:
+    fb = {"schema": "feedback_effect/1", "picked": []}
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "feedback_effect": fb,
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert meta.get("feedback_effect") == fb
+
+
+def test_build_route_skills_meta_ignores_non_dict_feedback_effect() -> None:
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "feedback_effect": [1, 2],
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert "feedback_effect" not in meta
+
+
+def test_build_route_skills_meta_includes_routing_overlay() -> None:
+    ro = {"schema": "routing_overlay/1", "exclude_skills": ["x"]}
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "routing_overlay": ro,
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert meta.get("routing_overlay") == ro
+
+
+def test_build_route_skills_meta_ignores_non_dict_routing_overlay() -> None:
+    meta = build_route_skills_meta(
+        result={
+            "candidates": [],
+            "session_id": "s",
+            "rerouted": False,
+            "change": 0.0,
+            "route_ms": 1.0,
+            "routing_overlay": "bad",
+        },
+        picked_names=[],
+        user_id="u",
+        db_path="db.sqlite",
+        skills_map={},
+        response_text="x",
+    )
+    assert "routing_overlay" not in meta

package/python/tests/test_route_eval_harness.py

@@ -0,0 +1,45 @@
+"""Unit tests for route eval fixture matcher (no embedding load)."""
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from app.route_eval_harness import evaluate_case_result, load_eval_fixture
+
+
+def _cands(names: list[str]) -> list:
+    return [(SimpleNamespace(name=n), 0.9) for n in names]
+
+
+def test_evaluate_case_expect_in_candidates() -> None:
+    r = {"candidates": _cands(["a", "b", "python-testing"]), "picked_names": ["a"]}
+    case = {"id": "t", "prompt": "x", "expect_in_candidates": ["python-testing"]}
+    assert evaluate_case_result(r, case, defaults={"candidate_window": 10}) == []
+
+
+def test_evaluate_case_missing_candidate() -> None:
+    r = {"candidates": _cands(["x", "y"]), "picked_names": ["x"]}
+    case = {"id": "t", "prompt": "x", "expect_in_candidates": ["python-testing"]}
+    err = evaluate_case_result(r, case, defaults={"candidate_window": 5})
+    assert err and "python-testing" in err[0]
+
+
+def test_evaluate_picked_any() -> None:
+    r = {"candidates": _cands(["a", "b"]), "picked_names": ["b"]}
+    case = {"id": "t", "expect_picked_any": ["b"]}
+    assert evaluate_case_result(r, case) == []
+
+
+def test_host_shortlist_fails() -> None:
+    r = {"host_pick_shortlist": True, "candidates": [], "picked_names": []}
+    err = evaluate_case_result(r, {"id": "h"}, defaults={})
+    assert any("host shortlist" in e for e in err)
+
+
+def test_load_fixture(tmp_path) -> None:
+    p = tmp_path / "f.json"
+    p.write_text(
+        '{"version":1,"cases":[{"prompt":"hi","expect_in_candidates":["z"]}]}',
+        encoding="utf-8",
+    )
+    data = load_eval_fixture(p)
+    assert len(data["cases"]) == 1

package/python/tests/test_route_policies.py

@@ -0,0 +1,115 @@
+"""Tests for route policy loading and merge."""
+from __future__ import annotations
+
+import pytest
+
+from app.main import Skill, init_db
+from app.route_policies import load_route_policies_config, merge_policy_includes
+
+
+@pytest.fixture
+def skill_alpha() -> Skill:
+    return Skill(
+        name="alpha-skill",
+        title="Alpha",
+        description="test",
+        body="body",
+        source="bundled",
+    )
+
+
+def test_merge_adds_on_regex_match(tmp_path, skill_alpha, monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES", raising=False)
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES_FILE", raising=False)
+    con = init_db(tmp_path / "x.db")
+    policies = {"rules": [{"if_text_matches": r"(?i)oauth", "include": ["alpha-skill"]}]}
+    by_name = {skill_alpha.name: skill_alpha}
+    merged, audit = merge_policy_includes(
+        "Fix OAuth callback",
+        ["other-skill"],
+        policies,
+        by_name,
+        con,
+        "",
+        max_active=7,
+    )
+    assert merged[0] == "other-skill"
+    assert "alpha-skill" in merged
+    assert any(r.get("effect") == "added" for r in audit)
+
+
+def test_merge_unknown_skill_audited(tmp_path, skill_alpha, monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES", raising=False)
+    con = init_db(tmp_path / "y.db")
+    policies = {"rules": [{"if_text_matches": "auth", "include": ["missing"]}]}
+    by_name = {skill_alpha.name: skill_alpha}
+    merged, audit = merge_policy_includes(
+        "auth bug",
+        [],
+        policies,
+        by_name,
+        con,
+        "",
+        max_active=7,
+    )
+    assert merged == []
+    assert any(r.get("effect") == "unknown_skill" for r in audit)
+
+
+def test_merge_respects_max_active(tmp_path, skill_alpha, monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES", raising=False)
+    con = init_db(tmp_path / "z.db")
+    policies = {"rules": [{"if_text_matches": "x", "include": ["alpha-skill"]}]}
+    by_name = {skill_alpha.name: skill_alpha}
+    picked = ["a", "b", "c", "d", "e", "f", "g"]
+    merged, audit = merge_policy_includes(
+        "x",
+        picked,
+        policies,
+        by_name,
+        con,
+        "",
+        max_active=7,
+    )
+    assert len(merged) == 7
+    assert "alpha-skill" not in merged
+    assert any(r.get("effect") == "skipped_max_active" for r in audit)
+
+
+def test_load_from_project_file(tmp_path, monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES", raising=False)
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES_FILE", raising=False)
+    p = tmp_path / "skillforge-policies.json"
+    p.write_text(
+        '{"rules": [{"if_text_matches": "hi", "include": ["z"]}]}',
+        encoding="utf-8",
+    )
+    root = str(tmp_path)
+    cfg = load_route_policies_config(root)
+    assert len(cfg.get("rules") or []) == 1
+
+
+def test_load_inline_env_json(monkeypatch) -> None:
+    monkeypatch.setenv(
+        "SKILLFORGE_ROUTE_POLICIES",
+        '{"rules": [{"if_text_matches": "a", "include": ["b"]}]}',
+    )
+    cfg = load_route_policies_config(None)
+    assert cfg["rules"][0]["include"] == ["b"]
+
+
+def test_invalid_regex_recorded(tmp_path, skill_alpha, monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES", raising=False)
+    con = init_db(tmp_path / "r.db")
+    policies = {"rules": [{"if_text_matches": "(bad[regex", "include": ["alpha-skill"]}]}
+    by_name = {skill_alpha.name: skill_alpha}
+    _m, audit = merge_policy_includes(
+        "x",
+        [],
+        policies,
+        by_name,
+        con,
+        "",
+        max_active=7,
+    )
+    assert any(r.get("effect") == "invalid_regex" for r in audit)