@heytherevibin/skillforge 0.7.0 → 0.10.0

package/python/tests/test_route_quality.py

@@ -0,0 +1,120 @@
+"""Tests for route_quality telemetry (robust coercion, hybrid diagnostics)."""
+from __future__ import annotations
+
+from app.route_quality import (
+    build_route_quality,
+    coerce_route_float,
+    policy_includes_added_count,
+    top1_cosine_vs_routing_agreement,
+)
+
+
+def test_coerce_route_float() -> None:
+    assert coerce_route_float("0.5") == 0.5
+    assert coerce_route_float(None) == 0.0
+    assert coerce_route_float("nope") == 0.0
+    assert coerce_route_float(float("nan")) == 0.0
+    assert coerce_route_float(float("inf")) == 0.0
+    assert coerce_route_float(1.0, default=-1.0) == 1.0
+
+
+def test_policy_includes_added_count() -> None:
+    assert policy_includes_added_count(None) == 0
+    assert policy_includes_added_count([{"effect": "added"}, {"effect": "skip"}, "bad", {}]) == 1
+
+
+def test_top1_agreement() -> None:
+    assert top1_cosine_vs_routing_agreement([]) is None
+    assert top1_cosine_vs_routing_agreement([{"name": "a"}]) is None
+    facets = [
+        {"name": "high_route", "cosine_similarity": 0.1, "routing_score": 1.0},
+        {"name": "high_cos", "cosine_similarity": 0.9, "routing_score": 0.2},
+    ]
+    assert top1_cosine_vs_routing_agreement(facets) is False
+    facets2 = [
+        {"name": "winner", "cosine_similarity": 0.9, "routing_score": 1.0},
+        {"name": "lose", "cosine_similarity": 0.1, "routing_score": 0.2},
+    ]
+    assert top1_cosine_vs_routing_agreement(facets2) is True
+
+
+def test_build_route_quality_empty_facets() -> None:
+    rq = build_route_quality(
+        facet_list=[],
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=float("nan"),
+        policy_rules_loaded="bogus",
+        policy_audit=None,
+        host_picked=False,
+        pick_path="embedding_top",
+    )
+    assert rq["shortlist"]["size"] == 0
+    assert rq["shortlist"]["top_cosine_similarity"] is None
+    assert rq["session"]["change_jaccard"] == 0.0
+    assert rq["policy"]["rules_loaded"] == 0
+
+
+def test_build_route_quality_malformed_metrics() -> None:
+    facets = [
+        {"name": "a", "cosine_similarity": "bad", "routing_score": float("nan")},
+        {"name": "b", "cosine_similarity": 0.5, "routing_score": 0.2},
+    ]
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="host",
+        router_hybrid="weighted",
+        picked_names=["a"],
+        rerouted=True,
+        change=0.25,
+        policy_rules_loaded=-3,
+        policy_audit=[{"effect": "added"}, {"effect": "added"}],
+        host_picked=False,
+        pick_path="haiku_pick",
+    )
+    assert rq["shortlist"]["top_cosine_similarity"] == 0.0
+    assert rq["shortlist"]["top_routing_score"] == 0.0
+    assert rq["shortlist"]["second_cosine_similarity"] == 0.5
+    assert rq["shortlist"]["cosine_margin"] == round(-0.5, 6)
+    assert rq["shortlist"]["top1_dense_and_fused_agree"] is False
+    assert rq["policy"]["rules_loaded"] == 0
+    assert rq["policy"]["includes_added"] == 2
+    assert rq["picked_count"] == 1
+
+
+def test_build_route_quality_hybrid_off_skips_agree() -> None:
+    facets = [
+        {"name": "x", "cosine_similarity": 0.1},
+        {"name": "y", "cosine_similarity": 0.9},
+    ]
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=0,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="embedding_top",
+    )
+    assert rq["shortlist"]["top1_dense_and_fused_agree"] is None
+
+
+def test_build_route_quality_rules_loaded_ok() -> None:
+    rq = build_route_quality(
+        facet_list=[],
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=12,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="host_shortlist",
+    )
+    assert rq["policy"]["rules_loaded"] == 12

package/python/tests/test_routing_overlay.py

@@ -0,0 +1,55 @@
+"""Project routing overlay: notes, excludes, boosts."""
+from __future__ import annotations
+
+from app.route_policies import (
+    build_routing_overlay_payload,
+    merge_project_notes_into_route_query,
+    parse_routing_overlay,
+)
+
+
+def test_merge_notes_requires_project_root() -> None:
+    rq = merge_project_notes_into_route_query("hello", "note text", None)
+    assert rq == "hello"
+    rq2 = merge_project_notes_into_route_query("hello", "note text", "")
+    assert rq2 == "hello"
+
+
+def test_merge_notes_prepends_when_project_set() -> None:
+    rq = merge_project_notes_into_route_query("task", "We use Django 5.", "/repo", max_chars=100)
+    assert rq.startswith("Project routing notes:\n")
+    assert "Django" in rq
+    assert "task" in rq
+
+
+def test_parse_routing_overlay_boost_clamp() -> None:
+    ex, boosts, notes = parse_routing_overlay(
+        {"routing_boosts": {"a": 9.0, "b": -9.0}},
+        by_name={"a": 1, "b": 1},
+    )
+    assert not ex
+    assert boosts["a"] == 2.0
+    assert boosts["b"] == -2.0
+    assert notes == ""
+
+
+def test_parse_exclude_unknown_with_audit() -> None:
+    audit = []
+    ex, _b, _n = parse_routing_overlay(
+        {"exclude_skills": ["ghost"]},
+        by_name={"real": 1},
+        audit_out=audit,
+    )
+    assert "ghost" not in ex
+    assert any(a.get("effect") == "unknown_skill" for a in audit)
+
+
+def test_build_payload_none_when_empty() -> None:
+    assert build_routing_overlay_payload(
+        project_root="",
+        exclude_skills=frozenset(),
+        routing_boosts={},
+        project_notes_applied=False,
+        project_notes_len=0,
+        audit=[],
+    ) is None

package/python/tests/test_routing_signals.py

@@ -0,0 +1,112 @@
+"""Tests for conversation-aware route text, skill cards, and hybrid helpers."""
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from app.main import Skill, parse_skill_md
+from app.routing_signals import (
+    build_route_query_text,
+    host_pick_shortlist_lines,
+    keyword_overlap_scores,
+    normalize_minmax,
+    skill_routing_card,
+)
+
+
+def test_build_route_query_legacy(monkeypatch) -> None:
+    monkeypatch.setenv("SKILLFORGE_ROUTER_CONV_MAX_TURNS", "0")
+    out = build_route_query_text("hello", [{"role": "user", "content": "prev"}])
+    assert out == "hello"
+
+
+def test_build_route_query_merges_turns(monkeypatch) -> None:
+    monkeypatch.setenv("SKILLFORGE_ROUTER_CONV_MAX_TURNS", "2")
+    monkeypatch.setenv("SKILLFORGE_ROUTER_CONV_MSG_CHARS", "80")
+    conv = [
+        {"role": "user", "content": "first msg"},
+        {"role": "assistant", "content": "reply"},
+    ]
+    out = build_route_query_text("current ask", conv)
+    assert "user: first msg" in out
+    assert "assistant: reply" in out
+    assert "Current user message:" in out
+    assert out.endswith("current ask")
+
+
+def test_skill_routing_card_includes_triggers() -> None:
+    s = Skill(
+        name="x",
+        title="X Skill",
+        description="does things",
+        body="",
+        source="bundled",
+        triggers="when foo",
+        anti_triggers="not bar",
+    )
+    card = skill_routing_card(s)
+    assert "X Skill" in card
+    assert "Triggers: when foo" in card
+    assert "Anti-triggers: not bar" in card
+
+
+def test_normalize_minmax() -> None:
+    a = np.array([1.0, 3.0, 5.0])
+    assert np.allclose(normalize_minmax(a), [0.0, 0.5, 1.0])
+    flat = np.array([2.0, 2.0, 2.0])
+    assert np.allclose(normalize_minmax(flat), [0.0, 0.0, 0.0])
+
+
+def test_keyword_overlap_scores() -> None:
+    cards = ["alpha beta gamma", "foo bar"]
+    q = "beta search"
+    sc = keyword_overlap_scores(q, cards)
+    assert sc[0] > sc[1]
+
+
+def test_host_pick_shortlist_lines_basic() -> None:
+    facets = [
+        {
+            "name": "alpha-skill",
+            "title": "Alpha",
+            "cosine_similarity": 0.42,
+            "description_preview": "Does alpha testing patterns for flaky CI.",
+        }
+    ]
+    md, rows = host_pick_shortlist_lines(
+        prompt="fix flaky tests",
+        route_query="fix flaky tests",
+        facet_rows=facets,
+        max_candidates=5,
+        line_chars=90,
+    )
+    assert "alpha-skill" in md
+    assert "fix flaky" in md
+    assert len(rows) == 1
+    assert rows[0]["name"] == "alpha-skill"
+    assert rows[0]["id"] == "alpha-skill"
+    assert rows[0]["rank"] == 1
+
+
+def test_normalize_host_picked_main() -> None:
+    from app.main import Skill, normalize_host_picked_names
+
+    a = Skill(name="a", title="A", description="", body="", source="bundled")
+    b = Skill(name="b", title="B", description="", body="", source="bundled")
+    by_name = {"a": a, "b": b}
+    assert normalize_host_picked_names(["b", "a", "b", "unknown"], by_name, 1) == ["b"]
+    assert normalize_host_picked_names([], by_name, 7) == []
+
+
+def test_parse_skill_triggers(tmp_path) -> None:
+    md = tmp_path / "my-skill" / "SKILL.md"
+    md.parent.mkdir(parents=True, exist_ok=True)
+    md.write_text(
+        "---\nname: Nice\ndescription: Desc\ntriggers: when testing\n"
+        "anti_triggers: never for prod\n---\n\n# Body\n",
+        encoding="utf-8",
+    )
+    s = parse_skill_md(md, "bundled")
+    assert s is not None
+    assert s.triggers == "when testing"
+    assert s.anti_triggers == "never for prod"