@heytherevibin/skillforge 0.10.1 → 0.11.7

package/python/tests/test_route_cli_pick.py

@@ -0,0 +1,25 @@
+"""Parsing interactive CLI host picks."""
+
+from __future__ import annotations
+
+from app.route_cli_pick import parse_interactive_skill_pick
+
+
+def test_parse_ranks_and_names() -> None:
+    rows = [
+        {"rank": 1, "name": "alpha", "id": "alpha"},
+        {"rank": 2, "name": "beta", "id": "beta"},
+        {"rank": 3, "name": "gamma-sk", "id": "gamma-sk"},
+    ]
+    out = parse_interactive_skill_pick("1, gamma-sk ", rows)
+    assert out == ["alpha", "gamma-sk"]
+
+
+def test_parse_quit() -> None:
+    assert parse_interactive_skill_pick("q", [{}]) == []
+    assert parse_interactive_skill_pick("", []) == []
+
+
+def test_strip_backticks() -> None:
+    rows = [{"rank": 1, "name": "x"}]
+    assert parse_interactive_skill_pick("`x`", rows) == ["x"]

package/python/tests/test_route_policies.py

@@ -113,3 +113,32 @@ def test_invalid_regex_recorded(tmp_path, skill_alpha, monkeypatch) -> None:
         max_active=7,
     )
     assert any(r.get("effect") == "invalid_regex" for r in audit)
+
+
+def test_bad_inline_policies_warns_stderr(
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("SKILLFORGE_ROUTE_POLICIES", "{not-json")
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES_FILE", raising=False)
+    cfg = load_route_policies_config(None)
+    assert cfg == {"rules": []}
+    err = capsys.readouterr().err
+    assert "SKILLFORGE_ROUTE_POLICIES" in err
+    assert "invalid json" in err.lower()
+
+
+def test_bad_policies_file_warns_stderr(
+    capsys: pytest.CaptureFixture[str],
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path,
+) -> None:
+    monkeypatch.delenv("SKILLFORGE_ROUTE_POLICIES", raising=False)
+    p = tmp_path / "broken.json"
+    p.write_text("{bad", encoding="utf-8")
+    monkeypatch.setenv("SKILLFORGE_ROUTE_POLICIES_FILE", str(p))
+    cfg = load_route_policies_config(None)
+    assert cfg == {"rules": []}
+    err = capsys.readouterr().err
+    assert str(p) in err
+    assert "invalid json" in err.lower()

package/python/tests/test_route_quality.py

@@ -53,6 +53,9 @@ def test_build_route_quality_empty_facets() -> None:
     )
     assert rq["shortlist"]["size"] == 0
     assert rq["shortlist"]["top_cosine_similarity"] is None
+    assert rq["shortlist"]["confidence_tier"] is None
+    assert rq["schema"] == "route_quality/2"
+    assert rq["router"]["pick_diversify"]["applied"] is False
     assert rq["session"]["change_jaccard"] == 0.0
     assert rq["policy"]["rules_loaded"] == 0
 
@@ -78,6 +81,11 @@ def test_build_route_quality_malformed_metrics() -> None:
     assert rq["shortlist"]["top_routing_score"] == 0.0
     assert rq["shortlist"]["second_cosine_similarity"] == 0.5
     assert rq["shortlist"]["cosine_margin"] == round(-0.5, 6)
+    assert rq["shortlist"]["ambiguous"] is True
+    assert rq["shortlist"]["confidence_tier"] == "low"
+    assert rq["shortlist"]["routing_score_margin"] == round(-0.2, 6)
+    assert rq["shortlist"]["second_routing_score"] == 0.2
+    assert rq["schema"] == "route_quality/2"
     assert rq["shortlist"]["top1_dense_and_fused_agree"] is False
     assert rq["policy"]["rules_loaded"] == 0
     assert rq["policy"]["includes_added"] == 2
@@ -102,6 +110,7 @@ def test_build_route_quality_hybrid_off_skips_agree() -> None:
         pick_path="embedding_top",
     )
     assert rq["shortlist"]["top1_dense_and_fused_agree"] is None
+    assert rq["shortlist"]["cosine_leader_matches_routing_top"] is None
 
 
 def test_build_route_quality_rules_loaded_ok() -> None:
@@ -118,3 +127,66 @@ def test_build_route_quality_rules_loaded_ok() -> None:
         pick_path="host_shortlist",
     )
     assert rq["policy"]["rules_loaded"] == 12
+
+
+def test_build_route_quality_pick_diversify_meta() -> None:
+    facets = [{"name": "x", "cosine_similarity": 0.5, "routing_score": 0.8}]
+    div = {"applied": True, "dropped": ["z"], "max_per_source": 1}
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=["x"],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=0,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="haiku_pick",
+        pick_diversify=div,
+    )
+    assert rq["router"]["pick_diversify"] == div
+
+
+def test_build_route_quality_clear_shortlist_not_ambiguous(monkeypatch) -> None:
+    facets = [
+        {"name": "a", "cosine_similarity": 0.95, "routing_score": 0.9},
+        {"name": "b", "cosine_similarity": 0.91, "routing_score": 0.84},
+    ]
+    monkeypatch.delenv("SKILLFORGE_ROUTE_AMBIGUITY_DISABLE", raising=False)
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="host",
+        router_hybrid="keyword",
+        picked_names=[],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=0,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="host_shortlist",
+    )
+    assert rq["shortlist"]["ambiguous"] is False
+    assert rq["shortlist"]["confidence_tier"] == "high"
+
+
+def test_build_route_quality_ambiguity_disable_env(monkeypatch) -> None:
+    facets = [
+        {"name": "a", "cosine_similarity": 0.5, "routing_score": 0.9},
+        {"name": "b", "cosine_similarity": 0.498, "routing_score": 0.89},
+    ]
+    monkeypatch.setenv("SKILLFORGE_ROUTE_AMBIGUITY_DISABLE", "1")
+    rq = build_route_quality(
+        facet_list=facets,
+        router_mode="auto",
+        router_hybrid="off",
+        picked_names=[],
+        rerouted=False,
+        change=0.0,
+        policy_rules_loaded=0,
+        policy_audit=[],
+        host_picked=False,
+        pick_path="embedding_top",
+    )
+    assert rq["shortlist"]["ambiguous"] is False
+    assert rq["shortlist"]["confidence_tier"] == "medium"

package/python/tests/test_router_llm.py

@@ -0,0 +1,63 @@
+"""Lightweight assertions for RouterLLM helpers and MCP transport snapshot."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from app.mcp_operator import build_router_status_dict, build_capabilities_bundle
+from app.router_llm import OpenAIRouterLLM, resolve_openai_router_defaults, transport_is_mcp
+
+
+def test_resolve_openai_router_defaults_contains_base() -> None:
+    base, _, _ = resolve_openai_router_defaults()
+    assert "v1" in base.lower() or base.startswith("http")
+
+
+def test_transport_is_mcp_false_by_default(monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_TRANSPORT", raising=False)
+    assert transport_is_mcp() is False
+
+
+def test_transport_is_mcp_true(monkeypatch) -> None:
+    monkeypatch.setenv("SKILLFORGE_TRANSPORT", "mcp")
+    assert transport_is_mcp() is True
+
+
+def test_router_status_includes_router_llm_fields() -> None:
+    r = SimpleNamespace(
+        anthropic=None,
+        router_llm=None,
+        context_mode="chunks",
+        _hybrid_mode="off",
+    )
+    snap = build_router_status_dict(r, skill_count=1)
+    assert snap["router_llm_backend"] == "none"
+    assert snap["router_llm_active"] is False
+    assert snap["anthropic_available"] is False
+
+
+def test_router_status_detects_fake_openai_router_llm(monkeypatch) -> None:
+    monkeypatch.delenv("SKILLFORGE_OPENAI_API_KEY", raising=False)
+
+    llm = OpenAIRouterLLM(api_key="", base_url="http://localhost:11434/v1", default_model="t")
+    r = SimpleNamespace(
+        anthropic=None,
+        router_llm=llm,
+        context_mode="chunks",
+        _hybrid_mode="off",
+    )
+    snap = build_router_status_dict(r, skill_count=1)
+    assert snap["router_llm_backend"] == "openai_compatible"
+    assert snap["router_llm_active"] is True
+
+
+def test_capabilities_bundle_has_standalone_agent_hint() -> None:
+    router = SimpleNamespace(
+        anthropic=None,
+        context_mode="chunks",
+        _hybrid_mode="off",
+        _by_name=None,
+    )
+    bundle = build_capabilities_bundle(router, skill_count=0)
+    assert "standalone_agent" in bundle
+    assert bundle["user_env_profile"]["validate_command"] == "skillforge config validate"

package/python/tests/test_router_mode_env.py

@@ -0,0 +1,21 @@
+"""SKILLFORGE_ROUTER_MODE normalization (default host vs explicit auto)."""
+from __future__ import annotations
+
+from app.router_mode import normalise_skillforge_router_mode
+
+
+def test_normalise_defaults_to_literal_host_from_value() -> None:
+    assert normalise_skillforge_router_mode("host") == "host"
+    assert normalise_skillforge_router_mode("HOST") == "host"
+
+
+def test_normalise_auto_aliases() -> None:
+    assert normalise_skillforge_router_mode("") == ""
+    assert normalise_skillforge_router_mode("   ") == ""
+    assert normalise_skillforge_router_mode("auto") == ""
+    assert normalise_skillforge_router_mode("AuTo") == ""
+
+
+def test_normalise_embedding_full() -> None:
+    assert normalise_skillforge_router_mode("embedding") == "embedding"
+    assert normalise_skillforge_router_mode("full") == "full"

package/python/tests/test_routing_signals.py

@@ -88,6 +88,26 @@ def test_host_pick_shortlist_lines_basic() -> None:
     assert rows[0]["rank"] == 1
 
 
+def test_host_pick_shortlist_defaults_respect_top_k_and_host_pick_max(monkeypatch) -> None:
+    monkeypatch.setenv("SKILLFORGE_TOP_K", "50")
+    monkeypatch.setenv("SKILLFORGE_HOST_PICK_MAX", "20")
+    facets = [
+        {
+            "name": f"c{i}",
+            "title": "t",
+            "cosine_similarity": 0.1 - i * 0.001,
+            "description_preview": "",
+        }
+        for i in range(40)
+    ]
+    _md, rows = host_pick_shortlist_lines(
+        prompt="p",
+        route_query="p",
+        facet_rows=facets,
+    )
+    assert len(rows) == 20
+
+
 def test_normalize_host_picked_main() -> None:
     from app.main import Skill, normalize_host_picked_names
 

package/python/tests/test_skill_manifest.py

@@ -0,0 +1,48 @@
+"""SKILL manifest validation rules."""
+
+from __future__ import annotations
+
+from app.main import load_all_skills, parse_skill_md
+from app.skill_manifest import validate_skill_manifest
+
+
+def test_validate_manifest_warns_without_frontmatter(tmp_path) -> None:
+    d = tmp_path / "my-skill"
+    d.mkdir()
+    md = d / "SKILL.md"
+    md.write_text("# Hello\n\nBody text here " * 20, encoding="utf-8")
+    s = parse_skill_md(md, "user")
+    assert s is not None
+    errs, warns = validate_skill_manifest(s, md)
+    assert errs == []
+    assert any("frontmatter" in w for w in warns)
+
+
+def test_load_all_skills_skips_bad_manifest_only_when_strict(monkeypatch, tmp_path) -> None:
+    monkeypatch.setenv("SKILLFORGE_BUNDLED_SKILLS", str(tmp_path / "bundled"))
+    monkeypatch.setenv("SKILLFORGE_USER_SKILLS", str(tmp_path / "user"))
+
+    bundled = tmp_path / "bundled" / "ok-skill"
+    bundled.mkdir(parents=True)
+    (bundled / "SKILL.md").write_text(
+        "---\nname: OK\ndescription: " + ("x" * 50) + "\n---\n\n# OK\nbody " * 30,
+        encoding="utf-8",
+    )
+    weird = tmp_path / "bundled" / "Weird_Case_Skill"
+    weird.mkdir(parents=True)
+    (weird / "SKILL.md").write_text(
+        "---\nname: Weird\ndescription: " + ("y" * 50) + "\n---\n\nbody " * 30,
+        encoding="utf-8",
+    )
+
+    monkeypatch.setenv("SKILLFORGE_SKILL_MANIFEST_STRICT", "0")
+    lax = load_all_skills(manifest_log_prefix="[test]")
+    lax_names = {x.name for x in lax}
+    assert "ok-skill" in lax_names
+    assert "Weird_Case_Skill" in lax_names
+
+    monkeypatch.setenv("SKILLFORGE_SKILL_MANIFEST_STRICT", "1")
+    strict = load_all_skills(manifest_log_prefix="[test]")
+    names = {x.name for x in strict}
+    assert "ok-skill" in names
+    assert "Weird_Case_Skill" not in names

package/python/tests/test_tools_cli.py

@@ -0,0 +1,69 @@
+"""Smoke tests for tools_cli argument wiring (no heavy router setup)."""
+
+from __future__ import annotations
+
+import pytest
+
+from app.tools_cli import build_parser
+def test_tools_cli_search_parses() -> None:
+    p = build_parser()
+    ns = p.parse_args(["search", "hello", "world", "--limit", "5"])
+    assert ns.tool == "search"
+    assert ns.query == ["hello", "world"]
+    assert ns.limit == 5
+
+
+def test_tools_cli_disable_polarity() -> None:
+    p = build_parser()
+    ns_off = p.parse_args(["disable", "--skill-name", "x", "--off"])
+    assert ns_off.disabled is True
+    ns_on = p.parse_args(["disable", "--skill-name", "x", "--on"])
+    assert ns_on.disabled is False
+
+
+def test_tools_cli_global_flags_with_subcommand() -> None:
+    p = build_parser()
+    ns = p.parse_args(
+        ["--project-root", "/tmp/ws", "--user-id", "u1", "catalog"],
+    )
+    assert ns.tool == "catalog"
+    assert ns.project_root == "/tmp/ws"
+    assert ns.user_id == "u1"
+
+
+def test_tools_cli_materialize_roots() -> None:
+    p = build_parser()
+    ns = p.parse_args(
+        [
+            "materialize",
+            "--root",
+            "/repo",
+            "--names",
+            "skill_a, skill_b",
+            "--hosts",
+            "both",
+            "--no-merge",
+        ],
+    )
+    assert ns.mat_root == "/repo"
+    assert ns.names == "skill_a, skill_b"
+    assert ns.hosts == "both"
+    assert ns.no_merge is True
+
+
+def test_tools_cli_router_status_hyphen_alias() -> None:
+    p = build_parser()
+    ns = p.parse_args(["router-status"])
+    assert ns.tool == "router-status"
+
+
+def test_tools_cli_feedback_thumbs_negative() -> None:
+    p = build_parser()
+    ns = p.parse_args(["feedback", "--skill-name", "s", "--thumbs=-1"])
+    assert ns.thumbs == -1
+
+
+def test_tools_cli_feedback_rejects_bad_thumbs() -> None:
+    p = build_parser()
+    with pytest.raises(SystemExit):
+        p.parse_args(["feedback", "--skill-name", "s", "--thumbs", "2"])