@heytherevibin/skillforge 0.10.1 → 0.11.7

package/python/app/route_quality.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import math
+import os
 from typing import Any
 
 
@@ -31,6 +32,53 @@ def top1_cosine_vs_routing_agreement(facets: list[dict[str, Any]]) -> bool | Non
     return top_route == best_cos_name
 
 
+def _env_float(name: str, default_str: str) -> float:
+    raw = os.getenv(name, default_str).strip()
+    try:
+        return float(raw)
+    except ValueError:
+        return float(default_str)
+
+
+def _compute_ambiguous_and_tier(
+    *,
+    n: int,
+    cosine_margin: float | None,
+    routing_score_margin: float | None,
+) -> tuple[bool, str | None]:
+    """Return (ambiguous, confidence_tier)."""
+    if n == 0:
+        return False, None
+    if n == 1:
+        return False, "high"
+    ambig_off = os.getenv("SKILLFORGE_ROUTE_AMBIGUITY_DISABLE", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    )
+    if ambig_off:
+        ambiguous = False
+    else:
+        cos_thr = _env_float("SKILLFORGE_ROUTE_AMBIGUITY_COS_MARGIN", "0.012")
+        route_thr = _env_float("SKILLFORGE_ROUTE_AMBIGUITY_ROUTE_MARGIN", "0.018")
+        ambiguous = False
+        if cosine_margin is not None and cosine_margin < cos_thr:
+            ambiguous = True
+        if routing_score_margin is not None and routing_score_margin < route_thr:
+            ambiguous = True
+    tier: str
+    if ambiguous:
+        tier = "low"
+    elif cosine_margin is not None and routing_score_margin is not None:
+        if cosine_margin >= 0.04 and routing_score_margin >= 0.06:
+            tier = "high"
+        else:
+            tier = "medium"
+    else:
+        tier = "medium"
+    return ambiguous, tier
+
+
 def build_route_quality(
     *,
     facet_list: list[dict[str, Any]],
@@ -45,6 +93,7 @@ def build_route_quality(
     host_shortlist_only: bool = False,
     haiku_rerank_applied: bool = False,
     pick_path: str,
+    pick_diversify: dict[str, Any] | None = None,
 ) -> dict[str, Any]:
     """Structured signals for operators and MCP hosts (JSON-serializable)."""
     n = len(facet_list)
@@ -52,14 +101,24 @@ def build_route_quality(
     second_cos: float | None = None
     margin: float | None = None
     top_routing_score: float | None = None
+    second_routing_score: float | None = None
+    routing_score_margin: float | None = None
     if facet_list:
         top_cos = round(coerce_route_float(facet_list[0].get("cosine_similarity")), 6)
         top_routing_score = round(coerce_route_float(facet_list[0].get("routing_score")), 6)
         if len(facet_list) > 1:
             second_cos = round(coerce_route_float(facet_list[1].get("cosine_similarity")), 6)
             margin = round(float(top_cos - second_cos), 6)
+            second_routing_score = round(coerce_route_float(facet_list[1].get("routing_score")), 6)
+            if top_routing_score is not None and second_routing_score is not None:
+                routing_score_margin = round(float(top_routing_score - second_routing_score), 6)
 
     agree = top1_cosine_vs_routing_agreement(facet_list) if router_hybrid not in ("", "off", None) else None
+    ambiguous, confidence_tier = _compute_ambiguous_and_tier(
+        n=n,
+        cosine_margin=margin,
+        routing_score_margin=routing_score_margin,
+    )
 
     try:
         prl = int(policy_rules_loaded)
@@ -67,16 +126,25 @@ def build_route_quality(
         prl = 0
     prl = max(0, prl)
 
+    div = pick_diversify if isinstance(pick_diversify, dict) else None
+    if div is None:
+        div = {"applied": False, "dropped": [], "max_per_source": None}
+
     return {
-        "schema": "route_quality/1",
+        "schema": "route_quality/2",
         "shortlist": {
             "size": n,
             "top_cosine_similarity": top_cos,
             "second_cosine_similarity": second_cos,
             "cosine_margin": margin,
+            "second_routing_score": second_routing_score,
+            "routing_score_margin": routing_score_margin,
+            "ambiguous": ambiguous,
+            "confidence_tier": confidence_tier,
             "top_routing_score": top_routing_score,
             "hybrid_mode": router_hybrid or "off",
             "top1_dense_and_fused_agree": agree,
+            "cosine_leader_matches_routing_top": agree,
         },
         "router": {
             "mode": router_mode,
@@ -84,6 +152,7 @@ def build_route_quality(
             "host_picked": host_picked,
             "host_shortlist_only": host_shortlist_only,
             "haiku_rerank_applied": haiku_rerank_applied,
+            "pick_diversify": div,
         },
         "session": {
             "rerouted": rerouted,

package/python/app/router_llm.py

@@ -0,0 +1,85 @@
+"""Pluggable backend for Router rerank + final-pick LLM phases.
+
+When ``SKILLFORGE_TRANSPORT=mcp`` (stdio MCP only), only Anthropic-backed clients
+are wired from ``build_router_and_skills`` (legacy semantics).
+
+Standalone CLI may set ``SKILLFORGE_ROUTER_LLM_BACKEND=openai_compatible`` targeting
+``OPENAI_API_BASE`` (e.g. Ollama ``http://localhost:11434/v1``).
+"""
+from __future__ import annotations
+
+import os
+from typing import Protocol
+
+from anthropic import AsyncAnthropic
+
+
+class RouterLLM(Protocol):
+    backend_name: str
+
+    async def complete(self, *, system: str, user: str, max_tokens: int, model: str) -> str: ...
+
+
+def transport_is_mcp() -> bool:
+    return os.getenv("SKILLFORGE_TRANSPORT", "").strip().lower() == "mcp"
+
+
+class AnthropicRouterLLM:
+    __slots__ = ("client", "backend_name")
+
+    def __init__(self, client: AsyncAnthropic) -> None:
+        self.client = client
+        self.backend_name = "anthropic"
+
+    async def complete(self, *, system: str, user: str, max_tokens: int, model: str) -> str:
+        resp = await self.client.messages.create(
+            model=model,
+            max_tokens=max_tokens,
+            system=system,
+            messages=[{"role": "user", "content": user}],
+        )
+        block = resp.content[0]
+        return block.text.strip()  # type: ignore[attr-defined]
+
+
+class OpenAIRouterLLM:
+    __slots__ = ("_client", "default_model", "backend_name")
+
+    def __init__(self, *, api_key: str, base_url: str, default_model: str) -> None:
+        from openai import AsyncOpenAI
+
+        self._client = AsyncOpenAI(api_key=api_key or "ollama", base_url=base_url.rstrip("/"))
+        self.default_model = default_model
+        self.backend_name = "openai_compatible"
+
+    async def complete(self, *, system: str, user: str, max_tokens: int, model: str) -> str:
+        m = model or self.default_model
+        resp = await self._client.chat.completions.create(
+            model=m,
+            max_tokens=max_tokens,
+            messages=[
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+            temperature=0,
+        )
+        choice = resp.choices[0]
+        return (choice.message.content or "").strip()
+
+
+def resolve_openai_router_defaults() -> tuple[str, str, str]:
+    base = (
+        os.getenv("OPENAI_API_BASE", "").strip()
+        or os.getenv("SKILLFORGE_OPENAI_API_BASE", "").strip()
+        or "http://localhost:11434/v1"
+    )
+    api_key = (
+        os.getenv("OPENAI_API_KEY", "").strip()
+        or os.getenv("SKILLFORGE_OPENAI_API_KEY", "").strip()
+    )
+    model = (
+        os.getenv("SKILLFORGE_OPENAI_ROUTER_MODEL", "").strip()
+        or os.getenv("SKILLFORGE_CHAT_MODEL", "").strip()
+        or "llama3.2"
+    )
+    return base, api_key, model

package/python/app/router_mode.py

@@ -0,0 +1,21 @@
+"""Normalize ``SKILLFORGE_ROUTER_MODE`` (default **host**, explicit ``auto`` for legacy routing)."""
+
+
+def normalise_skillforge_router_mode(env_value: str) -> str:
+    """Return router mode token for ``app.main``
+
+    Expect ``env_value`` from ``os.getenv("SKILLFORGE_ROUTER_MODE", "host")``.
+
+    Returns
+    -------
+    ""
+        **auto** — use Haiku when ``ANTHROPIC_API_KEY`` is set, else embedding-first.
+    "host" | "embedding" | "full"
+        As selected.
+
+    **Unset** env → pass default ``"host"`` from ``getenv``. **Explicit auto:** ``auto`` or empty string.
+    """
+    s = (env_value or "").strip().lower()
+    if s in ("", "auto"):
+        return ""
+    return s

package/python/app/routing_signals.py

@@ -12,6 +12,11 @@ from app.route_quality import coerce_route_float
 _TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9_\-./]{2,}", re.I)
 
 
+def host_pick_max_candidates(*, top_k_cap: int) -> int:
+    """Caps host-mode numbered shortlist size (parity with ``run_route_turn``)."""
+    return max(3, min(top_k_cap, int(os.getenv("SKILLFORGE_HOST_PICK_MAX", "12"))))
+
+
 class _SkillCard(Protocol):
     title: str
     description: str
@@ -108,7 +113,8 @@ def host_pick_shortlist_lines(
     """Tight numbered list + structured rows for MCP host-pick phase (no in-process LLM)."""
     mc = max_candidates
     if mc is None:
-        mc = max(3, int(os.getenv("SKILLFORGE_HOST_PICK_MAX", "12")))
+        top_k_cap = int(os.getenv("SKILLFORGE_TOP_K", "15"))
+        mc = host_pick_max_candidates(top_k_cap=top_k_cap)
     lc = line_chars if line_chars is not None else int(os.getenv("SKILLFORGE_HOST_PICK_LINE_CHARS", "120"))
     prompt_one = (prompt or "").strip().replace("\n", " ")
     if len(prompt_one) > 160:

package/python/app/skill_manifest.py

@@ -0,0 +1,67 @@
+"""SKILL.md manifest checks (warnings + optional strict exclusion from catalog load)."""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class SkillLike(Protocol):
+    name: str
+    description: str
+    body: str
+    triggers: str
+    anti_triggers: str
+
+
+_SKILL_DIR_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
+
+
+def skill_md_has_yaml_frontmatter(path: Path) -> bool:
+    try:
+        head = path.read_text(encoding="utf-8")[:4096]
+    except OSError:
+        return False
+    return head.lstrip().startswith("---")
+
+
+def validate_skill_manifest(skill: SkillLike, path: Path) -> tuple[list[str], list[str]]:
+    """Return (errors, warnings). Errors justify exclusion under strict manifest mode."""
+    errors: list[str] = []
+    warnings: list[str] = []
+    dir_name = path.parent.name
+    if skill.name != dir_name:
+        warnings.append(f"parsed name {skill.name!r} != dirname {dir_name!r} (unexpected)")
+
+    if not _SKILL_DIR_PATTERN.match(dir_name):
+        errors.append(f"skill directory {dir_name!r} should match {_SKILL_DIR_PATTERN.pattern}")
+
+    body = (skill.body or "").strip()
+    if len(body) < 1:
+        errors.append("empty SKILL.md body after frontmatter")
+
+    desc = (skill.description or "").strip()
+    if len(desc) < 40:
+        warnings.append(f"short description ({len(desc)} chars) — consider expanding frontmatter description")
+
+    if len(desc) > 2800:
+        warnings.append(f"very long description ({len(desc)} chars) — routing card may truncate awkwardly")
+
+    if not skill_md_has_yaml_frontmatter(path):
+        warnings.append("no YAML frontmatter block (---) — title/description inferred from filename/body")
+
+    trig = ((skill.triggers or "") + (skill.anti_triggers or "")).strip()
+    if len(trig) > 4000:
+        warnings.append(f"very long triggers/anti_triggers ({len(trig)} chars total)")
+
+    return errors, warnings
+
+
+def skill_manifest_strict_exclusion() -> bool:
+    """When true, skip skills that fail manifest *errors* (see validate_skill_manifest)."""
+    import os
+
+    raw = os.getenv("SKILLFORGE_SKILL_MANIFEST_STRICT", "0").strip().lower()
+    return raw not in ("0", "false", "no", "")
+

package/python/app/skills_author_cli.py

@@ -0,0 +1,117 @@
+"""Scaffold user skills (`init`) and validate SKILL manifests (`lint`)."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+
+from app.main import parse_skill_md
+from app.skill_manifest import validate_skill_manifest
+
+_SAFE_NAME = re.compile(r"^[a-z][a-z0-9_-]{1,62}$")
+
+
+def skill_init(slug: str) -> Path:
+    if not _SAFE_NAME.match(slug):
+        raise ValueError(f"Skill folder name must match {_SAFE_NAME.pattern} — got {slug!r}")
+
+    skills_root = Path(os.getenv("SKILLFORGE_USER_SKILLS", str(Path.home() / ".skillforge" / "skills"))).expanduser().resolve()
+    dest = skills_root / slug
+    skill_md = dest / "SKILL.md"
+    if skill_md.exists():
+        raise ValueError(f"Already exists: {skill_md}")
+
+    dest.mkdir(parents=True, exist_ok=True)
+    body = (
+        "---\n"
+        f"name: {slug.replace('-', ' ').title()}\n"
+        "description: One-line pitch for routing (embedding + triggers card).\n"
+        "triggers: When does this skill apply?\n"
+        "anti_triggers: When should an agent skip this skill?\n"
+        "---\n\n"
+        f"# {slug}\n\n"
+        "## When to apply\n\n"
+        "- …\n\n"
+        "## Patterns\n\n"
+        "- …\n\n"
+        "## Verification\n\n"
+        "- …\n\n"
+        "## Outputs\n\n"
+        "- Expected shape of the assistant response\n"
+    )
+    skill_md.write_text(body, encoding="utf-8")
+    return skill_md
+
+
+def skill_lint_roots(roots: list[Path]) -> int:
+    """Return count of ERROR-level manifest issues."""
+    errors = 0
+    for root in roots:
+        r = root.expanduser().resolve()
+        globs = [r] if r.is_file() and r.name == "SKILL.md" else sorted(r.glob("**/SKILL.md"))
+        if not globs:
+            print(f"No SKILL.md found under {r}")
+            errors += 1
+            continue
+        for md in globs:
+            parsed = parse_skill_md(md, "lint")
+            if not parsed:
+                print(f"ERROR {md}: unreadable parse")
+                errors += 1
+                continue
+            errs, warns = validate_skill_manifest(parsed, md)
+            for w in warns:
+                print(f"WARN {md}: {w}")
+            for e in errs:
+                print(f"ERROR {md}: {e}")
+                errors += 1
+    return errors
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="Skillforge skill authoring helpers (scaffold + manifest lint).")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    p_init = sub.add_parser(
+        "init",
+        help="Create ~/.skillforge/skills/<slug>/SKILL.md (override SKILLFORGE_USER_SKILLS)",
+    )
+    p_init.add_argument("slug", help="Directory name under user skills folder (lowercase-kebab-case)")
+
+    p_lint = sub.add_parser(
+        "lint",
+        help="Validate SKILL.md files (see SKILLFORGE_SKILL_MANIFEST_STRICT for load-time exclusions)",
+    )
+    p_lint.add_argument(
+        "roots",
+        nargs="*",
+        default=[],
+        help="Directories or SKILL.md paths (default bundled + user trees from env)",
+    )
+
+    args = ap.parse_args()
+
+    try:
+        if args.cmd == "init":
+            p = skill_init(args.slug.strip().lower())
+            print(f"Wrote template: {p}")
+            print("(Reload MCP / hot reload to pick up the new catalog entry.)")
+            return
+        if args.cmd == "lint":
+            roots = [Path(p) for p in args.roots] if args.roots else []
+            if not roots:
+                bf = Path(os.getenv("SKILLFORGE_BUNDLED_SKILLS", "./skills")).expanduser().resolve()
+                usr = Path(os.getenv("SKILLFORGE_USER_SKILLS", str(Path.home() / ".skillforge" / "skills"))).expanduser()
+                roots = [bf, usr]
+            err_count = skill_lint_roots(roots)
+            raise SystemExit(1 if err_count else 0)
+    except ValueError as e:
+        print(f"{e}", file=sys.stderr)
+        raise SystemExit(1) from e
+
+
+if __name__ == "__main__":
+    main()

package/python/app/tips_cli.py

@@ -0,0 +1,37 @@
+"""Short terminal+MCP cheatsheet emitted on demand (avoid loading the full README)."""
+
+
+def main() -> None:
+    text = """# Skillforge — quick tips
+
+## After install (`skillforge install` or first `npx` run)
+
+Your machine gets `~/.skillforge/` (venv, global DB, user skills).
+
+Stable env (keys + tunables without shell paste): **`~/.skillforge/env`** — **`skillforge config path`**, **`skillforge config init`**, **`skillforge config validate`** (see README Configuration).
+
+## MCP (Cursor / Claude / …)
+
+- Wire with `skillforge mcp config` (see README for JSON merge targets).
+- Default routing is **host**: first `route_skills` returns a **shortlist**, second call sends `picked_names`.
+- **`capabilities`** tool: one JSON bundle — schema version, tool list, router snapshot — good for session start.
+
+## Terminal (same engine as MCP)
+
+- `skillforge route "your prompt"` — two-step **host** mode mirrors MCP (shortlist stdout, then re-run):
+  `skillforge route "…"` → `skillforge route "…" --picked-names=id1,id2`
+- **`skillforge route -i`** (TTY) prompts for ranks/`1,3` or skill ids after the shortlist step.
+  Or set **`SKILLFORGE_ROUTE_INTERACTIVE=1`** (still requires a tty).
+- **`skillforge route --json`** prints one JSON envelope (**`phase`**: `host_shortlist_prompt`, `host_shortlist_static`, `context`, `explain_only`) with `route_meta` for scripts.
+- **`skillforge route --explain`** attaches routing diagnostics (**stderr** Markdown, unless `--json` embeds **`explain`**).
+- **`skillforge route --explain-only`** — diagnostics only (**no finalize** / no session-heavy side effects comparable to MCP `explain_route` intent).
+- **`skillforge tools …`** (**`skillforge tools -h`**): MCP-equivalent subcommands; **`--json`** emits full tool envelopes.
+- **`skillforge agent`**: OpenAI-compatible chat calling the same MCP tool handlers (**`OPENAI_API_BASE`**, **`SKILLFORGE_AGENT_*`**). Run **`skillforge install`** after upgrades so **`openai`** is installed into **`~/.skillforge/venv`**.
+
+Docs: **`package/README.md`**.
+"""
+    print(text, end="")
+
+
+if __name__ == "__main__":
+    main()