@heytherevibin/skillforge 0.7.0 → 0.10.0

package/lib/host-setup.js

@@ -0,0 +1,312 @@
+/**
+ * Detect MCP-friendly editors and install global slash-command files (Cursor, Claude Code).
+ */
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { spawnSync } = require('child_process');
+
+const MANAGED_SUBSTRING = 'skillforge-managed';
+
+/** @returns {string} */
+function homeDir() {
+  return os.homedir();
+}
+
+/** @returns {string} */
+function claudeConfigBase() {
+  const override = process.env.CLAUDE_CONFIG_DIR;
+  if (override && String(override).trim()) {
+    return String(override).trim();
+  }
+  return path.join(homeDir(), '.claude');
+}
+
+function claudeCliOnPath() {
+  try {
+    const isWin = process.platform === 'win32';
+    const bin = isWin ? 'where' : 'which';
+    const r = spawnSync(bin, ['claude'], { encoding: 'utf8', stdio: 'pipe', timeout: 4000 });
+    return r.status === 0;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * @returns {{ id: string, label: string, detail: string }[]}
+ */
+function detectMcpFriendlyHosts() {
+  const h = homeDir();
+  /** @type {{ id: string, label: string, detail: string }[]} */
+  const out = [];
+
+  const cursorDir = path.join(h, '.cursor');
+  if (fs.existsSync(cursorDir)) {
+    out.push({
+      id: 'cursor',
+      label: 'Cursor',
+      detail: `~/.cursor (mcp.json, commands/)`,
+    });
+  }
+
+  const claudeJson = path.join(h, '.claude.json');
+  const ccSettings = path.join(claudeConfigBase(), 'settings.json');
+  if (fs.existsSync(claudeJson) || fs.existsSync(ccSettings) || claudeCliOnPath()) {
+    out.push({
+      id: 'claude-code',
+      label: 'Claude Code',
+      detail: fs.existsSync(claudeJson)
+        ? '~/.claude.json + ~/.claude/'
+        : '~/.claude/ (add MCP via claude mcp or .mcp.json)',
+    });
+  }
+
+  if (process.platform === 'darwin') {
+    const claude = path.join(h, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json');
+    if (fs.existsSync(claude)) {
+      out.push({
+        id: 'claude-desktop',
+        label: 'Claude Desktop',
+        detail: '~/Library/Application Support/Claude/',
+      });
+    }
+    const cursorApp = path.join('/Applications', 'Cursor.app');
+    const cursorAppUser = path.join(h, 'Applications', 'Cursor.app');
+    if (fs.existsSync(cursorApp) || fs.existsSync(cursorAppUser)) {
+      if (!out.some(x => x.id === 'cursor' || x.id === 'cursor-app')) {
+        out.push({
+          id: 'cursor-app',
+          label: 'Cursor (app)',
+          detail: 'Cursor.app installed — use Cursor MCP settings if ~/.cursor is missing',
+        });
+      }
+    }
+  } else if (process.platform === 'win32') {
+    const appData = process.env.APPDATA || path.join(h, 'AppData', 'Roaming');
+    const claudeWin = path.join(appData, 'Claude', 'claude_desktop_config.json');
+    if (fs.existsSync(claudeWin)) {
+      out.push({
+        id: 'claude-desktop',
+        label: 'Claude Desktop',
+        detail: path.join(appData, 'Claude'),
+      });
+    }
+  }
+
+  return out;
+}
+
+/**
+ * @returns {boolean}
+ */
+function looksLikeCursorEnvironment() {
+  const h = homeDir();
+  if (fs.existsSync(path.join(h, '.cursor'))) return true;
+  if (process.platform === 'darwin') {
+    if (fs.existsSync(path.join('/Applications', 'Cursor.app'))) return true;
+    if (fs.existsSync(path.join(h, 'Applications', 'Cursor.app'))) return true;
+  }
+  if (process.env.SKILLFORGE_CURSOR_GLOBAL_COMMAND === '1' || process.env.SKILLFORGE_CURSOR_GLOBAL_COMMAND === 'true') {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * @returns {boolean}
+ */
+function looksLikeClaudeCodeEnvironment() {
+  const h = homeDir();
+  if (fs.existsSync(path.join(h, '.claude.json'))) return true;
+  const base = claudeConfigBase();
+  if (fs.existsSync(path.join(base, 'settings.json'))) return true;
+  if (claudeCliOnPath()) return true;
+  if (process.env.SKILLFORGE_CLAUDE_CODE_GLOBAL_COMMAND === '1' || process.env.SKILLFORGE_CLAUDE_CODE_GLOBAL_COMMAND === 'true') {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * @param {{ force?: boolean, log?: (msg: string) => void, err?: (msg: string) => void, pkgRoot: string, pkgVersion: string }} opts
+ * @returns {{ wrote: boolean, path: string | null, skippedReason: string | null }}
+ */
+function installGlobalCursorSkillforgeCommand(opts) {
+  const log = opts.log || (() => {});
+  const errFn = opts.err || log;
+  const force = !!opts.force;
+  const skip = process.env.SKILLFORGE_SKIP_CURSOR_SETUP === '1' || process.env.SKILLFORGE_SKIP_CURSOR_SETUP === 'true';
+  if (skip) {
+    return { wrote: false, path: null, skippedReason: 'SKILLFORGE_SKIP_CURSOR_SETUP' };
+  }
+
+  if (!looksLikeCursorEnvironment()) {
+    return { wrote: false, path: null, skippedReason: 'cursor_not_detected' };
+  }
+
+  const cmdDir = path.join(homeDir(), '.cursor', 'commands');
+  const cmdFile = path.join(cmdDir, 'skillforge.md');
+
+  try {
+    if (fs.existsSync(cmdFile) && !force) {
+      const prev = fs.readFileSync(cmdFile, 'utf8');
+      if (!prev.includes(MANAGED_SUBSTRING)) {
+        log(
+          `Cursor: left ${cmdFile} unchanged (custom file; not skillforge-managed). Replace with: skillforge hosts init --force`
+        );
+        return { wrote: false, path: cmdFile, skippedReason: 'custom_file' };
+      }
+    }
+
+    const tplPath = path.join(opts.pkgRoot, 'lib', 'templates', 'cursor-skillforge-global.md');
+    if (!fs.existsSync(tplPath)) {
+      errFn(`Cursor: template missing at ${tplPath}`);
+      return { wrote: false, path: null, skippedReason: 'no_template' };
+    }
+    let body = fs.readFileSync(tplPath, 'utf8');
+    body = body.replace(/PACKAGE_VERSION/g, opts.pkgVersion);
+
+    fs.mkdirSync(cmdDir, { recursive: true });
+    fs.writeFileSync(cmdFile, body, 'utf8');
+    return { wrote: true, path: cmdFile, skippedReason: null };
+  } catch (e) {
+    errFn(`Cursor: could not write ${cmdFile}: ${/** @type {Error} */ (e).message}`);
+    return { wrote: false, path: cmdFile, skippedReason: 'write_error' };
+  }
+}
+
+/**
+ * @param {{ force?: boolean, log?: (msg: string) => void, err?: (msg: string) => void, pkgRoot: string, pkgVersion: string }} opts
+ * @returns {{ wrote: boolean, path: string | null, skippedReason: string | null }}
+ */
+function installGlobalClaudeCodeSkillforgeCommand(opts) {
+  const log = opts.log || (() => {});
+  const errFn = opts.err || log;
+  const force = !!opts.force;
+  const skip = process.env.SKILLFORGE_SKIP_CLAUDE_CODE_SETUP === '1' || process.env.SKILLFORGE_SKIP_CLAUDE_CODE_SETUP === 'true';
+  if (skip) {
+    return { wrote: false, path: null, skippedReason: 'SKILLFORGE_SKIP_CLAUDE_CODE_SETUP' };
+  }
+
+  if (!looksLikeClaudeCodeEnvironment()) {
+    return { wrote: false, path: null, skippedReason: 'claude_code_not_detected' };
+  }
+
+  const base = claudeConfigBase();
+  const cmdDir = path.join(base, 'commands');
+  const cmdFile = path.join(cmdDir, 'skillforge.md');
+
+  try {
+    if (fs.existsSync(cmdFile) && !force) {
+      const prev = fs.readFileSync(cmdFile, 'utf8');
+      if (!prev.includes(MANAGED_SUBSTRING)) {
+        log(
+          `Claude Code: left ${cmdFile} unchanged (custom file; not skillforge-managed). Replace with: skillforge hosts init --force`
+        );
+        return { wrote: false, path: cmdFile, skippedReason: 'custom_file' };
+      }
+    }
+
+    const tplPath = path.join(opts.pkgRoot, 'lib', 'templates', 'claude-code-skillforge-global.md');
+    if (!fs.existsSync(tplPath)) {
+      errFn(`Claude Code: template missing at ${tplPath}`);
+      return { wrote: false, path: null, skippedReason: 'no_template' };
+    }
+    let body = fs.readFileSync(tplPath, 'utf8');
+    body = body.replace(/PACKAGE_VERSION/g, opts.pkgVersion);
+
+    fs.mkdirSync(cmdDir, { recursive: true });
+    fs.writeFileSync(cmdFile, body, 'utf8');
+    return { wrote: true, path: cmdFile, skippedReason: null };
+  } catch (e) {
+    errFn(`Claude Code: could not write ${cmdFile}: ${/** @type {Error} */ (e).message}`);
+    return { wrote: false, path: cmdFile, skippedReason: 'write_error' };
+  }
+}
+
+/**
+ * @param {{ force?: boolean, log?: typeof console.error, ok?: typeof console.error, err?: typeof console.error, dim?: (s: string) => void, pkgRoot: string, pkgVersion: string }} opts
+ */
+function reportHostsAndInstallAgentCommands(opts) {
+  const log = opts.log || console.error;
+  const ok = opts.ok || console.error;
+  const errLine = opts.err || log;
+  const dim =
+    opts.dim ||
+    ((s) => {
+      console.error(s);
+    });
+
+  const hosts = detectMcpFriendlyHosts();
+  if (hosts.length > 0) {
+    log('');
+    ok(`Detected MCP-friendly environment(s): ${hosts.map(h => h.label).join(', ')}`);
+    hosts.forEach(h => dim(`  • ${h.label}: ${h.detail}`));
+  }
+  if (hosts.some(h => h.id === 'claude-desktop')) {
+    dim('  Tip (Claude Desktop): merge `skillforge mcp config` into claude_desktop_config.json');
+  }
+  if (hosts.some(h => h.id === 'claude-code')) {
+    dim(
+      '  Tip (Claude Code): add skillforge MCP (`skillforge mcp config`, project `.mcp.json`, or `claude mcp`). Global `/skillforge` → ~/.claude/commands/skillforge.md'
+    );
+  }
+
+  const rCur = installGlobalCursorSkillforgeCommand({
+    force: opts.force,
+    log,
+    err: errLine,
+    pkgRoot: opts.pkgRoot,
+    pkgVersion: opts.pkgVersion,
+  });
+
+  const rCc = installGlobalClaudeCodeSkillforgeCommand({
+    force: opts.force,
+    log,
+    err: errLine,
+    pkgRoot: opts.pkgRoot,
+    pkgVersion: opts.pkgVersion,
+  });
+
+  if (rCur.skippedReason === 'cursor_not_detected') {
+    dim('');
+    dim(
+      'Cursor: ~/.cursor not found. For global /skillforge: SKILLFORGE_CURSOR_GLOBAL_COMMAND=1 skillforge install  (or skillforge hosts init)'
+    );
+  } else if (rCur.skippedReason === 'SKILLFORGE_SKIP_CURSOR_SETUP') {
+    dim('Cursor integration skipped (SKILLFORGE_SKIP_CURSOR_SETUP).');
+  }
+  if (rCur.wrote && rCur.path) {
+    ok(`Wrote Cursor command: ${rCur.path} (use /skillforge in chat)`);
+  }
+
+  if (rCc.skippedReason === 'claude_code_not_detected') {
+    dim('');
+    dim(
+      'Claude Code not detected (~/.claude.json, ~/.claude/settings.json, or `claude` on PATH). For global /skillforge: SKILLFORGE_CLAUDE_CODE_GLOBAL_COMMAND=1 skillforge install  (or skillforge hosts init)'
+    );
+  } else if (rCc.skippedReason === 'SKILLFORGE_SKIP_CLAUDE_CODE_SETUP') {
+    dim('Claude Code integration skipped (SKILLFORGE_SKIP_CLAUDE_CODE_SETUP).');
+  }
+  if (rCc.wrote && rCc.path) {
+    ok(`Wrote Claude Code command/skill: ${rCc.path} (use /skillforge in Terminal/IDE)`);
+  }
+}
+
+/** @deprecated use reportHostsAndInstallAgentCommands */
+function reportHostsAndInstallCursorCommand(opts) {
+  reportHostsAndInstallAgentCommands(opts);
+}
+
+module.exports = {
+  detectMcpFriendlyHosts,
+  claudeConfigBase,
+  looksLikeCursorEnvironment,
+  looksLikeClaudeCodeEnvironment,
+  installGlobalCursorSkillforgeCommand,
+  installGlobalClaudeCodeSkillforgeCommand,
+  reportHostsAndInstallAgentCommands,
+  reportHostsAndInstallCursorCommand,
+  MANAGED_SUBSTRING,
+};

package/lib/templates/claude-code-skillforge-global.md

@@ -0,0 +1,19 @@
+---
+description: Run Skillforge MCP (route_skills) to load routed SKILL.md context. Use when the user invokes /skillforge, asks for Skillforge, or needs catalog skills for the repo.
+---
+
+<!-- skillforge-managed vPACKAGE_VERSION — remove this line to stop auto-updates from `skillforge install` -->
+
+# Skillforge — route SKILL.md context (MCP)
+
+Global **`/skillforge`** for **Claude Code** (same mechanism as `.claude/commands/*.md`). Configure the **skillforge** MCP server (see **`skillforge mcp config`**, project **`.mcp.json`**, or `claude mcp`). Optional: **`skillforge health`**, **`skillforge route-eval`**, **`skillforge weights export|import`**. MCP **`_meta.feedback_effect`** explains learned-ranking bias for picked skills when present.
+
+## Do this
+
+1. **`route_skills`** (MCP): pass **`project_root`** as the **current project root** (absolute path) so SQLite lives in **`<project>/.skillforge/`**. Pass the **user's task** as **`prompt`**. Reuse **`session_id`** across turns when the MCP returns it.
+
+   - **`SKILLFORGE_ROUTER_MODE=host`**: first call **without** **`picked_names`** (shortlist only); second call **with** **`picked_names`**.
+
+2. **Use the returned skill text** in your answer.
+
+3. **Per-repo list:** run **`materialize_project`** after **`route_skills`** to refresh **`.claude/commands/skillforge.md`** and **`CLAUDE.md`**.

package/lib/templates/cursor-skillforge-global.md

@@ -0,0 +1,16 @@
+<!-- skillforge-managed vPACKAGE_VERSION — remove this line to stop auto-updates from `skillforge install` -->
+
+# Skillforge — route SKILL.md context (MCP)
+
+Global **`/skillforge`** command. Use the **skillforge** MCP server (configure in **`~/.cursor/mcp.json`** or your host; run **`skillforge mcp config`** for a JSON snippet). For local checks: **`skillforge health`** (preflight), **`skillforge route-eval`** (embedding routing smoke tests), and **`skillforge weights export|import`** (backup learned weights). **`route_skills`** responses can include **`_meta.feedback_effect`** (how thumbs / uses bias ranking for picked skills).
+
+## Do this
+
+1. **`route_skills`** (MCP): pass **`project_root`** as the **current workspace root** (absolute path) so SQLite lives in **`<workspace>/.skillforge/`**. Pass the **user's task** as **`prompt`**. Reuse **`session_id`** across turns when the tool returns one.
+
+   - **`SKILLFORGE_ROUTER_MODE=host`**: call once **without** **`picked_names`** (shortlist only); then call again with **`picked_names`** (exact catalog ids) to load skill context.
+   - Optional: **`conversation`** when recent turns should influence routing.
+
+2. **Use the returned skill text** in your answer.
+
+3. **Project-specific lists:** run **`materialize_project`** in a repo (after **`route_skills`**) to write **`.cursor/commands/skillforge.md`**, **`.cursor/rules/skillforge.mdc`**, and **`docs/SKILLFORGE-PRD.md`** with the latest **`skill_names`**.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@heytherevibin/skillforge",
-  "version": "0.7.0",
-  "description": "Skill orchestration for Claude: hybrid embedding and router-based routing, MCP stdio server, per-user learning, and a large bundled SKILL.md catalog.",
+  "version": "0.10.0",
+  "description": "SKILL.md orchestration for AI agents: MCP-first routing with local embeddings, optional LLM stages, project RAG, policy overlays, portable learning state, and auditable telemetry.",
   "keywords": [
     "claude",
     "skills",
@@ -17,6 +17,7 @@
   },
   "files": [
     "bin/",
+    "ci/",
     "lib/",
     "python/",
     "skills/",

package/python/app/eval_cli.py

@@ -0,0 +1,133 @@
+"""Run route quality eval fixtures (deterministic embedding routing)."""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+import tempfile
+from pathlib import Path
+
+
+def _parse_args(argv: list[str] | None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        description=(
+            "Evaluate route_skills-style routing against a JSON fixture "
+            "(defaults to SKILLFORGE_ROUTER_MODE=embedding for stable retrieval)."
+        )
+    )
+    p.add_argument(
+        "--fixture",
+        "-f",
+        type=Path,
+        required=True,
+        help="Path to JSON fixture (cases[].prompt, expect_in_candidates, …).",
+    )
+    p.add_argument(
+        "--router-mode",
+        default="embedding",
+        help="Router mode for this process (set before loading app.main). Default: embedding",
+    )
+    p.add_argument(
+        "--json",
+        action="store_true",
+        help="Print one JSON object per line to stdout (cases + errors); stderr stays human.",
+    )
+    return p.parse_args(argv)
+
+
+async def _run_case(
+    *,
+    con,
+    router,
+    case: dict,
+    defaults: dict,
+) -> list[str]:
+    from app.main import run_route_turn
+
+    prompt = (case.get("prompt") or "").strip()
+    if not prompt:
+        return [f"{case.get('id', '?')}: empty prompt"]
+
+    result = await run_route_turn(
+        con,
+        router,
+        prompt,
+        conversation=[],
+        user_id="__eval__",
+        session_id=None,
+        project_root=None,
+        include_project_rag=False,
+        picked_names_from_host=None,
+        picked_names_from_host_supplied=False,
+    )
+
+    from app.route_eval_harness import evaluate_case_result
+
+    return evaluate_case_result(result, case, defaults=defaults)
+
+
+async def _async_main(args: argparse.Namespace) -> int:
+    fixture_path = args.fixture.expanduser().resolve()
+    if not fixture_path.is_file():
+        print(f"skillforge route-eval: fixture not found: {fixture_path}", file=sys.stderr)
+        return 2
+
+    os.environ["SKILLFORGE_ROUTER_MODE"] = args.router_mode.strip().lower()
+
+    from app.route_eval_harness import load_eval_fixture
+    from app.main import build_router_and_skills, init_db
+
+    data = load_eval_fixture(fixture_path)
+    defaults = data["defaults"] if isinstance(data.get("defaults"), dict) else {}
+    cases = data["cases"]
+
+    fd, tmp_name = tempfile.mkstemp(suffix=".sqlite", prefix="skillforge-eval-")
+    os.close(fd)
+    db_path = Path(tmp_name)
+    try:
+        con = init_db(db_path)
+        try:
+            router, _skills = await asyncio.to_thread(
+                build_router_and_skills,
+                log=not args.json,
+                log_prefix="[skillforge-eval]",
+            )
+            all_errs: list[str] = []
+            summaries: list[dict] = []
+            for case in cases:
+                if not isinstance(case, dict):
+                    all_errs.append("non-dict case entry")
+                    continue
+                errs = await _run_case(con=con, router=router, case=case, defaults=defaults)
+                cid = case.get("id") or case.get("name") or "?"
+                summaries.append({"id": cid, "ok": not errs, "errors": errs})
+                all_errs.extend(errs)
+                if errs and not args.json:
+                    for e in errs:
+                        print(e, file=sys.stderr)
+                elif not errs and not args.json:
+                    print(f"ok  {cid}", file=sys.stderr)
+
+            if args.json:
+                out = {"fixture": str(fixture_path), "cases": summaries, "failed": len(all_errs)}
+                print(json.dumps(out, indent=2))
+
+            return 1 if all_errs else 0
+        finally:
+            con.close()
+    finally:
+        try:
+            db_path.unlink(missing_ok=True)
+        except OSError:
+            pass
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = _parse_args(argv)
+    raise SystemExit(asyncio.run(_async_main(args)))
+
+
+if __name__ == "__main__":
+    main()

package/python/app/feedback_meta.py

@@ -0,0 +1,96 @@
+"""Transparency for per-user learning: how feedback stats affect routing scores."""
+from __future__ import annotations
+
+import sqlite3
+from typing import Any
+
+
+def get_skill_weight_detail(con: sqlite3.Connection, skill_name: str, user_id: str = "") -> dict[str, Any] | None:
+    cur = con.execute(
+        """
+        SELECT weight, uses, referenced, thumbs_up, thumbs_down, disabled, updated_at
+        FROM skill_weights WHERE user_id = ? AND skill_name = ?
+        """,
+        (user_id, skill_name),
+    )
+    row = cur.fetchone()
+    if not row:
+        return None
+    w, uses, ref, up, down, dis, ts = row
+    uses_i, ref_i = int(uses), int(ref)
+    up_i, down_i = int(up), int(down)
+    ref_rate = (ref_i / uses_i) if uses_i > 0 else 0.0
+    return {
+        "learned_weight": round(float(w), 4),
+        "uses": uses_i,
+        "referenced": ref_i,
+        "thumbs_up": up_i,
+        "thumbs_down": down_i,
+        "net_thumbs": up_i - down_i,
+        "reference_rate": round(float(ref_rate), 4),
+        "disabled": bool(dis),
+        "updated_at": float(ts) if ts is not None else None,
+    }
+
+
+def build_feedback_effect(
+    con: sqlite3.Connection,
+    picked_names: list[str],
+    user_id: str = "",
+) -> dict[str, Any]:
+    """JSON-serializable snapshot of learning stats for picked skills (after this route's use counts)."""
+    seen: set[str] = set()
+    ordered: list[str] = []
+    for n in picked_names:
+        if n not in seen:
+            seen.add(n)
+            ordered.append(n)
+
+    picked_out: list[dict[str, Any]] = []
+    nonzero = 0
+    max_abs = 0.0
+
+    for name in ordered:
+        row = get_skill_weight_detail(con, name, user_id=user_id)
+        if row is None:
+            picked_out.append({
+                "skill": name,
+                "has_db_row": False,
+                "learned_weight": 0.0,
+                "uses": 0,
+                "referenced": 0,
+                "thumbs_up": 0,
+                "thumbs_down": 0,
+                "net_thumbs": 0,
+                "reference_rate": None,
+                "disabled": False,
+            })
+            continue
+        lw = float(row["learned_weight"])
+        if abs(lw) > 1e-9:
+            nonzero += 1
+        max_abs = max(max_abs, abs(lw))
+        picked_out.append({
+            "skill": name,
+            "has_db_row": True,
+            "learned_weight": row["learned_weight"],
+            "uses": row["uses"],
+            "referenced": row["referenced"],
+            "thumbs_up": row["thumbs_up"],
+            "thumbs_down": row["thumbs_down"],
+            "net_thumbs": row["net_thumbs"],
+            "reference_rate": row["reference_rate"],
+            "disabled": row["disabled"],
+        })
+
+    return {
+        "schema": "feedback_effect/1",
+        "weight_formula": "weight = (referenced/uses - 0.5) * 0.3 + (thumbs_up - thumbs_down) * 0.1; reference_rate=referenced/uses if uses>0 else 0",
+        "routing_applies": "rank_score += learned_weight (disabled skills get large negative score)",
+        "picked": picked_out,
+        "summary": {
+            "picked_count": len(ordered),
+            "picked_with_nonzero_learned_weight": nonzero,
+            "max_abs_learned_weight": round(float(max_abs), 4),
+        },
+    }