@heytherevibin/skillforge 0.8.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/CONTRIBUTING.md +30 -19
- package/README.md +243 -235
- package/RELEASING.md +19 -7
- package/SECURITY.md +61 -13
- package/STRATEGY.md +40 -14
- package/bin/cli.js +112 -5
- package/ci/bundle-gate.json +4 -0
- package/lib/host-setup.js +312 -0
- package/lib/templates/claude-code-skillforge-global.md +19 -0
- package/lib/templates/cursor-skillforge-global.md +16 -0
- package/package.json +3 -2
- package/python/app/eval_cli.py +133 -0
- package/python/app/feedback_meta.py +96 -0
- package/python/app/health_cli.py +160 -0
- package/python/app/main.py +261 -22
- package/python/app/materialize.py +72 -4
- package/python/app/mcp_contract.py +13 -1
- package/python/app/mcp_server.py +124 -27
- package/python/app/route_cli.py +32 -13
- package/python/app/route_eval_harness.py +98 -0
- package/python/app/route_policies.py +110 -0
- package/python/app/route_quality.py +99 -0
- package/python/app/routing_signals.py +60 -0
- package/python/app/weights_cli.py +152 -0
- package/python/fixtures/route_eval/smoke.json +18 -0
- package/python/tests/test_feedback_weights.py +77 -0
- package/python/tests/test_materialize.py +51 -0
- package/python/tests/test_mcp_contract.py +117 -0
- package/python/tests/test_route_eval_harness.py +45 -0
- package/python/tests/test_route_quality.py +120 -0
- package/python/tests/test_routing_overlay.py +55 -0
- package/python/tests/test_routing_signals.py +35 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Preflight / health checks for Skillforge (paths, catalog, optional full router load)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import sqlite3
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from app.db_paths import resolve_orchestrator_db
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _bundled_skills_dir() -> Path:
|
|
15
|
+
return Path(os.getenv("SKILLFORGE_BUNDLED_SKILLS", "./skills"))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _user_skills_dir() -> Path:
|
|
19
|
+
return Path(os.getenv("SKILLFORGE_USER_SKILLS", str(Path.home() / ".skillforge" / "skills")))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _parse_args(argv: list[str] | None) -> argparse.Namespace:
|
|
23
|
+
p = argparse.ArgumentParser(
|
|
24
|
+
description="Check Skillforge install paths, skill catalog, and optionally load the embedding router."
|
|
25
|
+
)
|
|
26
|
+
p.add_argument(
|
|
27
|
+
"--quick",
|
|
28
|
+
action="store_true",
|
|
29
|
+
help="Skip embedding model load (fast; checks paths + skill file counts only).",
|
|
30
|
+
)
|
|
31
|
+
p.add_argument(
|
|
32
|
+
"--project-root",
|
|
33
|
+
default="",
|
|
34
|
+
help="If set, also checks <root>/.skillforge/ DB path resolution.",
|
|
35
|
+
)
|
|
36
|
+
p.add_argument("--json", action="store_true", help="Machine-readable output on stdout.")
|
|
37
|
+
return p.parse_args(argv)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _count_skill_md(root: Path) -> int:
|
|
41
|
+
if not root.is_dir():
|
|
42
|
+
return 0
|
|
43
|
+
return sum(1 for _ in root.glob("*/SKILL.md"))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_health(*, quick: bool, project_root: str, json_out: bool) -> int:
|
|
47
|
+
checks: list[dict] = []
|
|
48
|
+
failed = False
|
|
49
|
+
|
|
50
|
+
bundled = _bundled_skills_dir()
|
|
51
|
+
user_skills = _user_skills_dir()
|
|
52
|
+
bundled_n = _count_skill_md(bundled)
|
|
53
|
+
|
|
54
|
+
b_ok = bundled.is_dir() and bundled_n > 0
|
|
55
|
+
checks.append({
|
|
56
|
+
"name": "bundled_skills",
|
|
57
|
+
"ok": b_ok,
|
|
58
|
+
"path": str(bundled.resolve()) if bundled.exists() else str(bundled),
|
|
59
|
+
"skill_md_count": bundled_n,
|
|
60
|
+
})
|
|
61
|
+
if not b_ok:
|
|
62
|
+
failed = True
|
|
63
|
+
|
|
64
|
+
user_n = _count_skill_md(user_skills)
|
|
65
|
+
u_ok = True
|
|
66
|
+
u_err: str | None = None
|
|
67
|
+
if not user_skills.is_dir():
|
|
68
|
+
try:
|
|
69
|
+
user_skills.mkdir(parents=True, exist_ok=True)
|
|
70
|
+
except OSError as e:
|
|
71
|
+
u_ok = False
|
|
72
|
+
u_err = str(e)
|
|
73
|
+
failed = True
|
|
74
|
+
checks.append({
|
|
75
|
+
"name": "user_skills",
|
|
76
|
+
"ok": u_ok,
|
|
77
|
+
"path": str(user_skills),
|
|
78
|
+
"skill_md_count": user_n,
|
|
79
|
+
"error": u_err,
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
pr = (project_root or "").strip() or None
|
|
83
|
+
db_path = resolve_orchestrator_db(pr)
|
|
84
|
+
db_ok = True
|
|
85
|
+
db_err: str | None = None
|
|
86
|
+
try:
|
|
87
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
con = sqlite3.connect(str(db_path))
|
|
89
|
+
try:
|
|
90
|
+
con.execute("SELECT 1")
|
|
91
|
+
finally:
|
|
92
|
+
con.close()
|
|
93
|
+
except OSError as e:
|
|
94
|
+
db_ok = False
|
|
95
|
+
db_err = str(e)
|
|
96
|
+
failed = True
|
|
97
|
+
checks.append({
|
|
98
|
+
"name": "orchestrator_db",
|
|
99
|
+
"ok": db_ok,
|
|
100
|
+
"path": str(db_path),
|
|
101
|
+
"error": db_err,
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
router_skill_count: int | None = None
|
|
105
|
+
if not quick:
|
|
106
|
+
try:
|
|
107
|
+
from app.main import build_router_and_skills
|
|
108
|
+
|
|
109
|
+
_router, skills = build_router_and_skills(log=not json_out, log_prefix="[skillforge-health]")
|
|
110
|
+
router_skill_count = len(skills)
|
|
111
|
+
if router_skill_count <= 0:
|
|
112
|
+
failed = True
|
|
113
|
+
checks.append({
|
|
114
|
+
"name": "router_load",
|
|
115
|
+
"ok": router_skill_count > 0,
|
|
116
|
+
"skill_count": router_skill_count,
|
|
117
|
+
"error": None if router_skill_count and router_skill_count > 0 else "empty catalog",
|
|
118
|
+
})
|
|
119
|
+
except Exception as e:
|
|
120
|
+
failed = True
|
|
121
|
+
checks.append({
|
|
122
|
+
"name": "router_load",
|
|
123
|
+
"ok": False,
|
|
124
|
+
"skill_count": None,
|
|
125
|
+
"error": str(e),
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
payload = {
|
|
129
|
+
"ok": not failed,
|
|
130
|
+
"quick": quick,
|
|
131
|
+
"checks": checks,
|
|
132
|
+
}
|
|
133
|
+
if json_out:
|
|
134
|
+
print(json.dumps(payload, indent=2))
|
|
135
|
+
else:
|
|
136
|
+
for c in checks:
|
|
137
|
+
sym = "✓" if c.get("ok") else "✗"
|
|
138
|
+
print(f"{sym} {c['name']}", file=sys.stderr)
|
|
139
|
+
if c.get("path"):
|
|
140
|
+
print(f" path: {c['path']}", file=sys.stderr)
|
|
141
|
+
if c.get("skill_md_count") is not None:
|
|
142
|
+
print(f" SKILL.md count: {c['skill_md_count']}", file=sys.stderr)
|
|
143
|
+
if c.get("skill_count") is not None:
|
|
144
|
+
print(f" router skills: {c['skill_count']}", file=sys.stderr)
|
|
145
|
+
if c.get("error"):
|
|
146
|
+
print(f" error: {c['error']}", file=sys.stderr)
|
|
147
|
+
print("health: ok" if not failed else "health: failed", file=sys.stderr)
|
|
148
|
+
|
|
149
|
+
return 0 if not failed else 1
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def main(argv: list[str] | None = None) -> None:
|
|
153
|
+
args = _parse_args(argv)
|
|
154
|
+
raise SystemExit(
|
|
155
|
+
run_health(quick=bool(args.quick), project_root=args.project_root, json_out=bool(args.json))
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
if __name__ == "__main__":
|
|
160
|
+
main()
|
package/python/app/main.py
CHANGED
|
@@ -32,9 +32,18 @@ from app.project_index import (
|
|
|
32
32
|
retrieve_project_context_items,
|
|
33
33
|
)
|
|
34
34
|
from app.redaction import redaction_enabled, redact_secret_patterns, sanitize_context_items
|
|
35
|
-
from app.
|
|
35
|
+
from app.feedback_meta import build_feedback_effect
|
|
36
|
+
from app.route_policies import (
|
|
37
|
+
build_routing_overlay_payload,
|
|
38
|
+
load_route_policies_config,
|
|
39
|
+
merge_policy_includes,
|
|
40
|
+
merge_project_notes_into_route_query,
|
|
41
|
+
parse_routing_overlay,
|
|
42
|
+
)
|
|
43
|
+
from app.route_quality import build_route_quality, coerce_route_float
|
|
36
44
|
from app.routing_signals import (
|
|
37
45
|
build_route_query_text,
|
|
46
|
+
host_pick_shortlist_lines,
|
|
38
47
|
keyword_overlap_scores,
|
|
39
48
|
normalize_minmax,
|
|
40
49
|
skill_routing_card,
|
|
@@ -54,7 +63,7 @@ ROUTER_MODEL = os.getenv("SKILLFORGE_ROUTER_MODEL", "claude-haiku-4-5-20251001")
|
|
|
54
63
|
TOP_K_CANDIDATES = int(os.getenv("SKILLFORGE_TOP_K", "15"))
|
|
55
64
|
MAX_ACTIVE_SKILLS = int(os.getenv("SKILLFORGE_MAX_ACTIVE", "7"))
|
|
56
65
|
REROUTE_THRESHOLD = float(os.getenv("SKILLFORGE_REROUTE_THRESHOLD", "0.4"))
|
|
57
|
-
# "" | "full" | "embedding" — embedding skips Haiku
|
|
66
|
+
# "" | "full" | "embedding" | "host" — embedding skips Haiku; host skips in-process pick (MCP must pass picked_names).
|
|
58
67
|
SKILLFORGE_ROUTER_MODE = os.getenv("SKILLFORGE_ROUTER_MODE", "").strip().lower()
|
|
59
68
|
# chunks: RAG-style line-bounded chunks from picked skills. full_body: inject entire SKILL.md per pick (legacy).
|
|
60
69
|
SKILLFORGE_CONTEXT_MODE = os.getenv("SKILLFORGE_CONTEXT_MODE", "chunks").strip().lower()
|
|
@@ -105,6 +114,12 @@ def build_router_and_skills(
|
|
|
105
114
|
if mode == "embedding":
|
|
106
115
|
anthropic = None
|
|
107
116
|
router_note = "embedding-only (SKILLFORGE_ROUTER_MODE=embedding)"
|
|
117
|
+
elif mode == "host":
|
|
118
|
+
anthropic = None
|
|
119
|
+
router_note = (
|
|
120
|
+
"host-pick (SKILLFORGE_ROUTER_MODE=host): no in-process router LLM; "
|
|
121
|
+
"first route_skills call returns a shortlist — call again with picked_names"
|
|
122
|
+
)
|
|
108
123
|
elif mode == "full":
|
|
109
124
|
if key:
|
|
110
125
|
anthropic = AsyncAnthropic()
|
|
@@ -446,19 +461,50 @@ class Router:
|
|
|
446
461
|
fused = self._hybrid_alpha * d_norm + (1.0 - self._hybrid_alpha) * s_norm
|
|
447
462
|
return sims, fused
|
|
448
463
|
|
|
449
|
-
def
|
|
464
|
+
def _bias_with_learning_and_overlay(
|
|
465
|
+
self,
|
|
466
|
+
con: sqlite3.Connection,
|
|
467
|
+
biased: np.ndarray,
|
|
468
|
+
user_id: str,
|
|
469
|
+
*,
|
|
470
|
+
exclude_skills: frozenset[str] | None = None,
|
|
471
|
+
routing_boosts: dict[str, float] | None = None,
|
|
472
|
+
) -> None:
|
|
473
|
+
excl = exclude_skills or frozenset()
|
|
474
|
+
boosts = routing_boosts or {}
|
|
475
|
+
for i, s in enumerate(self.skills):
|
|
476
|
+
w, disabled = get_skill_weight(con, s.name, user_id=user_id)
|
|
477
|
+
if disabled or s.name in excl:
|
|
478
|
+
biased[i] = -999.0
|
|
479
|
+
else:
|
|
480
|
+
biased[i] += w
|
|
481
|
+
extra = boosts.get(s.name)
|
|
482
|
+
if extra is not None:
|
|
483
|
+
biased[i] += float(extra)
|
|
484
|
+
|
|
485
|
+
def shortlist(
|
|
486
|
+
self,
|
|
487
|
+
route_query,
|
|
488
|
+
con,
|
|
489
|
+
k=TOP_K_CANDIDATES,
|
|
490
|
+
user_id="",
|
|
491
|
+
*,
|
|
492
|
+
exclude_skills: frozenset[str] | None = None,
|
|
493
|
+
routing_boosts: dict[str, float] | None = None,
|
|
494
|
+
):
|
|
450
495
|
if len(self.skills) == 0:
|
|
451
496
|
return []
|
|
452
497
|
q = self.embed_model.encode(route_query, convert_to_numpy=True)
|
|
453
498
|
q = q / np.linalg.norm(q)
|
|
454
499
|
sims, rank_scores = self._base_routing_scores(route_query, q)
|
|
455
500
|
biased = rank_scores.copy()
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
501
|
+
self._bias_with_learning_and_overlay(
|
|
502
|
+
con,
|
|
503
|
+
biased,
|
|
504
|
+
user_id,
|
|
505
|
+
exclude_skills=exclude_skills,
|
|
506
|
+
routing_boosts=routing_boosts,
|
|
507
|
+
)
|
|
462
508
|
top_idx = np.argsort(-biased)[:k]
|
|
463
509
|
return [(self.skills[i], float(sims[i])) for i in top_idx if biased[i] > -100]
|
|
464
510
|
|
|
@@ -469,6 +515,8 @@ class Router:
|
|
|
469
515
|
*,
|
|
470
516
|
k: int | None = None,
|
|
471
517
|
user_id: str = "",
|
|
518
|
+
exclude_skills: frozenset[str] | None = None,
|
|
519
|
+
routing_boosts: dict[str, float] | None = None,
|
|
472
520
|
) -> list[dict[str, Any]]:
|
|
473
521
|
"""Embedding shortlist with cosine sim, learned weight, and routing score (no LLM)."""
|
|
474
522
|
limit = k if k is not None else TOP_K_CANDIDATES
|
|
@@ -483,12 +531,13 @@ class Router:
|
|
|
483
531
|
)
|
|
484
532
|
)
|
|
485
533
|
biased = rank_scores.copy()
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
534
|
+
self._bias_with_learning_and_overlay(
|
|
535
|
+
con,
|
|
536
|
+
biased,
|
|
537
|
+
user_id,
|
|
538
|
+
exclude_skills=exclude_skills,
|
|
539
|
+
routing_boosts=routing_boosts,
|
|
540
|
+
)
|
|
492
541
|
top_idx = np.argsort(-biased)[:limit]
|
|
493
542
|
out: list[dict[str, Any]] = []
|
|
494
543
|
for i in top_idx:
|
|
@@ -845,6 +894,26 @@ def format_context_items_markdown(context_items: list[dict[str, Any]]) -> str:
|
|
|
845
894
|
return "\n".join(blocks)
|
|
846
895
|
|
|
847
896
|
|
|
897
|
+
def normalize_host_picked_names(
|
|
898
|
+
raw: list[str] | None,
|
|
899
|
+
by_name: dict[str, Skill],
|
|
900
|
+
cap: int,
|
|
901
|
+
) -> list[str]:
|
|
902
|
+
"""Dedupe, order-stable, cap length; only catalog names."""
|
|
903
|
+
out: list[str] = []
|
|
904
|
+
seen: set[str] = set()
|
|
905
|
+
for item in raw or []:
|
|
906
|
+
if not isinstance(item, str):
|
|
907
|
+
continue
|
|
908
|
+
n = item.strip()
|
|
909
|
+
if n in by_name and n not in seen:
|
|
910
|
+
out.append(n)
|
|
911
|
+
seen.add(n)
|
|
912
|
+
if len(out) >= cap:
|
|
913
|
+
break
|
|
914
|
+
return out
|
|
915
|
+
|
|
916
|
+
|
|
848
917
|
async def run_route_turn(
|
|
849
918
|
con: sqlite3.Connection,
|
|
850
919
|
router: Router,
|
|
@@ -855,21 +924,156 @@ async def run_route_turn(
|
|
|
855
924
|
*,
|
|
856
925
|
project_root: str | None = None,
|
|
857
926
|
include_project_rag: bool = False,
|
|
927
|
+
picked_names_from_host: list[str] | None = None,
|
|
928
|
+
picked_names_from_host_supplied: bool = False,
|
|
858
929
|
) -> dict[str, Any]:
|
|
859
930
|
"""Shared routing + session + telemetry for MCP route_skills and ``skillforge route``.
|
|
860
931
|
|
|
861
|
-
|
|
932
|
+
``SKILLFORGE_ROUTER_MODE=host`` without ``picked_names`` returns a tight shortlist only (no ``uses``,
|
|
933
|
+
no skill chunks). Pass ``picked_names`` on the next call to finalize context.
|
|
934
|
+
|
|
935
|
+
When ``picked_names_from_host_supplied`` is True, skips rerank/Haiku and uses the supplied names
|
|
936
|
+
(after validation) in any router mode.
|
|
862
937
|
"""
|
|
863
938
|
sid = session_id or str(uuid.uuid4())
|
|
864
939
|
t0 = time.time()
|
|
865
940
|
route_query = build_route_query_text(prompt, conversation)
|
|
866
|
-
candidates = router.shortlist(route_query, con, user_id=user_id)
|
|
867
|
-
candidates = await router.rerank_candidates_haiku(route_query, conversation, candidates)
|
|
868
|
-
picked_names, reasoning = await router.pick_final(
|
|
869
|
-
prompt, conversation, candidates, route_query=route_query
|
|
870
|
-
)
|
|
871
941
|
pr = (project_root or "").strip()
|
|
872
942
|
policies_cfg = load_route_policies_config(pr or None)
|
|
943
|
+
overlay_audit: list[dict[str, Any]] = []
|
|
944
|
+
exclude_skills, routing_boosts, project_notes_raw = parse_routing_overlay(
|
|
945
|
+
policies_cfg,
|
|
946
|
+
by_name=router._by_name,
|
|
947
|
+
audit_out=overlay_audit,
|
|
948
|
+
)
|
|
949
|
+
route_query = merge_project_notes_into_route_query(route_query, project_notes_raw, pr)
|
|
950
|
+
notes_effective = bool(project_notes_raw.strip() and pr)
|
|
951
|
+
routing_overlay_meta = build_routing_overlay_payload(
|
|
952
|
+
project_root=pr,
|
|
953
|
+
exclude_skills=exclude_skills,
|
|
954
|
+
routing_boosts=routing_boosts,
|
|
955
|
+
project_notes_applied=notes_effective,
|
|
956
|
+
project_notes_len=len(project_notes_raw) if project_notes_raw else 0,
|
|
957
|
+
audit=overlay_audit,
|
|
958
|
+
)
|
|
959
|
+
rules_list_early = policies_cfg.get("rules") if isinstance(policies_cfg.get("rules"), list) else []
|
|
960
|
+
rules_n = len(rules_list_early)
|
|
961
|
+
|
|
962
|
+
host_router = SKILLFORGE_ROUTER_MODE == "host"
|
|
963
|
+
|
|
964
|
+
if host_router and not picked_names_from_host_supplied:
|
|
965
|
+
k = max(3, min(TOP_K_CANDIDATES, int(os.getenv("SKILLFORGE_HOST_PICK_MAX", "12"))))
|
|
966
|
+
facets = router.shortlist_with_facets(
|
|
967
|
+
route_query,
|
|
968
|
+
con,
|
|
969
|
+
k=k,
|
|
970
|
+
user_id=user_id,
|
|
971
|
+
exclude_skills=exclude_skills,
|
|
972
|
+
routing_boosts=routing_boosts,
|
|
973
|
+
)
|
|
974
|
+
candidates = [
|
|
975
|
+
(router._by_name[nm], coerce_route_float(f.get("cosine_similarity")))
|
|
976
|
+
for f in facets
|
|
977
|
+
if (nm := f.get("name")) in router._by_name
|
|
978
|
+
]
|
|
979
|
+
md, rows = host_pick_shortlist_lines(
|
|
980
|
+
prompt=prompt,
|
|
981
|
+
route_query=route_query,
|
|
982
|
+
facet_rows=facets,
|
|
983
|
+
max_candidates=k,
|
|
984
|
+
)
|
|
985
|
+
route_ms = (time.time() - t0) * 1000
|
|
986
|
+
safe_prompt_snip = prompt[:300]
|
|
987
|
+
if redaction_enabled():
|
|
988
|
+
safe_prompt_snip, _ = redact_secret_patterns(prompt[:300])
|
|
989
|
+
route_quality = build_route_quality(
|
|
990
|
+
facet_list=facets,
|
|
991
|
+
router_mode=SKILLFORGE_ROUTER_MODE or "auto",
|
|
992
|
+
router_hybrid=router._hybrid_mode,
|
|
993
|
+
picked_names=[],
|
|
994
|
+
rerouted=False,
|
|
995
|
+
change=0.0,
|
|
996
|
+
policy_rules_loaded=rules_n,
|
|
997
|
+
policy_audit=[],
|
|
998
|
+
host_picked=False,
|
|
999
|
+
host_shortlist_only=True,
|
|
1000
|
+
haiku_rerank_applied=False,
|
|
1001
|
+
pick_path="host_shortlist",
|
|
1002
|
+
)
|
|
1003
|
+
feedback_effect = build_feedback_effect(con, [], user_id=user_id)
|
|
1004
|
+
event = {
|
|
1005
|
+
"type": "host_shortlist",
|
|
1006
|
+
"session_id": sid,
|
|
1007
|
+
"user_id": user_id,
|
|
1008
|
+
"prompt": safe_prompt_snip,
|
|
1009
|
+
"candidates": [{"name": s.name, "score": sc} for s, sc in candidates[:15]],
|
|
1010
|
+
"picked": [],
|
|
1011
|
+
"reasoning": "host_pick_shortlist",
|
|
1012
|
+
"route_ms": round(route_ms, 1),
|
|
1013
|
+
"ts": time.time(),
|
|
1014
|
+
"host_pick_candidates": rows,
|
|
1015
|
+
"policy": {"rules_loaded": rules_n, "audit": []},
|
|
1016
|
+
"route_quality": route_quality,
|
|
1017
|
+
"feedback_effect": feedback_effect,
|
|
1018
|
+
}
|
|
1019
|
+
if routing_overlay_meta is not None:
|
|
1020
|
+
event["routing_overlay"] = routing_overlay_meta
|
|
1021
|
+
log_event(con, sid, "host_shortlist", event, user_id=user_id)
|
|
1022
|
+
con.commit()
|
|
1023
|
+
ret_host: dict[str, Any] = {
|
|
1024
|
+
"session_id": sid,
|
|
1025
|
+
"picked_names": [],
|
|
1026
|
+
"reasoning": (
|
|
1027
|
+
"host_pick_shortlist — choose names from the list; call route_skills again "
|
|
1028
|
+
"with picked_names (reuse session_id if you use sessions)."
|
|
1029
|
+
),
|
|
1030
|
+
"candidates": candidates,
|
|
1031
|
+
"route_ms": route_ms,
|
|
1032
|
+
"rerouted": False,
|
|
1033
|
+
"change": 0.0,
|
|
1034
|
+
"event": event,
|
|
1035
|
+
"context_items": [],
|
|
1036
|
+
"host_pick_shortlist": True,
|
|
1037
|
+
"host_pick_markdown": md,
|
|
1038
|
+
"host_pick_candidates": rows,
|
|
1039
|
+
"route_query": route_query,
|
|
1040
|
+
"route_quality": route_quality,
|
|
1041
|
+
"feedback_effect": feedback_effect,
|
|
1042
|
+
}
|
|
1043
|
+
if routing_overlay_meta is not None:
|
|
1044
|
+
ret_host["routing_overlay"] = routing_overlay_meta
|
|
1045
|
+
return ret_host
|
|
1046
|
+
|
|
1047
|
+
facet_list = router.shortlist_with_facets(
|
|
1048
|
+
route_query,
|
|
1049
|
+
con,
|
|
1050
|
+
k=TOP_K_CANDIDATES,
|
|
1051
|
+
user_id=user_id,
|
|
1052
|
+
exclude_skills=exclude_skills,
|
|
1053
|
+
routing_boosts=routing_boosts,
|
|
1054
|
+
)
|
|
1055
|
+
candidates = [
|
|
1056
|
+
(router._by_name[nm], coerce_route_float(f.get("cosine_similarity")))
|
|
1057
|
+
for f in facet_list
|
|
1058
|
+
if (nm := f.get("name")) in router._by_name
|
|
1059
|
+
]
|
|
1060
|
+
haiku_rerank_applied = False
|
|
1061
|
+
if picked_names_from_host_supplied:
|
|
1062
|
+
picked_names = normalize_host_picked_names(
|
|
1063
|
+
picked_names_from_host, router._by_name, MAX_ACTIVE_SKILLS
|
|
1064
|
+
)
|
|
1065
|
+
reasoning = "host-picked: MCP picked_names"
|
|
1066
|
+
else:
|
|
1067
|
+
names_before = [s.name for s, _ in candidates]
|
|
1068
|
+
rerank_eligible = bool(
|
|
1069
|
+
candidates and router.anthropic is not None and _env_truthy("SKILLFORGE_HAIKU_RERANK", "0")
|
|
1070
|
+
)
|
|
1071
|
+
candidates = await router.rerank_candidates_haiku(route_query, conversation, candidates)
|
|
1072
|
+
names_after = [s.name for s, _ in candidates]
|
|
1073
|
+
haiku_rerank_applied = rerank_eligible and names_before != names_after
|
|
1074
|
+
picked_names, reasoning = await router.pick_final(
|
|
1075
|
+
prompt, conversation, candidates, route_query=route_query
|
|
1076
|
+
)
|
|
873
1077
|
picked_names, policy_audit = merge_policy_includes(
|
|
874
1078
|
prompt,
|
|
875
1079
|
picked_names,
|
|
@@ -984,6 +1188,29 @@ async def run_route_turn(
|
|
|
984
1188
|
|
|
985
1189
|
project_rag_items_count = sum(1 for c in context_items if c.get("path"))
|
|
986
1190
|
|
|
1191
|
+
if picked_names_from_host_supplied:
|
|
1192
|
+
pick_path = "host_picked"
|
|
1193
|
+
elif router.anthropic is None or SKILLFORGE_ROUTER_MODE == "embedding":
|
|
1194
|
+
pick_path = "embedding_top"
|
|
1195
|
+
else:
|
|
1196
|
+
pick_path = "haiku_pick"
|
|
1197
|
+
|
|
1198
|
+
rules_list = policies_cfg.get("rules") if isinstance(policies_cfg.get("rules"), list) else []
|
|
1199
|
+
route_quality = build_route_quality(
|
|
1200
|
+
facet_list=facet_list,
|
|
1201
|
+
router_mode=SKILLFORGE_ROUTER_MODE or "auto",
|
|
1202
|
+
router_hybrid=router._hybrid_mode,
|
|
1203
|
+
picked_names=picked_names,
|
|
1204
|
+
rerouted=rerouted,
|
|
1205
|
+
change=change,
|
|
1206
|
+
policy_rules_loaded=len(rules_list),
|
|
1207
|
+
policy_audit=policy_audit,
|
|
1208
|
+
host_picked=picked_names_from_host_supplied,
|
|
1209
|
+
host_shortlist_only=False,
|
|
1210
|
+
haiku_rerank_applied=haiku_rerank_applied,
|
|
1211
|
+
pick_path=pick_path,
|
|
1212
|
+
)
|
|
1213
|
+
|
|
987
1214
|
reasoning_out = reasoning
|
|
988
1215
|
safe_prompt_snip = prompt[:300]
|
|
989
1216
|
context_redaction_stats: dict[str, Any] = {"enabled": False, "secret_hits": 0, "path_hits": 0}
|
|
@@ -1003,6 +1230,8 @@ async def run_route_turn(
|
|
|
1003
1230
|
for n in picked_names:
|
|
1004
1231
|
update_skill_stat(con, n, "uses", 1, user_id=user_id)
|
|
1005
1232
|
|
|
1233
|
+
feedback_effect = build_feedback_effect(con, picked_names, user_id=user_id)
|
|
1234
|
+
|
|
1006
1235
|
event = {
|
|
1007
1236
|
"type": "route",
|
|
1008
1237
|
"session_id": sid,
|
|
@@ -1035,9 +1264,14 @@ async def run_route_turn(
|
|
|
1035
1264
|
}
|
|
1036
1265
|
for c in context_items[:24]
|
|
1037
1266
|
],
|
|
1267
|
+
"host_picked": bool(picked_names_from_host_supplied),
|
|
1268
|
+
"route_quality": route_quality,
|
|
1269
|
+
"feedback_effect": feedback_effect,
|
|
1038
1270
|
}
|
|
1271
|
+
if routing_overlay_meta is not None:
|
|
1272
|
+
event["routing_overlay"] = routing_overlay_meta
|
|
1039
1273
|
log_event(con, sid, "route", event, user_id=user_id)
|
|
1040
|
-
|
|
1274
|
+
ret_main: dict[str, Any] = {
|
|
1041
1275
|
"session_id": sid,
|
|
1042
1276
|
"picked_names": picked_names,
|
|
1043
1277
|
"reasoning": reasoning_out,
|
|
@@ -1047,4 +1281,9 @@ async def run_route_turn(
|
|
|
1047
1281
|
"change": change,
|
|
1048
1282
|
"event": event,
|
|
1049
1283
|
"context_items": context_items,
|
|
1284
|
+
"route_quality": route_quality,
|
|
1285
|
+
"feedback_effect": feedback_effect,
|
|
1050
1286
|
}
|
|
1287
|
+
if routing_overlay_meta is not None:
|
|
1288
|
+
ret_main["routing_overlay"] = routing_overlay_meta
|
|
1289
|
+
return ret_main
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Write project-local Skillforge bootstrap files (.cursor/
|
|
1
|
+
"""Write project-local Skillforge bootstrap files (.cursor, .claude/commands, PRD, CLAUDE.md)."""
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import re
|
|
@@ -30,7 +30,12 @@ def materialize_project_files(
|
|
|
30
30
|
*,
|
|
31
31
|
merge: bool = True,
|
|
32
32
|
) -> dict[str, Any]:
|
|
33
|
-
"""Create or update Cursor rule, PRD stub, and CLAUDE.md section.
|
|
33
|
+
"""Create or update Cursor rule + command, Claude Code command, PRD stub, and CLAUDE.md section.
|
|
34
|
+
|
|
35
|
+
merge=False skips overwriting existing `.cursor/rules/skillforge.mdc`,
|
|
36
|
+
`.cursor/commands/skillforge.md`, and `.claude/commands/skillforge.md` if they already exist
|
|
37
|
+
(other files still update).
|
|
38
|
+
"""
|
|
34
39
|
root = _safe_root(project_root)
|
|
35
40
|
written: list[str] = []
|
|
36
41
|
|
|
@@ -51,9 +56,11 @@ alwaysApply: false
|
|
|
51
56
|
|
|
52
57
|
# Skillforge
|
|
53
58
|
|
|
54
|
-
When the user invokes **Skillforge**, **`/skillforge
|
|
59
|
+
When the user invokes **Skillforge**, types **`/skillforge`** (Cursor or **Claude Code** project command), or needs deep SKILL.md guidance for this codebase:
|
|
55
60
|
|
|
56
61
|
1. Call **route_skills** with **`project_root`** set to this workspace root (so learning and SQLite live in **`.skillforge/`** here), the user's task, and optional **session_id** (reuse within a thread for reroute stats). If the host sets **`SKILLFORGE_PROJECT_ROOT`**, you can omit **project_root** on each call.
|
|
62
|
+
|
|
63
|
+
If **`SKILLFORGE_ROUTER_MODE=host`**: first **`route_skills`** without **`picked_names`** (shortlist only); then call again with **`picked_names`** for the chosen catalog ids before continuing.
|
|
57
64
|
2. Inject the returned skill bodies into context before continuing.
|
|
58
65
|
3. To refresh project files, call **materialize_project** with **project_root** set to this workspace root and **skill_names** from the last **route_skills** result.
|
|
59
66
|
|
|
@@ -67,6 +74,65 @@ When the user invokes **Skillforge**, **`/skillforge`**, or needs deep SKILL.md
|
|
|
67
74
|
cursor_rule.write_text(mdc_body, encoding="utf-8")
|
|
68
75
|
written.append(str(cursor_rule.relative_to(root)))
|
|
69
76
|
|
|
77
|
+
cursor_cmd = root / ".cursor" / "commands" / "skillforge.md"
|
|
78
|
+
_assert_under(root, cursor_cmd)
|
|
79
|
+
cursor_cmd.parent.mkdir(parents=True, exist_ok=True)
|
|
80
|
+
cmd_body = f"""# Skillforge — route SKILL.md context (MCP)
|
|
81
|
+
|
|
82
|
+
The user chose the **`/skillforge`** project command. Use the **skillforge** MCP server.
|
|
83
|
+
|
|
84
|
+
## Do this
|
|
85
|
+
|
|
86
|
+
1. **`route_skills`**: pass **`project_root`** as this workspace root (absolute path) so SQLite lives in **`.skillforge/`** here. Pass the **current user task** as **`prompt`**. Reuse **`session_id`** across turns in the same thread when the MCP returns one.
|
|
87
|
+
|
|
88
|
+
- **`SKILLFORGE_ROUTER_MODE=host`**: call once **without** **`picked_names`** (shortlist in the response); then call again with **`picked_names`** (exact catalog ids) to load skill context.
|
|
89
|
+
- Optional: pass **`conversation`** when recent turns should influence routing.
|
|
90
|
+
|
|
91
|
+
2. **Use the returned skill text** in your answer (summarize or follow the SKILL.md guidance as appropriate).
|
|
92
|
+
|
|
93
|
+
3. Optionally **`materialize_project`** with the same **`project_root`** and **`skill_names`** from **`route_skills`** to refresh **`.cursor/rules`**, **`.cursor/commands`**, **`.claude/commands`**, and **docs/SKILLFORGE-PRD.md**.
|
|
94
|
+
|
|
95
|
+
## Skills last materialized for this project
|
|
96
|
+
|
|
97
|
+
{skills_md}
|
|
98
|
+
"""
|
|
99
|
+
if cursor_cmd.exists() and not merge:
|
|
100
|
+
pass
|
|
101
|
+
else:
|
|
102
|
+
cursor_cmd.write_text(cmd_body, encoding="utf-8")
|
|
103
|
+
written.append(str(cursor_cmd.relative_to(root)))
|
|
104
|
+
|
|
105
|
+
claude_cmd = root / ".claude" / "commands" / "skillforge.md"
|
|
106
|
+
_assert_under(root, claude_cmd)
|
|
107
|
+
claude_cmd.parent.mkdir(parents=True, exist_ok=True)
|
|
108
|
+
cc_body = f"""---
|
|
109
|
+
description: Use Skillforge MCP route_skills for this repo. Invoke when the user runs /skillforge or needs routed SKILL.md context.
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
# Skillforge — route SKILL.md context (MCP)
|
|
113
|
+
|
|
114
|
+
Project-local **`/skillforge`** for **Claude Code**. Use the **skillforge** MCP server.
|
|
115
|
+
|
|
116
|
+
## Do this
|
|
117
|
+
|
|
118
|
+
1. **`route_skills`**: pass **`project_root`** as this workspace root (absolute path). Pass the **current user task** as **`prompt`**. Reuse **`session_id`** when returned.
|
|
119
|
+
|
|
120
|
+
- **`SKILLFORGE_ROUTER_MODE=host`**: shortlist first, then **`picked_names`**.
|
|
121
|
+
|
|
122
|
+
2. **Use the returned skill text** in your answer.
|
|
123
|
+
|
|
124
|
+
3. Optionally **`materialize_project`** to refresh this file and **`.cursor`** files.
|
|
125
|
+
|
|
126
|
+
## Skills last materialized for this project
|
|
127
|
+
|
|
128
|
+
{skills_md}
|
|
129
|
+
"""
|
|
130
|
+
if claude_cmd.exists() and not merge:
|
|
131
|
+
pass
|
|
132
|
+
else:
|
|
133
|
+
claude_cmd.write_text(cc_body, encoding="utf-8")
|
|
134
|
+
written.append(str(claude_cmd.relative_to(root)))
|
|
135
|
+
|
|
70
136
|
docs_dir = root / "docs"
|
|
71
137
|
docs_dir.mkdir(parents=True, exist_ok=True)
|
|
72
138
|
prd = docs_dir / "SKILLFORGE-PRD.md"
|
|
@@ -82,6 +148,8 @@ Scaffold for goals and milestones. Re-run **materialize_project** after major ro
|
|
|
82
148
|
## How to run Skillforge here
|
|
83
149
|
|
|
84
150
|
- **MCP**: configure the `skillforge` server (e.g. `npx -y @heytherevibin/skillforge mcp`). No API key required for embedding-only routing.
|
|
151
|
+
- **Cursor**: use **`/skillforge`** (**`.cursor/commands/skillforge.md`**) to steer the agent through **route_skills** for this workspace.
|
|
152
|
+
- **Claude Code**: use **`/skillforge`** (**`.claude/commands/skillforge.md`**) the same way.
|
|
85
153
|
- **session_id**: reuse the same value across **route_skills** calls in one conversation thread.
|
|
86
154
|
- Re-bootstrap this project after new skills: **materialize_project** again.
|
|
87
155
|
|
|
@@ -110,7 +178,7 @@ Use [Skillforge](https://www.npmjs.com/package/@heytherevibin/skillforge) for **
|
|
|
110
178
|
|
|
111
179
|
- Call **route_skills** for the current task; reuse **session_id** within a thread.
|
|
112
180
|
- See **docs/SKILLFORGE-PRD.md** for the skill list and runbook.
|
|
113
|
-
-
|
|
181
|
+
- In Cursor, **`/skillforge`** is **`.cursor/commands/skillforge.md`**; in **Claude Code**, **`.claude/commands/skillforge.md`** (after **materialize_project**).
|
|
114
182
|
|
|
115
183
|
**Routed skills (last materialize):** {skill_list}
|
|
116
184
|
|
|
@@ -4,6 +4,9 @@ Versioned ``_meta`` with ``sources[]`` and ``budget``. Schema **1.1** adds
|
|
|
4
4
|
per-chunk ``sources`` when ``context_items`` (RAG chunks) are returned from Phase 1.
|
|
5
5
|
Schema **1.2** adds ``kind: file`` sources and project chunk char counts in ``budget``.
|
|
6
6
|
Schema **1.4** adds optional ``context_redaction`` (hit counts when scrubbing is on).
|
|
7
|
+
Schema **1.5** adds optional ``route_quality`` (shortlist margins, hybrid diagnostics, policy/session).
|
|
8
|
+
Schema **1.6** adds optional ``feedback_effect`` (per-pick learned weights / thumbs / uses used in ranking).
|
|
9
|
+
Schema **1.7** adds optional ``routing_overlay`` (project exclude/boost/notes audit for embedding shortlist).
|
|
7
10
|
"""
|
|
8
11
|
from __future__ import annotations
|
|
9
12
|
|
|
@@ -18,7 +21,7 @@ class _SkillBody(Protocol):
|
|
|
18
21
|
body: str
|
|
19
22
|
|
|
20
23
|
|
|
21
|
-
MCP_RESPONSE_SCHEMA_VERSION = "1.
|
|
24
|
+
MCP_RESPONSE_SCHEMA_VERSION = "1.7"
|
|
22
25
|
|
|
23
26
|
|
|
24
27
|
def build_route_skills_meta(
|
|
@@ -112,6 +115,15 @@ def build_route_skills_meta(
|
|
|
112
115
|
"candidates_preview": candidates_preview,
|
|
113
116
|
"context_items_count": len(context_items or []),
|
|
114
117
|
}
|
|
118
|
+
rq_meta = result.get("route_quality")
|
|
119
|
+
if isinstance(rq_meta, dict):
|
|
120
|
+
meta["route_quality"] = rq_meta
|
|
121
|
+
fb_meta = result.get("feedback_effect")
|
|
122
|
+
if isinstance(fb_meta, dict):
|
|
123
|
+
meta["feedback_effect"] = fb_meta
|
|
124
|
+
ro_meta = result.get("routing_overlay")
|
|
125
|
+
if isinstance(ro_meta, dict):
|
|
126
|
+
meta["routing_overlay"] = ro_meta
|
|
115
127
|
if fusion is not None and fusion.get("enabled"):
|
|
116
128
|
meta["fusion"] = fusion
|
|
117
129
|
if context_redaction is not None:
|