mishkan-harness 0.2.0 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +108 -172
- package/bin/mishkan.js +677 -16
- package/docs/design/MISHKAN_decisions.md +450 -8
- package/docs/design/MISHKAN_harness_design.md +1 -33
- package/docs/design/MISHKAN_observability.md +13 -9
- package/docs/engineer/profile-readable.md +201 -0
- package/docs/engineer/profile.md +754 -0
- package/docs/research/graphify-token-saving-poc.md +108 -0
- package/docs/usage/09-workflows.md +74 -4
- package/docs/usage/10-observability.md +37 -13
- package/docs/usage/11-graphify.md +264 -0
- package/docs/usage/12-skill-discovery.md +294 -0
- package/docs/usage/README.md +5 -3
- package/package.json +1 -1
- package/payload/install/settings.hooks.json +24 -0
- package/payload/mishkan/AGENT_SPEC.md +25 -7
- package/payload/mishkan/agents/ahikam.md +4 -2
- package/payload/mishkan/agents/aholiab.md +4 -2
- package/payload/mishkan/agents/asaph.md +5 -2
- package/payload/mishkan/agents/baruch.md +19 -2
- package/payload/mishkan/agents/benaiah.md +4 -2
- package/payload/mishkan/agents/bezalel.md +4 -2
- package/payload/mishkan/agents/caleb.md +4 -2
- package/payload/mishkan/agents/deborah.md +4 -2
- package/payload/mishkan/agents/elasah.md +4 -2
- package/payload/mishkan/agents/eliashib.md +4 -2
- package/payload/mishkan/agents/ezra.md +4 -2
- package/payload/mishkan/agents/hanun.md +5 -2
- package/payload/mishkan/agents/hiram.md +5 -2
- package/payload/mishkan/agents/hizkiah.md +5 -2
- package/payload/mishkan/agents/huldah.md +4 -2
- package/payload/mishkan/agents/huram.md +4 -2
- package/payload/mishkan/agents/hushai.md +5 -2
- package/payload/mishkan/agents/igal.md +4 -2
- package/payload/mishkan/agents/ira.md +5 -2
- package/payload/mishkan/agents/jahaziel.md +5 -2
- package/payload/mishkan/agents/jakin.md +4 -2
- package/payload/mishkan/agents/jehonathan.md +5 -2
- package/payload/mishkan/agents/jehoshaphat.md +4 -2
- package/payload/mishkan/agents/joab.md +5 -2
- package/payload/mishkan/agents/joah.md +5 -2
- package/payload/mishkan/agents/maaseiah.md +4 -2
- package/payload/mishkan/agents/meremoth.md +5 -2
- package/payload/mishkan/agents/meshullam.md +5 -2
- package/payload/mishkan/agents/nathan.md +5 -2
- package/payload/mishkan/agents/nehemiah.md +4 -2
- package/payload/mishkan/agents/obed.md +4 -2
- package/payload/mishkan/agents/oholiab.md +5 -2
- package/payload/mishkan/agents/palal.md +5 -2
- package/payload/mishkan/agents/phinehas.md +4 -2
- package/payload/mishkan/agents/rehum.md +4 -2
- package/payload/mishkan/agents/salma.md +5 -2
- package/payload/mishkan/agents/seraiah.md +4 -2
- package/payload/mishkan/agents/shallum.md +5 -2
- package/payload/mishkan/agents/shaphan.md +4 -2
- package/payload/mishkan/agents/shemaiah.md +4 -2
- package/payload/mishkan/agents/shevna.md +5 -2
- package/payload/mishkan/agents/uriah.md +5 -2
- package/payload/mishkan/agents/zaccur.md +4 -2
- package/payload/mishkan/agents/zadok.md +5 -2
- package/payload/mishkan/agents/zerubbabel.md +4 -2
- package/payload/mishkan/commands/code-graph.md +35 -0
- package/payload/mishkan/commands/eval-baruch.md +57 -0
- package/payload/mishkan/commands/mishkan-org-reference.md +58 -0
- package/payload/mishkan/commands/mishkan-resume.md +19 -5
- package/payload/mishkan/commands/mishkan-skills-misses.md +47 -0
- package/payload/mishkan/commands/mishkan-skills-reindex.md +33 -0
- package/payload/mishkan/commands/skills.md +35 -0
- package/payload/mishkan/commands/sprint-close.md +3 -1
- package/payload/mishkan/hooks/pre-tool-knowledge-route.sh +231 -0
- package/payload/mishkan/hooks/pre-tool-task-skill-route.sh +122 -0
- package/payload/mishkan/hooks/session-start-skill-index.sh +35 -0
- package/payload/mishkan/observability/schema.json +1 -1
- package/payload/mishkan/observability/watch/src/mishkan_watch/__main__.py +84 -2
- package/payload/mishkan/observability/watch/src/mishkan_watch/app.py +64 -3
- package/payload/mishkan/observability/watch/src/mishkan_watch/org_data.py +79 -0
- package/payload/mishkan/observability/watch/src/mishkan_watch/skills_data.py +267 -0
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/agents.py +33 -1
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/knowledge.py +118 -9
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/live.py +102 -2
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/org.py +139 -0
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/skills.py +187 -0
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/usage.py +377 -0
- package/payload/mishkan/observability/watch/src/mishkan_watch/tabs/workflows.py +199 -27
- package/payload/mishkan/observability/watch/src/mishkan_watch/theme.tcss +78 -4
- package/payload/mishkan/observability/watchd/src/mishkan_watchd/__main__.py +36 -6
- package/payload/mishkan/observability/watchd/src/mishkan_watchd/sources/bus_tail.py +16 -3
- package/payload/mishkan/observability/watchd/src/mishkan_watchd/sources/graphify_tail.py +220 -0
- package/payload/mishkan/observability/watchd/src/mishkan_watchd/sources/worktree_poll.py +13 -2
- package/payload/mishkan/observability/watchd/src/mishkan_watchd/state.py +157 -3
- package/payload/mishkan/observability/watchd/tests/test_state.py +6 -2
- package/payload/mishkan/org/org.json +141 -0
- package/payload/mishkan/scripts/pre-commit-eval-gate.sh +50 -0
- package/payload/mishkan/scripts/skill-discovery-indexer.py +462 -0
- package/payload/mishkan/scripts/skill-discovery-misses.py +173 -0
- package/payload/mishkan/scripts/skill-discovery-router.py +457 -0
- package/payload/mishkan/scripts/skill-discovery-smoke.sh +156 -0
- package/payload/mishkan/skills/graphify-query-craft/SKILL.md +162 -0
- package/payload/mishkan/skills/mishkan-init/SKILL.md +15 -0
- package/payload/mishkan/skills/skill-discovery/SKILL.md +120 -0
- package/payload/mishkan/templates/case-node.schema.json +1 -0
- package/payload/mishkan/templates/observability-log.schema.json +1 -0
- package/payload/mishkan/workflows/README.md +78 -13
- package/payload/mishkan/workflows/chosheb-feature-ship.js +75 -0
- package/payload/mishkan/workflows/migdal-dr-drill.js +94 -0
- package/payload/mishkan/workflows/migdal-infra-change.js +67 -0
- package/payload/mishkan/workflows/mishkan-blast-radius.js +144 -0
- package/payload/mishkan/workflows/mishkan-init.js +58 -0
- package/payload/mishkan/workflows/mishkan-knowledge-gap-discovery.js +121 -0
- package/payload/mishkan/workflows/mishkan-standards-rollout.js +153 -0
- package/payload/mishkan/workflows/mishmar-security-gate.js +83 -0
- package/payload/mishkan/workflows/panim-ds-rollout.js +83 -0
- package/payload/mishkan/workflows/sefer-release-notes.js +85 -0
- package/payload/mishkan/workflows/yasad-data-migration-wave.js +78 -0
- package/payload/mishkan/workflows/yasad-schema-evolution.js +76 -0
- package/payload/mishkan/templates/user-CLAUDE.md +0 -62
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
skill-discovery-router — read index.json, score skills for a task, emit 3 buckets.
|
|
4
|
+
|
|
5
|
+
Matching mechanisms (in this order, combined):
|
|
6
|
+
1. Trigger-phrase match — substring-tokenize task; weighted hits
|
|
7
|
+
(trigger phrase = 3.0, description keyword = 1.0).
|
|
8
|
+
2. Category prior — when invoked from inside a workflow with a
|
|
9
|
+
relevant_skill_categories filter, hits in those
|
|
10
|
+
categories multiplied by 1.5.
|
|
11
|
+
3. TF-IDF fallback — pure-stdlib TF-IDF over (description + triggers).
|
|
12
|
+
Used to *augment* trigger results when trigger
|
|
13
|
+
matches yield fewer than 3 entries.
|
|
14
|
+
|
|
15
|
+
Output JSON shape (stdout):
|
|
16
|
+
{
|
|
17
|
+
"task_summary": "...",
|
|
18
|
+
"must_load": [ {name, score, origin, category, source_path}, ... ] // ≤ 3
|
|
19
|
+
"should_consider": [ ... ] // ≤ 5
|
|
20
|
+
"adjacent": [ ... ] // ≤ 5
|
|
21
|
+
"total_returned": <int <= 13>,
|
|
22
|
+
"warnings": [ "stale_entry_dropped: <name>", ... ],
|
|
23
|
+
"stale_rebuild_needed": <bool>
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
Fail-open: any error → empty buckets, miss logged, exit 0.
|
|
27
|
+
|
|
28
|
+
Stdlib only.
|
|
29
|
+
|
|
30
|
+
CLI:
|
|
31
|
+
python3 skill-discovery-router.py --task "<task description>"
|
|
32
|
+
[--workflow <name>]
|
|
33
|
+
[--relevant-categories cat1,cat2]
|
|
34
|
+
[--threshold-high N]
|
|
35
|
+
[--threshold-mid M]
|
|
36
|
+
[--limit 13]
|
|
37
|
+
[--json]
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import argparse
|
|
43
|
+
import json
|
|
44
|
+
import math
|
|
45
|
+
import os
|
|
46
|
+
import re
|
|
47
|
+
import sys
|
|
48
|
+
import time
|
|
49
|
+
from collections import Counter
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
|
|
52
|
+
HOME = Path(os.path.expanduser("~"))
|
|
53
|
+
INDEX_DIR = HOME / ".claude" / "mishkan" / "skill-discovery"
|
|
54
|
+
INDEX_PATH = INDEX_DIR / "index.json"
|
|
55
|
+
MISSES_PATH = INDEX_DIR / "misses.jsonl"
|
|
56
|
+
|
|
57
|
+
STOPWORDS = {
|
|
58
|
+
"the", "a", "an", "and", "or", "of", "for", "to", "in", "on", "with",
|
|
59
|
+
"this", "that", "is", "are", "be", "by", "as", "at", "it", "its", "from",
|
|
60
|
+
"into", "i", "we", "you", "your", "my", "our", "use", "using", "when",
|
|
61
|
+
"how", "what", "which", "do", "does", "did", "should", "would", "could",
|
|
62
|
+
"have", "has", "had", "but", "not", "no", "yes", "if", "else", "then",
|
|
63
|
+
"can", "will", "may", "make", "made", "than", "so", "such", "also",
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9\-_/]{1,}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def tokenize(text: str) -> list[str]:
|
|
70
|
+
out: list[str] = []
|
|
71
|
+
for raw in TOKEN_RE.findall(text or ""):
|
|
72
|
+
t = raw.lower().strip("-_/")
|
|
73
|
+
if not t or t in STOPWORDS or len(t) < 2:
|
|
74
|
+
continue
|
|
75
|
+
out.append(t)
|
|
76
|
+
return out
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _now_iso() -> str:
|
|
80
|
+
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _log_miss(task: str, reason: str) -> None:
|
|
84
|
+
INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
rec = {
|
|
86
|
+
"task_summary": (task or "")[:500],
|
|
87
|
+
"error": reason,
|
|
88
|
+
"timestamp": _now_iso(),
|
|
89
|
+
}
|
|
90
|
+
try:
|
|
91
|
+
with MISSES_PATH.open("a", encoding="utf-8") as f:
|
|
92
|
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
93
|
+
except OSError:
|
|
94
|
+
pass
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def load_index() -> dict | None:
|
|
98
|
+
if not INDEX_PATH.is_file():
|
|
99
|
+
return None
|
|
100
|
+
try:
|
|
101
|
+
return json.loads(INDEX_PATH.read_text(encoding="utf-8"))
|
|
102
|
+
except (OSError, json.JSONDecodeError):
|
|
103
|
+
return None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# ---------------------------------------------------------------------------
|
|
107
|
+
# Scoring
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
TRIGGER_WEIGHT = 3.0
|
|
111
|
+
KEYWORD_WEIGHT = 1.0
|
|
112
|
+
CATEGORY_MULTIPLIER = 1.5
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def score_trigger(entry: dict, task_tokens: set[str], task_text_low: str) -> float:
|
|
116
|
+
"""Substring/token weighted match on triggers + description."""
|
|
117
|
+
score = 0.0
|
|
118
|
+
# Triggers: each trigger phrase contributes once if any of its tokens hit
|
|
119
|
+
for trig in entry.get("triggers", []) or []:
|
|
120
|
+
trig_low = (trig or "").lower()
|
|
121
|
+
trig_tokens = set(tokenize(trig))
|
|
122
|
+
if not trig_tokens:
|
|
123
|
+
continue
|
|
124
|
+
# Substring hit (whole phrase in task text) → full weight
|
|
125
|
+
if len(trig_low) >= 6 and trig_low in task_text_low:
|
|
126
|
+
score += TRIGGER_WEIGHT
|
|
127
|
+
continue
|
|
128
|
+
overlap = trig_tokens & task_tokens
|
|
129
|
+
if overlap:
|
|
130
|
+
# Proportional: full weight if half-or-more of trigger tokens hit
|
|
131
|
+
ratio = len(overlap) / max(1, len(trig_tokens))
|
|
132
|
+
score += TRIGGER_WEIGHT * min(1.0, ratio * 2.0)
|
|
133
|
+
# Description keywords
|
|
134
|
+
desc_tokens = set(tokenize(entry.get("description", "")))
|
|
135
|
+
if desc_tokens:
|
|
136
|
+
overlap = desc_tokens & task_tokens
|
|
137
|
+
score += KEYWORD_WEIGHT * len(overlap)
|
|
138
|
+
# Name match — small bonus
|
|
139
|
+
name_tokens = set(tokenize(entry.get("name", "").replace("-", " ")))
|
|
140
|
+
if name_tokens & task_tokens:
|
|
141
|
+
score += KEYWORD_WEIGHT * 0.5 * len(name_tokens & task_tokens)
|
|
142
|
+
return score
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def build_tfidf(entries: list[dict]) -> tuple[dict[str, float], list[set[str]]]:
|
|
146
|
+
"""IDF map + per-entry token sets (already lowercased)."""
|
|
147
|
+
n = len(entries)
|
|
148
|
+
df: Counter[str] = Counter()
|
|
149
|
+
docs: list[set[str]] = []
|
|
150
|
+
for e in entries:
|
|
151
|
+
toks = set(tokenize(e.get("description", "") + " " + " ".join(e.get("triggers", []) or []) + " " + e.get("name", "")))
|
|
152
|
+
docs.append(toks)
|
|
153
|
+
for t in toks:
|
|
154
|
+
df[t] += 1
|
|
155
|
+
idf = {t: math.log((1 + n) / (1 + c)) + 1.0 for t, c in df.items()}
|
|
156
|
+
return idf, docs
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def score_tfidf(task_tokens: list[str], idf: dict[str, float], doc_tokens: set[str]) -> float:
|
|
160
|
+
if not doc_tokens:
|
|
161
|
+
return 0.0
|
|
162
|
+
tf = Counter(task_tokens)
|
|
163
|
+
score = 0.0
|
|
164
|
+
for t, count in tf.items():
|
|
165
|
+
if t in doc_tokens:
|
|
166
|
+
score += count * idf.get(t, 0.0)
|
|
167
|
+
return score
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
# Main routing
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
|
|
174
|
+
def route(
|
|
175
|
+
task: str,
|
|
176
|
+
relevant_categories: list[str] | None = None,
|
|
177
|
+
workflow: str | None = None,
|
|
178
|
+
threshold_high: float = 4.0,
|
|
179
|
+
threshold_mid: float = 1.5,
|
|
180
|
+
limit: int = 13,
|
|
181
|
+
) -> dict:
|
|
182
|
+
payload = load_index()
|
|
183
|
+
warnings: list[str] = []
|
|
184
|
+
|
|
185
|
+
if payload is None:
|
|
186
|
+
_log_miss(task, "index_missing_or_unreadable")
|
|
187
|
+
return {
|
|
188
|
+
"task_summary": task,
|
|
189
|
+
"must_load": [], "should_consider": [], "adjacent": [],
|
|
190
|
+
"total_returned": 0,
|
|
191
|
+
"warnings": ["index_missing_or_unreadable — run /mishkan-skills-reindex"],
|
|
192
|
+
"stale_rebuild_needed": True,
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
entries = payload.get("entries", []) or []
|
|
196
|
+
|
|
197
|
+
# Stale-index protection: drop entries whose source_path no longer exists.
|
|
198
|
+
live: list[dict] = []
|
|
199
|
+
stale_rebuild_needed = False
|
|
200
|
+
for e in entries:
|
|
201
|
+
sp = e.get("source_path")
|
|
202
|
+
if sp and Path(sp).exists():
|
|
203
|
+
live.append(e)
|
|
204
|
+
else:
|
|
205
|
+
stale_rebuild_needed = True
|
|
206
|
+
warnings.append(f"stale_entry_dropped:{e.get('name')}")
|
|
207
|
+
entries = live
|
|
208
|
+
|
|
209
|
+
if not entries:
|
|
210
|
+
_log_miss(task, "no_live_entries")
|
|
211
|
+
return {
|
|
212
|
+
"task_summary": task,
|
|
213
|
+
"must_load": [], "should_consider": [], "adjacent": [],
|
|
214
|
+
"total_returned": 0,
|
|
215
|
+
"warnings": warnings or ["no_live_entries"],
|
|
216
|
+
"stale_rebuild_needed": True,
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
task_text_low = (task or "").lower()
|
|
220
|
+
task_tokens_list = tokenize(task)
|
|
221
|
+
task_tokens = set(task_tokens_list)
|
|
222
|
+
|
|
223
|
+
cat_set = {c.strip().lower() for c in (relevant_categories or []) if c.strip()}
|
|
224
|
+
|
|
225
|
+
# Pass 1: trigger + description scoring
|
|
226
|
+
scored: list[tuple[float, dict, dict]] = []
|
|
227
|
+
for e in entries:
|
|
228
|
+
s = score_trigger(e, task_tokens, task_text_low)
|
|
229
|
+
if cat_set and (e.get("category", "").lower() in cat_set):
|
|
230
|
+
s *= CATEGORY_MULTIPLIER
|
|
231
|
+
scored.append((s, e, {"trigger_score": s}))
|
|
232
|
+
|
|
233
|
+
above_trigger = [(s, e, m) for (s, e, m) in scored if s > 0]
|
|
234
|
+
above_trigger.sort(key=lambda x: x[0], reverse=True)
|
|
235
|
+
|
|
236
|
+
# Pass 2: TF-IDF fallback when trigger results thin
|
|
237
|
+
if len(above_trigger) < 3:
|
|
238
|
+
idf, docs = build_tfidf(entries)
|
|
239
|
+
tfidf_scored: list[tuple[float, dict]] = []
|
|
240
|
+
for e, dtoks in zip(entries, docs):
|
|
241
|
+
ts = score_tfidf(task_tokens_list, idf, dtoks)
|
|
242
|
+
if cat_set and (e.get("category", "").lower() in cat_set):
|
|
243
|
+
ts *= CATEGORY_MULTIPLIER
|
|
244
|
+
tfidf_scored.append((ts, e))
|
|
245
|
+
tfidf_scored.sort(key=lambda x: x[0], reverse=True)
|
|
246
|
+
# Inject top TF-IDF entries that aren't already in above_trigger,
|
|
247
|
+
# mapped to a lower-weighted comparable score.
|
|
248
|
+
existing = {id(e) for _s, e, _m in above_trigger}
|
|
249
|
+
max_tfidf = tfidf_scored[0][0] if tfidf_scored else 0.0
|
|
250
|
+
for ts, e in tfidf_scored:
|
|
251
|
+
if id(e) in existing or ts <= 0:
|
|
252
|
+
continue
|
|
253
|
+
# Normalise TF-IDF onto the trigger-score scale (cap below threshold_high
|
|
254
|
+
# so trigger hits always rank above pure TF-IDF unless very strong).
|
|
255
|
+
normalised = (ts / max_tfidf) * (threshold_high - 0.5) if max_tfidf else 0.0
|
|
256
|
+
above_trigger.append((normalised, e, {"trigger_score": 0.0, "tfidf_score": ts, "tfidf_normalised": normalised}))
|
|
257
|
+
above_trigger.sort(key=lambda x: x[0], reverse=True)
|
|
258
|
+
|
|
259
|
+
# Bucket
|
|
260
|
+
must_load: list[dict] = []
|
|
261
|
+
should_consider: list[dict] = []
|
|
262
|
+
adjacent: list[dict] = []
|
|
263
|
+
|
|
264
|
+
for s, e, meta in above_trigger:
|
|
265
|
+
item = {
|
|
266
|
+
"name": e.get("name"),
|
|
267
|
+
"score": round(float(s), 3),
|
|
268
|
+
"origin": e.get("origin"),
|
|
269
|
+
"category": e.get("category"),
|
|
270
|
+
"source_path": e.get("source_path"),
|
|
271
|
+
"description": e.get("description"),
|
|
272
|
+
"triggers": e.get("triggers", [])[:3],
|
|
273
|
+
}
|
|
274
|
+
# Mark non-mishkan origin for trust-asymmetry awareness
|
|
275
|
+
if e.get("origin") != "mishkan":
|
|
276
|
+
item["trust"] = f"third-party ({e.get('origin')}); not auto-loadable for stateful operations"
|
|
277
|
+
item["score_breakdown"] = meta
|
|
278
|
+
if s >= threshold_high and len(must_load) < 3:
|
|
279
|
+
must_load.append(item)
|
|
280
|
+
elif s >= threshold_mid and len(should_consider) < 5:
|
|
281
|
+
should_consider.append(item)
|
|
282
|
+
else:
|
|
283
|
+
if len(adjacent) < 5:
|
|
284
|
+
adjacent.append(item)
|
|
285
|
+
|
|
286
|
+
# If buckets all empty, log miss
|
|
287
|
+
if not (must_load or should_consider or adjacent):
|
|
288
|
+
_log_miss(task, "no_match_above_threshold")
|
|
289
|
+
warnings.append("no_match_above_threshold")
|
|
290
|
+
|
|
291
|
+
total = len(must_load) + len(should_consider) + len(adjacent)
|
|
292
|
+
# Enforce hard cap of 13 (sum of caps already 13, but trim defensively)
|
|
293
|
+
if total > limit:
|
|
294
|
+
# Trim adjacent first, then should_consider
|
|
295
|
+
while total > limit and adjacent:
|
|
296
|
+
adjacent.pop()
|
|
297
|
+
total -= 1
|
|
298
|
+
while total > limit and should_consider:
|
|
299
|
+
should_consider.pop()
|
|
300
|
+
total -= 1
|
|
301
|
+
|
|
302
|
+
return {
|
|
303
|
+
"task_summary": task,
|
|
304
|
+
"must_load": must_load,
|
|
305
|
+
"should_consider": should_consider,
|
|
306
|
+
"adjacent": adjacent,
|
|
307
|
+
"total_returned": total,
|
|
308
|
+
"warnings": warnings,
|
|
309
|
+
"stale_rebuild_needed": stale_rebuild_needed,
|
|
310
|
+
"workflow": workflow,
|
|
311
|
+
"relevant_categories": sorted(cat_set) or None,
|
|
312
|
+
"thresholds": {"high": threshold_high, "mid": threshold_mid},
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _truncate_desc(s: str, n: int = 110) -> str:
|
|
317
|
+
s = re.sub(r"\s+", " ", (s or "").strip())
|
|
318
|
+
if len(s) <= n:
|
|
319
|
+
return s
|
|
320
|
+
return s[: n - 1].rstrip() + "…"
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def render_injection(result: dict, max_tokens: int = 600) -> str:
|
|
324
|
+
"""Render the 3-bucket router result as a compact advisory markdown block.
|
|
325
|
+
|
|
326
|
+
Hard caps for Phase 2 auto-injection: 3 must_load + 3 should_consider,
|
|
327
|
+
adjacent is dropped (it's awareness-only and would pad without justifying
|
|
328
|
+
its tokens at injection time).
|
|
329
|
+
|
|
330
|
+
Trust marker preserved: non-mishkan entries get a `(community)` suffix.
|
|
331
|
+
|
|
332
|
+
Returns "" when every relevant bucket is empty — the caller should then
|
|
333
|
+
skip injection entirely rather than prepending "no skills found".
|
|
334
|
+
|
|
335
|
+
Token budget enforced naively: 4 chars ≈ 1 token rule of thumb. If the
|
|
336
|
+
rendered block exceeds the budget, drop should_consider entries first.
|
|
337
|
+
"""
|
|
338
|
+
must = (result.get("must_load") or [])[:3]
|
|
339
|
+
cons = (result.get("should_consider") or [])[:3]
|
|
340
|
+
if not must and not cons:
|
|
341
|
+
return ""
|
|
342
|
+
|
|
343
|
+
def _line(item: dict) -> str:
|
|
344
|
+
name = item.get("name", "")
|
|
345
|
+
desc = _truncate_desc(item.get("description", ""))
|
|
346
|
+
suffix = "" if item.get("origin") == "mishkan" else " (community)"
|
|
347
|
+
return f"- {name}{suffix}: {desc}"
|
|
348
|
+
|
|
349
|
+
parts: list[str] = ["## Discovered skills (advisory)", ""]
|
|
350
|
+
if must:
|
|
351
|
+
parts.append("**Load now (high relevance):**")
|
|
352
|
+
for it in must:
|
|
353
|
+
parts.append(_line(it))
|
|
354
|
+
parts.append("")
|
|
355
|
+
if cons:
|
|
356
|
+
parts.append("**Consider:**")
|
|
357
|
+
for it in cons:
|
|
358
|
+
parts.append(_line(it))
|
|
359
|
+
parts.append("")
|
|
360
|
+
parts.append(
|
|
361
|
+
"These skills were surfaced by the harness's skill-discovery router. "
|
|
362
|
+
"Loading is your call."
|
|
363
|
+
)
|
|
364
|
+
block = "\n".join(parts).rstrip() + "\n"
|
|
365
|
+
|
|
366
|
+
# Hard-cap by token estimate; drop should_consider lines tail-first.
|
|
367
|
+
budget_chars = max_tokens * 4
|
|
368
|
+
while len(block) > budget_chars and cons:
|
|
369
|
+
cons.pop()
|
|
370
|
+
parts = ["## Discovered skills (advisory)", ""]
|
|
371
|
+
if must:
|
|
372
|
+
parts.append("**Load now (high relevance):**")
|
|
373
|
+
for it in must:
|
|
374
|
+
parts.append(_line(it))
|
|
375
|
+
parts.append("")
|
|
376
|
+
if cons:
|
|
377
|
+
parts.append("**Consider:**")
|
|
378
|
+
for it in cons:
|
|
379
|
+
parts.append(_line(it))
|
|
380
|
+
parts.append("")
|
|
381
|
+
parts.append(
|
|
382
|
+
"These skills were surfaced by the harness's skill-discovery router. "
|
|
383
|
+
"Loading is your call."
|
|
384
|
+
)
|
|
385
|
+
block = "\n".join(parts).rstrip() + "\n"
|
|
386
|
+
return block
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def main() -> int:
|
|
390
|
+
parser = argparse.ArgumentParser(description="Skill-discovery router (advisory)")
|
|
391
|
+
parser.add_argument("--task", default=None, help="Task description; if absent, read from stdin")
|
|
392
|
+
parser.add_argument("--workflow", default=None)
|
|
393
|
+
parser.add_argument("--relevant-categories", default=None,
|
|
394
|
+
help="Comma-separated category names to boost ×1.5")
|
|
395
|
+
parser.add_argument("--threshold-high", type=float, default=4.0)
|
|
396
|
+
parser.add_argument("--threshold-mid", type=float, default=1.5)
|
|
397
|
+
parser.add_argument("--limit", type=int, default=13)
|
|
398
|
+
parser.add_argument("--json", action="store_true", help="(default) emit JSON")
|
|
399
|
+
parser.add_argument("--format", choices=("json", "injection"), default="json",
|
|
400
|
+
help="json (default, full router output) | injection "
|
|
401
|
+
"(compact markdown for PreToolUse hook prepend)")
|
|
402
|
+
parser.add_argument("--max-injection-tokens", type=int, default=600,
|
|
403
|
+
help="Hard cap for --format injection (default 600)")
|
|
404
|
+
args = parser.parse_args()
|
|
405
|
+
|
|
406
|
+
task = args.task
|
|
407
|
+
if not task:
|
|
408
|
+
task = sys.stdin.read().strip()
|
|
409
|
+
if not task:
|
|
410
|
+
# fail-open: empty task → empty buckets / empty injection
|
|
411
|
+
if args.format == "injection":
|
|
412
|
+
return 0
|
|
413
|
+
out = {
|
|
414
|
+
"task_summary": "",
|
|
415
|
+
"must_load": [], "should_consider": [], "adjacent": [],
|
|
416
|
+
"total_returned": 0,
|
|
417
|
+
"warnings": ["empty_task"],
|
|
418
|
+
"stale_rebuild_needed": False,
|
|
419
|
+
}
|
|
420
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
421
|
+
return 0
|
|
422
|
+
|
|
423
|
+
cats = None
|
|
424
|
+
if args.relevant_categories:
|
|
425
|
+
cats = [c.strip() for c in args.relevant_categories.split(",") if c.strip()]
|
|
426
|
+
|
|
427
|
+
try:
|
|
428
|
+
result = route(
|
|
429
|
+
task=task,
|
|
430
|
+
relevant_categories=cats,
|
|
431
|
+
workflow=args.workflow,
|
|
432
|
+
threshold_high=args.threshold_high,
|
|
433
|
+
threshold_mid=args.threshold_mid,
|
|
434
|
+
limit=args.limit,
|
|
435
|
+
)
|
|
436
|
+
except Exception as exc:
|
|
437
|
+
_log_miss(task, f"router_exception: {type(exc).__name__}: {exc}")
|
|
438
|
+
result = {
|
|
439
|
+
"task_summary": task,
|
|
440
|
+
"must_load": [], "should_consider": [], "adjacent": [],
|
|
441
|
+
"total_returned": 0,
|
|
442
|
+
"warnings": [f"router_exception: {type(exc).__name__}"],
|
|
443
|
+
"stale_rebuild_needed": False,
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
if args.format == "injection":
|
|
447
|
+
block = render_injection(result, max_tokens=args.max_injection_tokens)
|
|
448
|
+
if block:
|
|
449
|
+
sys.stdout.write(block)
|
|
450
|
+
return 0
|
|
451
|
+
|
|
452
|
+
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
453
|
+
return 0
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
if __name__ == "__main__":
|
|
457
|
+
sys.exit(main())
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# skill-discovery-smoke — end-to-end smoke test for the discovery layer.
|
|
3
|
+
#
|
|
4
|
+
# Runs the indexer against the harness payload itself (so we don't depend
|
|
5
|
+
# on a live ~/.claude/mishkan install), asserts non-zero entry count,
|
|
6
|
+
# runs the router on a known query, asserts a non-empty bucket result.
|
|
7
|
+
#
|
|
8
|
+
# Exit codes:
|
|
9
|
+
# 0 all assertions pass
|
|
10
|
+
# 1 setup failure
|
|
11
|
+
# 2 indexer assertion failure
|
|
12
|
+
# 3 router assertion failure
|
|
13
|
+
#
|
|
14
|
+
# stdlib only, fail-fast. Safe to run anywhere — uses a temp HOME so it
|
|
15
|
+
# never touches the real ~/.claude tree.
|
|
16
|
+
|
|
17
|
+
set -euo pipefail
|
|
18
|
+
|
|
19
|
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
20
|
+
REPO_ROOT="$(cd "$HERE/../../.." && pwd)"
|
|
21
|
+
INDEXER="$HERE/skill-discovery-indexer.py"
|
|
22
|
+
ROUTER="$HERE/skill-discovery-router.py"
|
|
23
|
+
|
|
24
|
+
if [[ ! -f "$INDEXER" ]]; then
|
|
25
|
+
echo "smoke: indexer missing at $INDEXER" >&2
|
|
26
|
+
exit 1
|
|
27
|
+
fi
|
|
28
|
+
if [[ ! -f "$ROUTER" ]]; then
|
|
29
|
+
echo "smoke: router missing at $ROUTER" >&2
|
|
30
|
+
exit 1
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
# Set up a fake HOME so we point the indexer at the harness payload as if
|
|
34
|
+
# it were ~/.claude/mishkan/skills/.
|
|
35
|
+
TMPHOME="$(mktemp -d)"
|
|
36
|
+
trap 'rm -rf "$TMPHOME"' EXIT
|
|
37
|
+
|
|
38
|
+
mkdir -p "$TMPHOME/.claude/mishkan"
|
|
39
|
+
ln -s "$REPO_ROOT/payload/mishkan/skills" "$TMPHOME/.claude/mishkan/skills"
|
|
40
|
+
|
|
41
|
+
export HOME="$TMPHOME"
|
|
42
|
+
|
|
43
|
+
echo "smoke: running indexer (--rebuild) against harness payload..."
|
|
44
|
+
INDEX_OUT="$(python3 "$INDEXER" --rebuild --cwd "$REPO_ROOT")"
|
|
45
|
+
echo "smoke: indexer output: $INDEX_OUT"
|
|
46
|
+
|
|
47
|
+
INDEX_PATH="$TMPHOME/.claude/mishkan/skill-discovery/index.json"
|
|
48
|
+
if [[ ! -f "$INDEX_PATH" ]]; then
|
|
49
|
+
echo "smoke: FAIL — index.json was not written at $INDEX_PATH" >&2
|
|
50
|
+
exit 2
|
|
51
|
+
fi
|
|
52
|
+
|
|
53
|
+
COUNT="$(python3 -c "import json,sys; p=json.load(open('$INDEX_PATH')); print(p['meta']['count'])")"
|
|
54
|
+
echo "smoke: indexed entries: $COUNT"
|
|
55
|
+
if [[ "$COUNT" -lt 5 ]]; then
|
|
56
|
+
echo "smoke: FAIL — expected at least 5 entries, got $COUNT" >&2
|
|
57
|
+
exit 2
|
|
58
|
+
fi
|
|
59
|
+
|
|
60
|
+
echo "smoke: running router on a known query..."
|
|
61
|
+
ROUTER_OUT="$(python3 "$ROUTER" --task "Write the architecture document for a new project: bounded contexts, data flow, failure modes." --workflow mishkan-init --relevant-categories craft,mishkan-workflow,docs)"
|
|
62
|
+
echo "$ROUTER_OUT" | python3 -m json.tool >/dev/null || {
|
|
63
|
+
echo "smoke: FAIL — router output is not valid JSON" >&2
|
|
64
|
+
echo "$ROUTER_OUT" >&2
|
|
65
|
+
exit 3
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
TOTAL="$(echo "$ROUTER_OUT" | python3 -c "import json,sys; print(json.load(sys.stdin)['total_returned'])")"
|
|
69
|
+
echo "smoke: router total_returned: $TOTAL"
|
|
70
|
+
if [[ "$TOTAL" -lt 1 ]]; then
|
|
71
|
+
echo "smoke: FAIL — router returned empty buckets for a clearly matching query" >&2
|
|
72
|
+
echo "$ROUTER_OUT" >&2
|
|
73
|
+
exit 3
|
|
74
|
+
fi
|
|
75
|
+
|
|
76
|
+
# Hard cap check
|
|
77
|
+
if [[ "$TOTAL" -gt 13 ]]; then
|
|
78
|
+
echo "smoke: FAIL — router exceeded the 13-entry cap ($TOTAL)" >&2
|
|
79
|
+
exit 3
|
|
80
|
+
fi
|
|
81
|
+
|
|
82
|
+
echo "smoke: indexer + router OK ($COUNT entries indexed, $TOTAL surfaced)."
|
|
83
|
+
|
|
84
|
+
# -----------------------------------------------------------------------------
|
|
85
|
+
# Phase 2 smoke — --format injection + PreToolUse hook timing.
|
|
86
|
+
# -----------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
echo "smoke: Phase 2 — --format injection (non-empty case)..."
|
|
89
|
+
INJ_GOOD="$(python3 "$ROUTER" \
|
|
90
|
+
--task "Write the architecture document for a new project: bounded contexts, data flow, failure modes." \
|
|
91
|
+
--workflow mishkan-init \
|
|
92
|
+
--relevant-categories craft,mishkan-workflow,docs \
|
|
93
|
+
--format injection 2>/dev/null || true)"
|
|
94
|
+
if [[ -z "$INJ_GOOD" ]]; then
|
|
95
|
+
echo "smoke: FAIL — --format injection produced empty output for a known-good query" >&2
|
|
96
|
+
exit 3
|
|
97
|
+
fi
|
|
98
|
+
if ! printf '%s' "$INJ_GOOD" | grep -q "Discovered skills (advisory)"; then
|
|
99
|
+
echo "smoke: FAIL — injection block missing the advisory header" >&2
|
|
100
|
+
printf '%s\n' "$INJ_GOOD" >&2
|
|
101
|
+
exit 3
|
|
102
|
+
fi
|
|
103
|
+
# Hard cap check: 600 tokens ≈ 2400 chars rule-of-thumb. Allow some slack
|
|
104
|
+
# (3000) — what we're guarding against is unbounded growth, not the exact
|
|
105
|
+
# token count which depends on Claude's tokeniser.
|
|
106
|
+
INJ_LEN=${#INJ_GOOD}
|
|
107
|
+
if (( INJ_LEN > 3000 )); then
|
|
108
|
+
echo "smoke: FAIL — injection block exceeded the soft char cap ($INJ_LEN > 3000)" >&2
|
|
109
|
+
exit 3
|
|
110
|
+
fi
|
|
111
|
+
echo "smoke: injection block OK ($INJ_LEN chars)"
|
|
112
|
+
|
|
113
|
+
echo "smoke: Phase 2 — --format injection (empty-bucket case)..."
|
|
114
|
+
# A task with no plausible match anywhere in the harness skill set. We use
|
|
115
|
+
# a string of gibberish unlikely to score on any trigger or description.
|
|
116
|
+
INJ_BAD="$(python3 "$ROUTER" \
|
|
117
|
+
--task "xqzzv frobnitz blortwhomp threnody unicorn snorblat" \
|
|
118
|
+
--format injection 2>/dev/null || true)"
|
|
119
|
+
if [[ -n "$INJ_BAD" ]]; then
|
|
120
|
+
echo "smoke: FAIL — --format injection produced output for an empty-bucket query" >&2
|
|
121
|
+
echo " Got: $INJ_BAD" >&2
|
|
122
|
+
exit 3
|
|
123
|
+
fi
|
|
124
|
+
echo "smoke: empty-bucket injection produces no output (correct)"
|
|
125
|
+
|
|
126
|
+
# -----------------------------------------------------------------------------
|
|
127
|
+
# PreToolUse hook timing — verify the auto-injection hook completes under
|
|
128
|
+
# the 100 ms p95 contract on a known-good query. We measure the worst of
|
|
129
|
+
# 5 runs (cold + warm) and fail at > 250 ms to give CI headroom on slow
|
|
130
|
+
# runners; the design target is 100 ms p95.
|
|
131
|
+
# -----------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
HOOK="$REPO_ROOT/payload/mishkan/hooks/pre-tool-task-skill-route.sh"
|
|
134
|
+
if [[ -f "$HOOK" && $(command -v jq >/dev/null 2>&1 && echo y) == "y" ]]; then
|
|
135
|
+
echo "smoke: Phase 2 — PreToolUse hook timing..."
|
|
136
|
+
HOOK_PAYLOAD='{"session_id":"smoke","tool_name":"Task","tool_input":{"prompt":"Write the architecture document for a new project: bounded contexts, data flow, failure modes."}}'
|
|
137
|
+
MAX_MS=0
|
|
138
|
+
for i in 1 2 3 4 5; do
|
|
139
|
+
T0=$(python3 -c "import time;print(int(time.time()*1000))")
|
|
140
|
+
printf '%s' "$HOOK_PAYLOAD" | bash "$HOOK" >/dev/null 2>&1 || true
|
|
141
|
+
T1=$(python3 -c "import time;print(int(time.time()*1000))")
|
|
142
|
+
DT=$(( T1 - T0 ))
|
|
143
|
+
echo " run $i: ${DT} ms"
|
|
144
|
+
(( DT > MAX_MS )) && MAX_MS=$DT
|
|
145
|
+
done
|
|
146
|
+
if (( MAX_MS > 250 )); then
|
|
147
|
+
echo "smoke: FAIL — hook worst-of-5 ${MAX_MS} ms exceeds the 250 ms CI ceiling (target 100 ms p95)" >&2
|
|
148
|
+
exit 3
|
|
149
|
+
fi
|
|
150
|
+
echo "smoke: PreToolUse hook OK (worst-of-5: ${MAX_MS} ms)"
|
|
151
|
+
else
|
|
152
|
+
echo "smoke: PreToolUse hook timing skipped (jq missing or hook absent)"
|
|
153
|
+
fi
|
|
154
|
+
|
|
155
|
+
echo "smoke: Phase 2 assertions OK."
|
|
156
|
+
exit 0
|