@ionivetech/mugiwara 0.5.0 → 0.5.1

package/scripts/run-evals.ts

@@ -16,12 +16,16 @@ const casesDir = join(root, 'evals', 'cases');
 const skillsDir = join(root, 'content', 'skills');
 const errors: string[] = [];
 
+type BehavioralTest = {
+  task: string;
+  rubric: string[];
+};
+
 type Case = {
   name: string;
   skill: string;
   type?: 'positive' | 'negative' | 'adversarial' | 'lane';
-  task: string;
-  rubric: string[];
+  behavioral?: BehavioralTest[];
   expect_lane?: string;
 };
 
@@ -40,34 +44,65 @@ function validateSuite(): Case[] {
   const cases: Case[] = [];
   const files = listCases(casesDir).sort();
   for (const rel of files) {
-    let c: Case;
-    try { c = JSON.parse(readFileSync(join(casesDir, rel), 'utf8')); }
+    let raw: any;
+    try { raw = JSON.parse(readFileSync(join(casesDir, rel), 'utf8')); }
     catch (e) { errors.push(`${rel}: invalid JSON (${(e as Error).message})`); continue; }
-    if (!c.name || !c.skill || !c.task || !Array.isArray(c.rubric) || !c.rubric.length)
-      errors.push(`${rel}: missing name/skill/task/nonempty rubric`);
-    else if (!existsSync(join(skillsDir, c.skill))) errors.push(`${rel}: unknown skill "${c.skill}"`);
-    if (c.type && !['positive', 'negative', 'adversarial', 'lane'].includes(c.type))
-      errors.push(`${rel}: bad type "${c.type}"`);
-    cases.push(c);
+    if (!raw.name || !raw.skill) { errors.push(`${rel}: missing name/skill`); continue; }
+
+    // skip files with no behavioral section (e.g. _no-skill.json)
+    if (!raw.behavioral || !Array.isArray(raw.behavioral) || raw.behavioral.length === 0) continue;
+    if (raw.skill === '_no-skill') continue;
+
+    // infer type from filename if not declared
+    const type = raw.type ?? (
+      rel.includes('positive') ? 'positive' :
+      rel.includes('negative') ? 'negative' :
+      rel.includes('adversarial') ? 'adversarial' :
+      rel.includes('lane') ? 'lane' :
+      undefined
+    );
+
+    // validate skill exists
+    if (!existsSync(join(skillsDir, raw.skill)))
+      errors.push(`${rel}: unknown skill "${raw.skill}"`);
+
+    if (type && !['positive', 'negative', 'adversarial', 'lane'].includes(type))
+      errors.push(`${rel}: bad type "${type}"`);
+
+    // register one case per behavioral test
+    for (const b of raw.behavioral) {
+      if (!b.task || !Array.isArray(b.rubric) || !b.rubric.length) {
+        errors.push(`${rel}: behavioral entry missing task/nonempty rubric`);
+        continue;
+      }
+      cases.push({
+        name: raw.name,
+        skill: raw.skill,
+        type,
+        behavioral: [b],
+        expect_lane: raw.expect_lane,
+      });
+    }
   }
-  if (!cases.length) { errors.push('no cases in evals/cases/'); }
+
+  if (!cases.length) { errors.push('no behavioral cases in evals/cases/'); }
 
   // coverage gates: the suite must exercise routing in all directions.
-  const types = new Set(cases.map(c => c.type).filter(Boolean));
-  if (cases.filter(c => c.type === 'positive').length < 2) errors.push('need ≥2 positive cases');
-  if (cases.filter(c => c.type === 'negative').length < 2) errors.push('need ≥2 negative cases');
-  if (cases.filter(c => c.type === 'adversarial').length < 2) errors.push('need ≥2 adversarial cases');
-  if (!types.has('lane')) errors.push('need ≥1 lane case');
+  const explicitTypes = cases.filter(c => c.type);
+  const allTypes = new Set(explicitTypes.map(c => c.type).filter(Boolean));
+  if (explicitTypes.filter(c => c.type === 'adversarial').length < 2) errors.push('need >=2 adversarial cases');
+  if (!allTypes.has('lane')) errors.push('need >=1 lane case');
   return cases;
 }
 
 function scoreRubric(answer: string, c: Case): { pass: number; total: number; matched: string[] } {
   const a = answer.toLowerCase();
-  const matched = c.rubric.filter(r => {
+  const rubric = c.behavioral?.[0]?.rubric ?? [];
+  const matched = rubric.filter(r => {
     const terms = r.toLowerCase().split(/[^a-z0-9]+/).filter(w => w.length > 3 && !['does', 'not', 'with', 'into', 'from'].includes(w));
     return terms.some(t => a.includes(t));
   });
-  return { pass: matched.length, total: c.rubric.length, matched };
+  return { pass: matched.length, total: rubric.length, matched };
 }
 
 async function runCases(env: { execFileSync: typeof import('node:child_process').execFileSync; bin: string; pre: string[] }): Promise<void> {
@@ -76,7 +111,8 @@ async function runCases(env: { execFileSync: typeof import('node:child_process')
   const { execFileSync, bin, pre } = env;
   let total = 0, passed = 0;
   for (const c of cases) {
-    const prompt = `Task: ${c.task}\n\nWhich mugiwara skill should run and why? Be concrete.`;
+    const task = c.behavioral?.[0]?.task ?? '';
+    const prompt = `Task: ${task}\n\nWhich mugiwara skill should run and why? Be concrete.`;
     let raw = '';
     try {
       raw = execFileSync(bin, [...pre, prompt], { encoding: 'utf8', timeout: 60000, maxBuffer: 10 * 1024 * 1024 });
@@ -89,7 +125,7 @@ async function runCases(env: { execFileSync: typeof import('node:child_process')
     const pct = Math.round((pass / t) * 100);
     console.log(`${pct >= 70 ? '✓' : '✗'} ${c.name} — ${pass}/${t} rubric (${pct}%)`);
     if (pass < t) {
-      console.log(`    task: ${c.task}`);
+      console.log(`    task: ${task}`);
       console.log(`    matched: ${matched.length ? matched.join('; ') : 'none'}`);
     }
   }

package/scripts/savepoint.sh

@@ -35,7 +35,7 @@ fi
 [ -d .git ] || die "not a git repository"
 
 # --- computed fields ---
-BASE_SHA=$(git merge-base HEAD main 2>/dev/null || git merge-base HEAD master 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "unknown")
+BASE_SHA=$(git merge-base HEAD main 2>/dev/null || git merge-base HEAD master 2>/dev/null || git merge-base HEAD "$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's|refs/remotes/origin/||')" 2>/dev/null || git rev-parse HEAD~1 2>/dev/null || echo "unknown")
 HEAD_SHA=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
 
 CHANGED_FILES=$(git diff --name-only "$BASE_SHA"..HEAD 2>/dev/null || git diff --name-only --cached 2>/dev/null || true)
@@ -43,11 +43,12 @@ FILES_TOUCHED=$( [ -n "$CHANGED_FILES" ] && echo "$CHANGED_FILES" | wc -l | tr -
 
 LOC_DELTA=0
 if [ "$BASE_SHA" != "unknown" ]; then
-  LOC_DELTA=$(git diff --shortstat "$BASE_SHA"..HEAD 2>/dev/null | \
-    awk '{ins=0; del=0; for(i=1;i<=NF;i++){if($i=="insertion") ins=$(i-1); if($i=="deletion") del=$(i-1)} print ins-del}' || echo 0)
+  STAT=$(git diff --shortstat "$BASE_SHA"..HEAD 2>/dev/null || echo "")
+  INS=$(echo "$STAT" | grep -oE '[0-9]+ insertion' | grep -oE '[0-9]+' || echo 0)
+  DEL=$(echo "$STAT" | grep -oE '[0-9]+ deletion'  | grep -oE '[0-9]+' || echo 0)
+  LOC_DELTA=$(( ${INS:-0} - ${DEL:-0} ))
 fi
 [ -z "$LOC_DELTA" ] && LOC_DELTA=0
-[ "$LOC_DELTA" = "0" ] || LOC_DELTA=${LOC_DELTA//[^0-9-]/}
 
 SENSITIVE_PATTERNS="auth/|payment/|billing/|crypto/|secrets/|\.env|config/|migration/|\.sql$|schema\.|\.prisma$"
 SENSITIVE_PATHS=$(echo "$CHANGED_FILES" | grep -E "$SENSITIVE_PATTERNS" 2>/dev/null | tr '\n' ',' | sed 's/,$//' || true)

package/evals/cases/negative-secrets-typo.json

@@ -1,12 +0,0 @@
-{
-  "name": "negative-secrets-typo",
-  "skill": "mugiwara-git",
-  "type": "negative",
-  "task": "Fix a typo in a comment in src/index.ts (one word). This is a trivial change.",
-  "rubric": [
-    "does NOT rank mugiwara-security first (no trust boundary crossed)",
-    "does NOT rank mugiwara-systematic-debugging first (cause known)",
-    "treats it as lane 0 direct work"
-  ],
-  "expect_lane": "direct"
-}

package/evals/cases/negative-security-docs-change.json

@@ -1,12 +0,0 @@
-{
-  "name": "negative-security-docs-change",
-  "skill": "mugiwara-git",
-  "type": "negative",
-  "task": "Update the README to fix a broken link. No code, no config, no data flow changes.",
-  "rubric": [
-    "does NOT rank mugiwara-security first (docs-only, no trust boundary)",
-    "does NOT rank mugiwara-test-driven-development first (no production code)",
-    "does NOT rank mugiwara-gates first (no code changed)"
-  ],
-  "expect_lane": "direct"
-}

package/evals/cases/routing-typo.json

@@ -1,13 +0,0 @@
-{
-  "name": "routing-typo",
-  "skill": "mugiwara-git",
-  "task": "Fix a typo in a comment in src/index.ts (one word).",
-  "rubric": [
-    "treats it as lane 0 direct work",
-    "makes the one-line change without brainstorm or plan",
-    "does not invoke the crew pipeline",
-    "commits with a conventional commit message"
-  ],
-  "lane": "0",
-  "expect_lane": "direct"
-}