kushi-agents 5.0.2 → 5.0.4

package/README.md

@@ -235,6 +235,41 @@ npm pack --dry-run
 
 The self-check validates frontmatter, agent inventory, prompt → skill routing, profile manifest, reference packs, cross-links, the verbs table in this README, and the layout diagram in `docs/reference/where-things-live.md`. Full reference: [docs/reference/self-check.md](docs/reference/self-check.md).
 
+## Authoring a new skill (v5.0.4+)
+
+Adding a new skill takes one command + a per-skill lint loop:
+
+```powershell
+node bin/cli.mjs create-skill --name my-thing --type writer --description "Generates the foo report from State/"
+node bin/cli.mjs check-skill --name my-thing
+node bin/cli.mjs check-skill --name my-thing --retrofit --apply   # auto-fix additive findings
+npm run eval -- my-thing
+```
+
+Full walkthrough: [`docs/contributing/skill-authoring.md`](docs/contributing/skill-authoring.md). Doctrine: [`plugin/instructions/skill-authoring.instructions.md`](plugin/instructions/skill-authoring.instructions.md). Repo-wide dogfood baseline: [`docs/audits/v5.0.4-skill-creator-dogfood.md`](docs/audits/v5.0.4-skill-creator-dogfood.md).
+
+## Evaluating skills (v5.0.3+)
+
+Every skill ships per-case evals at `plugin/skills/<name>/evals/evals.json`, aligned with the [agentskills.io evaluating-skills spec](https://agentskills.io/skill-creation/evaluating-skills). Doctrine: [`plugin/instructions/skill-evals.instructions.md`](plugin/instructions/skill-evals.instructions.md).
+
+Quickstart:
+
+```powershell
+npm run eval:canary        # ~6 skills, runs in seconds — what PRs run
+npm run eval:all           # full suite (every plugin/skills/<name>/)
+npm run eval -- ask-project   # one skill
+npm run eval:baseline      # maintainer-only: refresh evals/baseline.json
+```
+
+Outputs:
+
+- `Evidence/_evals/<utc-ts>.json` — per-run JSON (pass/fail + duration + tokens per case).
+- `Evidence/_evals/benchmark.json` — per-skill mean/stddev for `pass_rate`, `duration_ms`, `tokens_total` + regression flags vs `evals/baseline.json`.
+
+Regressions flagged at ≥10pp pass-rate drop OR ≥50% latency/token increase. The canary subset is `ask-project`, `bootstrap-project`, `refresh-project`, `link-entities`, `build-state`, `self-check`.
+
+**Privacy:** fixtures under `evals/fixtures/` are synthetic. NEVER copy real customer data into the evals tree.
+
 ## License
 
 See [LICENSE](LICENSE).

package/bin/cli.mjs

@@ -5,6 +5,16 @@ import { runMultiHost } from '../src/multi-host.mjs';
 
 const args = process.argv.slice(2);
 
+// ── skill-authoring verbs (v5.0.4+) ─────────────────────────────────────────
+// Dispatch directly to the skill-creator / skill-checker pwsh scripts.
+const SKILL_VERBS = new Set(['create-skill', 'check-skill', 'optimize-description', 'review-evals']);
+if (args.length > 0 && SKILL_VERBS.has(args[0])) {
+  const verb = args[0];
+  const rest = args.slice(1);
+  await dispatchSkillVerb(verb, rest);
+  process.exit(0);
+}
+
 if (args.includes('--help') || args.includes('-h')) {
   console.log(`
   Usage: npx kushi-agents [options]
@@ -41,6 +51,16 @@ if (args.includes('--help') || args.includes('-h')) {
 
     --help, -h          Show this help
 
+  Skill authoring (v5.0.4+):
+    create-skill <name> --type <pull|writer|orchestrator|other> --description "<d>"
+                         Scaffold a new plugin/skills/<name>/ tree.
+    check-skill <name>   Lint a skill against the agentskills.io blueprint.
+    check-skill --all [--retrofit [--apply]]
+                         Audit (or retrofit) every skill in plugin/skills/.
+    optimize-description <skill>
+                         Rewrite a skill's description per the optimizer rules.
+    review-evals <skill> Render an HTML side-by-side eval-review viewer.
+
   After install, talk to Kushi:
     bootstrap <project>     First-time setup
     refresh <project>       Incremental refresh + rebuild State/
@@ -114,3 +134,86 @@ function getFlag(flag) {
   const match = args.find((a) => a.startsWith(prefix));
   return match ? match.slice(prefix.length) : undefined;
 }
+
+// ── skill-authoring verb dispatch (v5.0.4+) ─────────────────────────────────
+async function dispatchSkillVerb(verb, rest) {
+  const { spawnSync } = await import('node:child_process');
+  const path = await import('node:path');
+  const url = await import('node:url');
+  const here = path.dirname(url.fileURLToPath(import.meta.url));
+  const repoRoot = path.resolve(here, '..');
+  const creatorDir = path.join(repoRoot, 'plugin', 'skills', 'skill-creator');
+  const checkerDir = path.join(repoRoot, 'plugin', 'skills', 'skill-checker');
+
+  let script, scriptArgs = [];
+  switch (verb) {
+    case 'create-skill': {
+      // Usage: kushi create-skill <name> [--type <t>] [--description "<d>"] [--force]
+      const name = rest.find((a) => !a.startsWith('-'));
+      if (!name) {
+        console.error('Usage: kushi-agents create-skill <name> --type <pull|writer|orchestrator|other> --description "USE WHEN ... DO NOT USE FOR ..."');
+        process.exit(1);
+      }
+      const type = pickFlag(rest, '--type') || 'other';
+      const desc = pickFlag(rest, '--description') || `USE WHEN ${name} is invoked. DO NOT USE FOR unrelated tasks.`;
+      script = path.join(creatorDir, 'scaffold.ps1');
+      scriptArgs = ['-Name', name, '-Type', type, '-Description', desc];
+      if (rest.includes('--force')) scriptArgs.push('-Force');
+      if (rest.includes('--dry-run')) scriptArgs.push('-DryRun');
+      break;
+    }
+    case 'check-skill': {
+      // Usage: kushi check-skill <name> | --all  [--retrofit] [--apply]
+      script = path.join(checkerDir, 'check-skill.ps1');
+      const allFlag = rest.includes('--all') || rest.includes('-All');
+      const name = rest.find((a) => !a.startsWith('-'));
+      if (allFlag) scriptArgs.push('-All');
+      else if (name) scriptArgs.push('-Skill', name);
+      else {
+        console.error('Usage: kushi-agents check-skill <name> | --all  [--retrofit] [--apply] [--dry-run]');
+        process.exit(1);
+      }
+      if (rest.includes('--retrofit')) scriptArgs.push('-Retrofit');
+      if (rest.includes('--apply')) scriptArgs.push('-Apply');
+      if (rest.includes('--dry-run')) scriptArgs.push('-DryRun');
+      if (rest.includes('--json')) scriptArgs.push('-Json');
+      break;
+    }
+    case 'optimize-description': {
+      // Usage: kushi optimize-description <skill>
+      const name = rest.find((a) => !a.startsWith('-'));
+      if (!name) {
+        console.error('Usage: kushi-agents optimize-description <skill>');
+        process.exit(1);
+      }
+      script = path.join(checkerDir, 'check-skill.ps1');
+      scriptArgs = ['-Skill', name, '-OptimizeDescription'];
+      break;
+    }
+    case 'review-evals': {
+      // Usage: kushi review-evals <skill>
+      const name = rest.find((a) => !a.startsWith('-'));
+      if (!name) {
+        console.error('Usage: kushi-agents review-evals <skill>');
+        process.exit(1);
+      }
+      script = path.join(checkerDir, 'check-skill.ps1');
+      scriptArgs = ['-Skill', name, '-Review'];
+      break;
+    }
+    default:
+      console.error(`Unknown skill verb: ${verb}`);
+      process.exit(1);
+  }
+
+  const result = spawnSync('pwsh', ['-NoProfile', '-File', script, ...scriptArgs], { stdio: 'inherit' });
+  process.exit(result.status ?? 1);
+}
+
+function pickFlag(args, flag) {
+  const idx = args.indexOf(flag);
+  if (idx !== -1 && idx + 1 < args.length) return args[idx + 1];
+  const prefix = flag + '=';
+  const m = args.find((a) => a.startsWith(prefix));
+  return m ? m.slice(prefix.length) : undefined;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kushi-agents",
-  "version": "5.0.2",
+  "version": "5.0.4",
   "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
   "type": "module",
   "bin": {
@@ -41,9 +41,13 @@
   },
   "license": "MIT",
   "scripts": {
-    "test": "node --test src/check-workiq.test.mjs src/seed-config.test.mjs src/sanitize-workiq-input.test.mjs src/detect-vertex-repo.test.mjs src/vertex-validate.test.mjs src/emit-vertex.e2e.test.mjs src/config-root-resolve.test.mjs src/forbidden-workiq-phrasings.test.mjs src/multi-host-install.test.mjs",
+    "test": "node --test src/check-workiq.test.mjs src/seed-config.test.mjs src/sanitize-workiq-input.test.mjs src/detect-vertex-repo.test.mjs src/vertex-validate.test.mjs src/emit-vertex.e2e.test.mjs src/config-root-resolve.test.mjs src/forbidden-workiq-phrasings.test.mjs src/multi-host-install.test.mjs src/eval-aggregator.test.mjs src/eval-runner.test.mjs src/skill-creator.test.mjs src/skill-checker.test.mjs",
     "test:integration:bootstrap": "node src/bootstrap-dryrun.integration.test.mjs",
     "smoke": "node scripts/smoke.mjs",
+    "eval": "pwsh plugin/skills/eval/run-evals.ps1 -Skill",
+    "eval:all": "pwsh plugin/skills/eval/run-evals.ps1 -All",
+    "eval:canary": "pwsh plugin/skills/eval/run-evals.ps1 -Canary",
+    "eval:baseline": "pwsh plugin/skills/eval/run-evals.ps1 -All -UpdateBaseline",
     "prepublishOnly": "npm test && npm run smoke"
   },
   "publishConfig": {

package/plugin/agents/kushi.agent.md

@@ -16,7 +16,7 @@ Kushi ships in three profiles. The installed profile is recorded in `kushi-insta
 
 | Profile | What's installed | Verbs available |
 |---|---|---|
-| `core` | Aggregator only: `setup`, `pull-*`, `consolidate-evidence`, `aggregate-project`, `ask-project`, `project-status`, `vertex-link`, `emit-vertex`, `self-check`, `intro` | `setup`, `aggregate`, `consolidate`, `status`, `pull`, `ask`, `vertex-link`, `emit-vertex` |
+| `core` | Aggregator only: `setup`, `pull-*`, `consolidate-evidence`, `aggregate-project`, `ask-project`, `project-status`, `vertex-link`, `emit-vertex`, `self-check`, `eval`, `intro` | `setup`, `aggregate`, `consolidate`, `status`, `pull`, `ask`, `vertex-link`, `emit-vertex` |
 | `standard` *(default)* | core + `bootstrap-project`, `refresh-project`, `fde-intake`, `fde-report`, `fde-triage` + FDE reference pack | core + `bootstrap`, `refresh`, `fde-intake`, `fde-report`, `fde-triage` |
 | `full` | standard + `build-state` | standard + `state` |
 | **`preview`** *(opt-in)* | standard + `propose-ado-update`, `apply-ado-update` | standard + `propose-ado`, `apply-ado` |
@@ -182,4 +182,6 @@ Meta skills (not called by verbs):
 | Skill | Role |
 |---|---|
 | `self-check` | Pre-commit consistency check across skills, instructions, prompts, and docs. Run with `pwsh plugin/skills/self-check/run.ps1` (or `./run.sh` on macOS/Linux) or by asking "kushi self-check". |
+| `skill-creator` | (v5.0.4) Scaffolds a new compliant skill — frontmatter + USE WHEN description + type-driven required section + starter evals. Run with `node bin/cli.mjs create-skill --name <kebab> --type <writer\|orchestrator\|pull\|other> --description "..."`. |
+| `skill-checker` | (v5.0.4) Lints + retrofits every SKILL.md against `skill-authoring.instructions.md`. Modes: `-Lint` / `-Retrofit` / `-Apply` / `-OptimizeDescription` / `-Review` / `-All`. Run with `node bin/cli.mjs check-skill --all`. |
 | `intro` | Self-introduction + interactive walkthrough. Triggered by "what is kushi", "what can you do", "kushi intro", "i'm new to kushi", "kushi help". |

package/plugin/instructions/skill-authoring.instructions.md

@@ -0,0 +1,147 @@
+---
+name: "skill-authoring"
+description: "v5.0.4 — How to author a new kushi skill so it ships conformant to the agentskills.io blueprint on day one. Codifies the required SKILL.md sections, file layout, evals starter, naming, and the description-optimization rules. Read this before running `npx kushi-agents create-skill`. Enforced by self-check D34.creator-conformance + by `kushi check-skill --lint`."
+applies_to: "every plugin/skills/<name>/ created from v5.0.4 onward; existing skills are audited via the dogfood gate"
+since: "kushi v5.0.4"
+---
+
+# skill-authoring — doctrine
+
+> Inspired by **Anthropic's [skill-creator](https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md)**. Adapted to kushi's PowerShell-first stack, our 2-host install matrix, and the reality that the first 30 kushi skills were authored before any harness existed — hence the **retrofit** path in `skill-checker`.
+
+## Why this exists
+
+A SKILL.md is the prompt that loads into the agent the moment its trigger fires. Drift between intent and spec is silent until evals catch it (and only if there are evals). This doctrine + the `skill-creator` + `skill-checker` skills make conformant authoring the default, not an afterthought.
+
+## Required files per skill
+
+```text
+plugin/skills/<name>/
+├── SKILL.md                 ← REQUIRED — agent-loaded prompt; ≤500 lines, ≤5000 tokens
+├── evals/
+│   └── evals.json           ← REQUIRED — ≥2 cases, each with ≥1 assertion
+├── references/              ← OPTIONAL — load-on-trigger bulk content (>500 lines splits here)
+└── .created-by-skill-creator  ← OPTIONAL marker — set by `scaffold.ps1`; opts into the strict D34 gate
+```
+
+A skill MAY also ship a runner (`run.ps1`, `run.sh`, `*.mjs`) when it's a tool-skill (`self-check`, `eval`, `skill-checker`, etc.). Pure prompt-skills don't need one.
+
+## Required SKILL.md sections
+
+Every SKILL.md MUST have:
+
+1. **YAML frontmatter** — `name` (kebab-case, matches dir) + `description` (lead with `USE WHEN`).
+2. **`# Skill: <name>`** H1.
+3. **One-paragraph purpose** immediately after the H1.
+4. **At least one** of these procedure-shape sections, picked by skill **type**:
+   - `## Gotchas` — REQUIRED for `pull-*` and discovery skills (top-5 failure modes).
+   - `## Step checklist` — REQUIRED for orchestrators (`bootstrap-project`, `refresh-project`, `build-state`, `link-entities`, `dashboard`, `tour`, etc.); use GitHub `- [ ]` checkboxes.
+   - `## Validation loop` — REQUIRED for writer skills (anything that writes to `Evidence/`, `State/`, `_graph/`, `dashboards/`, `tours/`).
+   - `## Steps` or `## Procedure` — acceptable for other skills, plus one of the three above where it applies.
+
+Skills can have more than one (e.g. `eval` ships all three). The checker is satisfied by **at least one** of `Gotchas` / `Step checklist` / `Validation loop` plus type-specific rules.
+
+## Description optimization (per agentskills.io)
+
+The `description:` is the sole trigger signal. Optimize it:
+
+```yaml
+description: "USE WHEN <situational trigger> AND <precondition>. DO NOT USE for <near-miss>. <one-line capability summary>."
+```
+
+Rules (enforced by `D30.description-optimized` + `skill-checker --optimize-description`):
+
+- Lead with `USE WHEN` (first 160 chars).
+- Include a `DO NOT USE` clause for the most likely near-miss invocation.
+- Be specific about the trigger (concrete user phrases or file/state conditions).
+- No marketing fluff ("powerful", "comprehensive", "blazing").
+- ≤1024 characters total.
+
+| Bad | Good |
+|---|---|
+| `"Pulls OneNote pages."` | `"USE WHEN refreshing project evidence for a known kushi project AND boundaries.onenote.section_ids is non-empty. DO NOT USE for global OneNote search."` |
+| `"Comprehensive eval framework."` | `"USE WHEN the user says 'run evals', 'eval canary', or before tagging a release. DO NOT USE for evidence validation of a real project."` |
+
+## Size caps
+
+- ≤ 500 lines (`D30.skill-size`)
+- ≤ 5000 tokens (~20 KB)
+
+When you exceed either, **split into `references/<topic>.md` files** and cite them with explicit triggers:
+
+```markdown
+Load `references/canonical-prompts.md` when constructing the WorkIQ query.
+Load `references/error-modes.md` if the API returns non-200.
+```
+
+Passive links (`[see foo](references/foo.md)`) do NOT count. The checker requires the literal substring `references/<file>.md` somewhere in SKILL.md if `references/` exists.
+
+## Evals (≥2 cases)
+
+Every skill MUST ship `evals/evals.json` validated against `plugin/skills/eval/evals.schema.json`. Per `skill-evals.instructions.md`:
+
+- `id`, `name`, `input`, `expected_assertions[]` (≥1), `grader_type` (`script` | `llm`).
+- ≥2 cases. Mark canary-worthy ones `"canary": true`.
+- Synthetic fixtures only — never real customer data.
+
+The `create-skill` scaffold emits a starter `evals.json` with one `file-exists` and one `regex-match` case so the skill ships green from minute one.
+
+## Naming conventions
+
+- **Skill directory + frontmatter `name`** — kebab-case, verb-led (`pull-onenote`, `consolidate-evidence`, `apply-ado-update`).
+- **Skill types** (informational; pick at create time so the scaffold picks the right sections):
+  - `pull` — fetches evidence from a source.
+  - `writer` — writes files to `Evidence/` or `State/`.
+  - `orchestrator` — coordinates other skills.
+  - `other` — utility / tool / meta.
+- **Instruction files** — `<topic>.instructions.md` in `plugin/instructions/`. Front-matter `name:` matches filename minus `.instructions.md`.
+
+## Contributor workflow
+
+```powershell
+# 1. Scaffold
+npx kushi-agents create-skill my-new-skill
+#   → answers: type (pull|writer|orchestrator|other), one-liner description
+#   → emits plugin/skills/my-new-skill/{SKILL.md, evals/evals.json}
+#   → marker file .created-by-skill-creator is written
+
+# 2. Fill in the placeholders (search for "TODO(skill-creator)")
+
+# 3. Validate
+npx kushi-agents check-skill my-new-skill        # lint mode
+npm run eval -- my-new-skill                     # run evals
+
+# 4. Optimize description before PR
+npx kushi-agents optimize-description my-new-skill
+#   → emits a rewritten description; you decide whether to apply
+
+# 5. Self-check + commit
+pwsh plugin/skills/self-check/run.ps1 -Deep
+git add plugin/skills/my-new-skill/
+git commit -m "v<x.y.z>: my-new-skill"
+```
+
+## Retrofit (existing skills predating the harness)
+
+Run `npx kushi-agents check-skill --all --retrofit` to identify gaps in legacy skills. `--apply` adds missing section stubs with `<!-- TODO(retrofit): fill in -->` markers — never overwrites existing content. The v5.0.4 dogfood report at `docs/audits/v5.0.4-skill-creator-dogfood.md` records the baseline.
+
+## Enforcement
+
+| Check | What it does |
+|---|---|
+| `D34.skill-creator-exists` | `plugin/skills/skill-creator/scaffold.ps1` is parseable. |
+| `D34.skill-checker-exists` | `plugin/skills/skill-checker/check-skill.ps1` is parseable. |
+| `D34.creator-output-conforms` | Every skill carrying `.created-by-skill-creator` passes `check-skill --lint`. |
+| `D34.retrofit-clean` | `check-skill --all --retrofit --dry-run` shows no unresolved non-additive gaps. |
+| `D34.dogfood-report-fresh` | `docs/audits/v5.0.4-skill-creator-dogfood.md` was touched within 14 days (warn-only). |
+
+## References
+
+- `plugin/instructions/agentskills-compliance.instructions.md` — the spec rules this builds on.
+- `plugin/instructions/skill-evals.instructions.md` — the evals doctrine.
+- `plugin/skills/skill-creator/SKILL.md` — the scaffolder.
+- `plugin/skills/skill-checker/SKILL.md` — the linter / retrofitter.
+- `docs/contributing/skill-authoring.md` — the human walkthrough.
+- <https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md> — upstream inspiration.
+- <https://agentskills.io/skill-creation/best-practices>
+- <https://agentskills.io/skill-creation/optimizing-descriptions>

package/plugin/instructions/skill-evals.instructions.md

@@ -0,0 +1,130 @@
+---
+description: "v5.0.3 — Skill evals doctrine, adapted from https://agentskills.io/skill-creation/evaluating-skills. Every skill MUST ship an evals/ folder with at least 2 deterministic cases plus structured assertions; a per-skill pass-rate is the objective regression signal. Canary subset runs on every PR; full suite runs on demand. Real customer data is FORBIDDEN in fixtures — use synthetic data only."
+---
+
+# Skill evals — doctrine
+
+> Inspired by **<https://agentskills.io/skill-creation/evaluating-skills>**. Adapted to kushi's PowerShell + Node test stack and to our 2-host install matrix.
+
+## Why
+
+Skills are prompts plus a runner. Prompts drift silently. Without an objective per-skill regression signal, every change is a gamble. Evals make that signal cheap:
+
+- **Per-skill pass-rate** is the headline metric.
+- **Latency** and **tokens** are secondary metrics (regressions ≥50% latency / ≥10pp pass-rate flag a baseline failure).
+- A **canary subset** runs on every PR (target: < 60s wall clock); the **full suite** runs on demand (`npm run eval:all`).
+
+## Where evals live
+
+```text
+plugin/skills/<name>/
+├── SKILL.md
+└── evals/
+    ├── evals.json        ← REQUIRED — case list + assertions
+    └── fixtures/         ← OPTIONAL per-skill fixtures
+```
+
+Cross-skill fixtures live at the repo root:
+
+```text
+evals/
+├── baseline.json         ← Committed; maintainer updates with `npm run eval:baseline`
+└── fixtures/             ← Tiny synthetic evidence trees, ADO fixtures, etc.
+```
+
+Per-run output goes to `Evidence/_evals/<timestamp>.json` (gitignored; not customer data).
+
+## Case schema
+
+```jsonc
+{
+  "skill": "<skill-name>",
+  "cases": [
+    {
+      "id": "ap-citations-format",
+      "name": "ask-project emits weekly-csc citation form",
+      "input": "what was decided about MACC for fixture-acme?",
+      "fixture": "evals/fixtures/fixture-acme",        // optional
+      "canary": true,
+      "grader_type": "script",                          // "script" | "llm"
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "\\[source:\\s*fixture-acme/email/weekly/" },
+        { "type": "regex-match", "pattern": "Source-layout:\\s*weekly-csc" }
+      ]
+    }
+  ]
+}
+```
+
+### Required fields per case
+
+- `id` — unique within the skill; kebab-case.
+- `name` — human-readable.
+- `input` — what gets passed to the skill (string OR object).
+- `expected_assertions` — array, **≥ 1** entry (enforced by `D33.evals-have-assertions`).
+- `grader_type` — `"script"` for deterministic graders, `"llm"` for rubric-based.
+
+### Optional fields
+
+- `fixture` — repo-relative path to the fixture to point the skill at.
+- `canary` — `true` to include in the fast CI subset.
+- `args` — extra args forwarded to the skill script (e.g. `{ "DryRun": true }`).
+- `skip` — `true` to skip (must include `skip_reason`).
+- `timeout_ms` — override the runner default (30 000 ms).
+
+## Assertion types
+
+| Type | Shape | Passes when |
+|---|---|---|
+| `file-exists` | `{ "type": "file-exists", "path": "..." }` | Path exists post-run (relative to fixture or evidence dir). |
+| `file-contains` | `{ "type": "file-contains", "path": "...", "needle": "..." }` | File exists and substring is present. |
+| `json-path-equals` | `{ "type": "json-path-equals", "path": "...", "json_path": "$.foo.bar", "equals": "v" }` | JSON file parses; dotted path value === expected. |
+| `regex-match` | `{ "type": "regex-match", "pattern": "...", "flags": "i" }` | Captured stdout matches the regex. |
+| `llm-rubric` | `{ "type": "llm-rubric", "rubric": "...", "min_score": 4 }` | LLM grader scores ≥ min on a 1–5 rubric. |
+
+## Run modes
+
+The runner (`plugin/skills/eval/run-evals.ps1`) supports three dispatch modes:
+
+1. **Direct invocation** (default for `script` graders). Runs the skill's executable artifact (`run.ps1`, `*.mjs`, or a small probe stub) with the given input and fixture. Pure deterministic.
+2. **Sub-agent dispatch** (optional, gated by `-Live`). Forwards the case to a sub-agent. Used only for `llm-rubric` cases. Skipped in canary mode.
+3. **Recorded fixture replay** (for `pull-*` skills). Reads a recorded `--cached` output of a real pull and asserts against that, so no live M365 calls are needed.
+
+For each case the runner records: `pass`, `duration_ms`, `tokens_in`, `tokens_out`, `stdout`, `stderr`, per-assertion `pass`/`reason`. The aggregate is a JSON file under `Evidence/_evals/` plus a one-line `benchmark.json` summary.
+
+## Canary set
+
+Marked with `"canary": true`. Kept tiny so PRs stay fast.
+
+Default canary set (v5.0.3):
+
+- `ask-project`
+- `bootstrap-project`
+- `refresh-project`
+- `link-entities`
+- `build-state`
+- `self-check`
+
+## Baseline + regression detection
+
+- `evals/baseline.json` is **committed**.
+- Each per-skill record carries the last green `pass_rate`, `mean_duration_ms`, and `mean_tokens_total`.
+- `src/eval-aggregator.mjs` flags **regressions**:
+  - `pass_rate` drop ≥ 10 percentage points
+  - `mean_duration_ms` increase ≥ 50 %
+  - `mean_tokens_total` increase ≥ 50 %
+- Maintainers refresh the baseline with `npm run eval:baseline` after deliberate behavior changes.
+
+## Privacy + safety
+
+- **No real customer data** in any fixture. Use `fixture-acme`-style synthetic names.
+- `Evidence/_evals/` is in `.gitignore`.
+- `pull-*` evals NEVER hit live M365 endpoints in canary mode. Use recorded `--cached` payloads or `--dry-run`.
+- Tenant IDs / GUIDs in fixtures must be obviously fake (e.g. `00000000-...`).
+
+## References
+
+- [agentskills.io — evaluating skills](https://agentskills.io/skill-creation/evaluating-skills) (source of truth)
+- `plugin/skills/eval/SKILL.md` (the runner skill)
+- `plugin/skills/eval/evals.schema.json` (JSON schema; self-check D33.evals-schema)
+- `plugin/instructions/agentskills-compliance.instructions.md` (sibling doctrine — size + section caps)

package/plugin/skills/aggregate-project/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "aggregate-project",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for aggregate-project. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "aggregate-project-smoke-1",
+      "name": "aggregate-project produces a non-empty response",
+      "input": "synthetic aggregate-project probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "aggregate-project-smoke-2",
+      "name": "aggregate-project echoes case id",
+      "input": "case-id aggregate-project-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "aggregate-project-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/apply-ado-update/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "apply-ado-update",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for apply-ado-update. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "apply-ado-update-smoke-1",
+      "name": "apply-ado-update produces a non-empty response",
+      "input": "synthetic apply-ado-update probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "apply-ado-update-smoke-2",
+      "name": "apply-ado-update echoes case id",
+      "input": "case-id apply-ado-update-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "apply-ado-update-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/ask-project/SKILL.md

@@ -197,3 +197,13 @@ Explicit triggers also accepted:
 
 - **v4.0.0 (kushi v5.0.0, 2026-05-26)**: graph-first cross-source resolution — consult `Evidence/_graph/project-graph.json` before walking weekly files when a question spans sources. Falls back to v4.9.0 walking strategy if graph absent/stale.
 - **v3.0.0 (kushi v4.9.0, 2026-05-26)**: 3-step reader fallback chain (`_index/entities.yml` → `weekly/*.md` → legacy `snapshot/` + `stream/`). New citation form `weekly/<YYYY-MM-DD>_<source>-csc.md#<anchor>`. Legacy citations suffixed `(legacy pre-v4.9.0 layout)`. Output marked with `Source-layout:` footer.
+
+
+## Validation loop
+
+<!-- TODO(retrofit): fill in — describe how to verify this skill ran correctly. Auto-added by skill-checker --retrofit --apply per skill-authoring.instructions.md. -->
+
+1. Run pwsh plugin/skills/self-check/run.ps1 -Targeted <area>.
+2. Fix any findings, then re-run the affected step.
+3. Repeat until self-check exits 0.
+4. Only then update 

package/plugin/skills/ask-project/evals/evals.json

@@ -0,0 +1,34 @@
+{
+  "skill": "ask-project",
+  "version": "1.0.0",
+  "description": "Verifies citation format (weekly-csc) and Source-layout footer.",
+  "cases": [
+    {
+      "id": "ap-macc-citation",
+      "name": "answers MACC question with weekly-csc citation",
+      "input": "what was decided about MACC for fixture-acme?",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "args": { "read_fixture": "../outputs/ask-project.macc.txt" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "MACC" },
+        { "type": "regex-match", "pattern": "\\[source:\\s*fixture-acme/email/weekly/" },
+        { "type": "regex-match", "pattern": "Source-layout:\\s*weekly-csc" }
+      ]
+    },
+    {
+      "id": "ap-em-question",
+      "name": "answers who-is-the-EM with cited person",
+      "input": "who is the lead on fixture-acme?",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "args": { "read_fixture": "../outputs/ask-project.lead.txt" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "alice@fixture\\.local" },
+        { "type": "regex-match", "pattern": "Source-layout:\\s*weekly-csc" }
+      ]
+    }
+  ]
+}

package/plugin/skills/bootstrap-project/evals/evals.json

@@ -0,0 +1,34 @@
+{
+  "skill": "bootstrap-project",
+  "version": "1.0.0",
+  "description": "Asserts the canonical engagement folder layout from a dry-run.",
+  "cases": [
+    {
+      "id": "bp-dryrun-layout",
+      "name": "dry-run prints the canonical layout",
+      "input": "bootstrap fixture-acme --dry-run",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "args": { "read_fixture": "../outputs/bootstrap-project.dryrun.txt" },
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "DRY-RUN" },
+        { "type": "regex-match", "pattern": "fixture-acme/Evidence/fixture-alias/email/weekly/" },
+        { "type": "regex-match", "pattern": "fixture-acme/State/" },
+        { "type": "regex-match", "pattern": "fixture-acme/Evidence/_graph/" }
+      ]
+    },
+    {
+      "id": "bp-fixture-tree-present",
+      "name": "fixture engagement tree exists on disk",
+      "input": "verify fixture tree",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "Evidence/fixture-alias/email/weekly/2026-05-18_email-csc.md" },
+        { "type": "file-exists", "path": "State/index.md" }
+      ]
+    }
+  ]
+}