kushi-agents 5.0.2 → 5.0.3

package/plugin/skills/pull-email/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-email",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-email. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-email-cached-1",
+      "name": "pull-email cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-email-rubric-1",
+      "name": "pull-email output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-email pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-email response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-loop/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-loop",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-loop. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-loop-cached-1",
+      "name": "pull-loop cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-loop-rubric-1",
+      "name": "pull-loop output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-loop pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-loop response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-meetings/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-meetings",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-meetings. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-meetings-cached-1",
+      "name": "pull-meetings cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-meetings-rubric-1",
+      "name": "pull-meetings output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-meetings pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-meetings response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-misc/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-misc",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-misc. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-misc-cached-1",
+      "name": "pull-misc cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-misc-rubric-1",
+      "name": "pull-misc output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-misc pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-misc response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-onenote/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-onenote",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-onenote. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-onenote-cached-1",
+      "name": "pull-onenote cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-onenote-rubric-1",
+      "name": "pull-onenote output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-onenote pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-onenote response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-sharepoint/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-sharepoint",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-sharepoint. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-sharepoint-cached-1",
+      "name": "pull-sharepoint cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-sharepoint-rubric-1",
+      "name": "pull-sharepoint output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-sharepoint pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-sharepoint response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/pull-teams/evals/evals.json

@@ -0,0 +1,35 @@
+{
+  "skill": "pull-teams",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for pull-teams. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "pull-teams-cached-1",
+      "name": "pull-teams cached/dry-run produces output",
+      "input": "--cached --dry-run fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "fixture-acme",
+          "flags": "i"
+        }
+      ]
+    },
+    {
+      "id": "pull-teams-rubric-1",
+      "name": "pull-teams output quality (LLM-rubric, skipped in canary)",
+      "input": "summarize fixture-acme pull-teams pulls",
+      "canary": false,
+      "grader_type": "llm",
+      "expected_assertions": [
+        {
+          "type": "llm-rubric",
+          "rubric": "Does the pull-teams response cite a source file path and an ISO timestamp?",
+          "min_score": 4
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/refresh-project/evals/evals.json

@@ -0,0 +1,31 @@
+{
+  "skill": "refresh-project",
+  "version": "1.0.0",
+  "description": "Verifies refresh chains link-entities -> dashboard -> tour on a fixture.",
+  "cases": [
+    {
+      "id": "rp-chain-mention",
+      "name": "refresh plan mentions link-entities + dashboard + tour",
+      "input": "refresh fixture-acme --dry-run\nWould chain: link-entities -> dashboard -> tour\nGraph: Evidence/_graph/project-graph.json\nDashboard: dashboard.html\nTour: State/tour.md",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": true,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "regex-match", "pattern": "link-entities" },
+        { "type": "regex-match", "pattern": "dashboard" },
+        { "type": "regex-match", "pattern": "tour" }
+      ]
+    },
+    {
+      "id": "rp-graph-fixture-present",
+      "name": "refresh has a graph fixture to update",
+      "input": "verify graph fixture",
+      "fixture": "evals/fixtures/fixture-acme",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "Evidence/_graph/project-graph.json" }
+      ]
+    }
+  ]
+}

package/plugin/skills/self-check/SKILL.md

@@ -69,6 +69,7 @@ Checks split into **core** (always run) and **deep** (opt-in).
 | D30.description-optimized | Trigger-based description | every SKILL.md `description:` front-matter leads with `USE WHEN` or `WHEN ` per <https://agentskills.io/skill-creation/optimizing-descriptions>. |
 | D31.genealogy | Release genealogy entry exists | every `git tag` matching `v<x.y.z>` MUST appear in `docs/genealogy.md` as a `## v<x.y.z>` heading or be named under a parent's "Patch lineage" line. See `release-genealogy.instructions.md`. |
 | D32.multi-host | Multi-host install integrity | validates `src/multi-host.mjs` exports + `bin/cli.mjs` flag handling, then performs a temp-dir dry-run install for BOTH supported hosts (Clawpilot + VS Code Chat) under a fake `$HOME` in `$env:TEMP`. Asserts SKILL.md + agent file + skills/ + prompts/ + skills-metadata.json with a kushi entry are present, then asserts a clean uninstall. NEVER touches the real `~/.copilot/` or `~/.vscode/`. See `multi-host-install.instructions.md`. |
+| D33.evals | Skill evals framework integrity | every `plugin/skills/<name>/` (except `eval`) ships `evals/evals.json` with ≥ 2 cases and ≥ 1 assertion per case; the runner (`plugin/skills/eval/run-evals.ps1`) and schema (`plugin/skills/eval/evals.schema.json`) are present; `evals/baseline.json` exists (warn-only). Six sub-checks: `D33.evals-exist`, `D33.evals-schema`, `D33.evals-min-cases`, `D33.evals-have-assertions`, `D33.eval-runner-exists`, `D33.baseline-exists`. See `skill-evals.instructions.md`. |
 | **CSC weekly-layout checks (kushi v4.9.0)** | | gated on `Resolve-EngagementRoots` — no-ops on the kushi repo itself. |
 | D11.csc | CSC entity coverage + depth | every `Evidence/<alias>/<source>/weekly/*-csc.md` has ≥ 1 entity heading; per-source minimum bullet count + populated-section count (meetings 25/6, email 8/4, teams 6/3, onenote 10/4, sharepoint 8/3, crm 12/5, ado 8/4). Coverage-Notes-only blocks (low-signal escape) are exempt. |
 | D12.csc | CSC section order | every entity block's `###` section headings appear in the canonical order: Participants → Topics → Q&A → Who Said What → Decisions → Dates & Numbers → Action Items → Next Steps → Open Questions → Risks → Customer Asks → Artifacts → Coverage Notes. |

package/plugin/skills/self-check/evals/evals.json

@@ -0,0 +1,28 @@
+{
+  "skill": "self-check",
+  "version": "1.0.0",
+  "description": "Meta — self-check's own run.ps1 is parseable and SKILL.md is present.",
+  "cases": [
+    {
+      "id": "sc-runps1-exists",
+      "name": "run.ps1 ships in the skill folder",
+      "input": "verify self-check artifacts",
+      "canary": true,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "plugin/skills/self-check/run.ps1" },
+        { "type": "file-exists", "path": "plugin/skills/self-check/SKILL.md" }
+      ]
+    },
+    {
+      "id": "sc-runsh-exists",
+      "name": "cross-platform run.sh ships too",
+      "input": "verify run.sh",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        { "type": "file-exists", "path": "plugin/skills/self-check/run.sh" }
+      ]
+    }
+  ]
+}

package/plugin/skills/self-check/run.ps1

@@ -1632,6 +1632,69 @@ process.stdout.write(JSON.stringify(out));
             if (Test-Path $fakeHome) { Remove-Item -LiteralPath $fakeHome -Recurse -Force -ErrorAction SilentlyContinue }
         }
     }
+
+    # === D33.evals — per-skill evals framework (v5.0.3+) ===
+    # Per skill-evals.instructions.md, every plugin/skills/<name>/ (except eval and
+    # self-check) MUST ship evals/evals.json with >=2 cases and every case MUST have
+    # >=1 assertion. Schema lives at plugin/skills/eval/evals.schema.json. Runner
+    # lives at plugin/skills/eval/run-evals.ps1. Baseline file evals/baseline.json
+    # is warn-only — maintainers seed it with `npm run eval:baseline`.
+    $evalSkillDir = Join-Path $Root 'plugin/skills/eval'
+    $evalRunner   = Join-Path $evalSkillDir 'run-evals.ps1'
+    $evalSchema   = Join-Path $evalSkillDir 'evals.schema.json'
+    $baselineFile = Join-Path $Root 'evals/baseline.json'
+
+    if (-not (Test-Path $evalRunner)) {
+        Add-Finding 'D33.eval-runner-exists' 'Evals' 'warning' 'plugin/skills/eval/run-evals.ps1 is missing' 'Restore the runner from git — it ships in v5.0.3+.' $evalRunner 0
+    } else {
+        # Quick parse-ability check: scan for at least one param( block.
+        try {
+            $rt = Get-Content -Raw $evalRunner
+            if ($rt -notmatch '(?ms)^\s*param\s*\(') {
+                Add-Finding 'D33.eval-runner-exists' 'Evals' 'warning' 'plugin/skills/eval/run-evals.ps1 has no param() block' 'Confirm the runner is valid pwsh; rerun seed-evals if it was clobbered.' $evalRunner 0
+            }
+        } catch {
+            Add-Finding 'D33.eval-runner-exists' 'Evals' 'warning' "Could not read run-evals.ps1: $($_.Exception.Message)" 'Check file permissions.' $evalRunner 0
+        }
+    }
+
+    if (-not (Test-Path $evalSchema)) {
+        Add-Finding 'D33.evals-schema' 'Evals' 'warning' 'plugin/skills/eval/evals.schema.json is missing' 'Restore the schema from git.' $evalSchema 0
+    }
+
+    $skillsRoot = Join-Path $Root 'plugin/skills'
+    $skillDirs = Get-ChildItem -Path $skillsRoot -Directory | Where-Object { $_.Name -notin @('eval', 'self-check') }
+    foreach ($sd in $skillDirs) {
+        $evalsFile = Join-Path $sd.FullName 'evals/evals.json'
+        if (-not (Test-Path $evalsFile)) {
+            Add-Finding 'D33.evals-exist' 'Evals' 'warning' "Skill '$($sd.Name)' has no evals/evals.json" "Create $evalsFile with >=2 cases per skill-evals.instructions.md. Quick start: run 'node scripts/seed-evals.mjs'." $evalsFile 0
+            continue
+        }
+        try {
+            $obj = Get-Content -Raw $evalsFile | ConvertFrom-Json
+        } catch {
+            Add-Finding 'D33.evals-schema' 'Evals' 'warning' "Skill '$($sd.Name)' evals.json is not valid JSON: $($_.Exception.Message)" 'Fix the JSON and rerun self-check.' $evalsFile 0
+            continue
+        }
+        if (-not $obj.skill -or $obj.skill -ne $sd.Name) {
+            Add-Finding 'D33.evals-schema' 'Evals' 'warning' "Skill '$($sd.Name)' evals.json declares skill='$($obj.skill)' (mismatch)" 'Set the skill field to match the directory name.' $evalsFile 0
+        }
+        if (-not $obj.cases -or $obj.cases.Count -lt 2) {
+            Add-Finding 'D33.evals-min-cases' 'Evals' 'warning' "Skill '$($sd.Name)' has fewer than 2 eval cases" 'Add at least 2 deterministic cases per skill-evals.instructions.md.' $evalsFile 0
+        }
+        foreach ($c in $obj.cases) {
+            if (-not $c.expected_assertions -or $c.expected_assertions.Count -lt 1) {
+                Add-Finding 'D33.evals-have-assertions' 'Evals' 'warning' "Skill '$($sd.Name)' case '$($c.id)' has no expected_assertions" 'Every case needs >=1 assertion (file-exists / file-contains / json-path-equals / regex-match / llm-rubric).' $evalsFile 0
+            }
+            if ($c.grader_type -and ($c.grader_type -notin 'script', 'llm')) {
+                Add-Finding 'D33.evals-schema' 'Evals' 'warning' "Skill '$($sd.Name)' case '$($c.id)' has invalid grader_type '$($c.grader_type)'" "Use 'script' or 'llm'." $evalsFile 0
+            }
+        }
+    }
+
+    if (-not (Test-Path $baselineFile)) {
+        Add-Finding 'D33.baseline-exists' 'Evals' 'warning' 'evals/baseline.json is missing' "Seed the baseline with 'npm run eval:baseline' (warn-only — does not block)." $baselineFile 0
+    }
 }
 
 # === Output ===

package/plugin/skills/setup/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "setup",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for setup. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "setup-smoke-1",
+      "name": "setup produces a non-empty response",
+      "input": "synthetic setup probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "setup-smoke-2",
+      "name": "setup echoes case id",
+      "input": "case-id setup-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "setup-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/tour/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "tour",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for tour. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "tour-smoke-1",
+      "name": "tour produces a non-empty response",
+      "input": "synthetic tour probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "tour-smoke-2",
+      "name": "tour echoes case id",
+      "input": "case-id tour-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "tour-smoke-2"
+        }
+      ]
+    }
+  ]
+}

package/plugin/skills/vertex-link/evals/evals.json

@@ -0,0 +1,33 @@
+{
+  "skill": "vertex-link",
+  "version": "1.0.0",
+  "description": "Auto-seeded evals for vertex-link. Replace with real cases as the skill matures.",
+  "cases": [
+    {
+      "id": "vertex-link-smoke-1",
+      "name": "vertex-link produces a non-empty response",
+      "input": "synthetic vertex-link probe — canary smoke",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": ".+"
+        }
+      ]
+    },
+    {
+      "id": "vertex-link-smoke-2",
+      "name": "vertex-link echoes case id",
+      "input": "case-id vertex-link-smoke-2",
+      "canary": false,
+      "grader_type": "script",
+      "expected_assertions": [
+        {
+          "type": "regex-match",
+          "pattern": "vertex-link-smoke-2"
+        }
+      ]
+    }
+  ]
+}