@mmerterden/multi-agent-pipeline 20.6.0 → 20.8.0

package/pipeline/skills/shared/external/skill-creator/scripts/audit-panel.js

@@ -1,119 +1,110 @@
-// audit-panel.js - Workflow template for the multi-architect skill audit.
-//
-// Run with the Workflow tool. Edit the CONFIG block for your target, then launch.
-// Returns { remaining, clearsBar, verdict }. If clearsBar is false, apply the
-// `remaining` blockers/majors, then RE-RUN A FRESH copy (do NOT resume - agents
-// must read the current files). Loop until clearsBar. See ../audit.md.
-
 export const meta = {
-  name: "skill-audit-panel",
-  description: "Adversarial N-architect audit of a skill against the skill-creator rubric",
-  phases: [{ title: "Audit" }, { title: "Synthesis" }],
-};
-
-// ── CONFIG - edit these ──────────────────────────────────────────────────────
-const TARGET = `
-- <abs path to SKILL.md>
-- <abs path to each bundled file / script>
-`;
-const RUBRIC = `<abs path to reference/skill-creator/SKILL.md> + checklist.md`;
-const EXEMPLAR = `<abs path to a known-good skill of the same layer>`;
-const GROUND_TRUTH = `<abs path to the real source the skill describes (e.g. the iOS repo), with key facts>`;
-// Pick a lens set from audit.md (general or prompt-engineering). [label, brief] pairs:
-const LENSES = [
-  [
-    "discovery",
-    "description-as-discovery: what+when+trigger words+example; would it activate on a real task?",
-  ],
-  [
-    "disambiguation",
-    "distinct from sibling skills; a reader can tell when to pick this vs a neighbor.",
-  ],
-  ["triggers", "real user phrasings + synonyms are covered."],
-  ["clarity", "instructions unambiguous, ordered, single-interpretation; no vague directives."],
-  [
-    "layer",
-    "degrees of freedom match the layer (reference=descriptive / workflow=spine / tool=thin-index).",
+  name: 'skill-audit-panel',
+  description: 'Runs an independent, multi-lens review of a single plugin skill, scored on the skill-creator rubric',
+  whenToUse: 'Gate before shipping a skill that is new or heavily reworked; start a new run once each round of fixes lands',
+  phases: [
+    { title: 'Audit', detail: 'one read-only reviewer per lens' },
+    { title: 'Synthesis', detail: 'dedup, verify, classify, decide clearsBar' },
   ],
-  ["leanness", "SKILL.md is an index; detail bundled one level deep; no duplication or bloat."],
-  ["tokens", "every line load-bearing; nothing restates what the model already knows."],
-  ["consistency", "terminology, naming, paths, cross-refs, examples all agree."],
-  [
-    "misfire",
-    "would the model over-apply, skip a gate, hallucinate a path, or follow it to a wrong output?",
-  ],
-  [
-    "redteam",
-    "adversarially make it not trigger / trigger wrongly / produce broken output; hunt overclaims (says X, script does not).",
-  ],
-];
-// ─────────────────────────────────────────────────────────────────────────────
+}
 
-const COMMON = `
-You are auditing a skill. Read the CURRENT files and report ONLY genuine, specific,
-reproducible findings for YOUR lens. Verify against the rubric + ground truth - do
-not assume. Read-only; do NOT edit. If your lens is clean, return an empty array.
+const TARGET = [
+  '/abs/path/to/plugins/<plugin>/skills/<layer>/<skill>/SKILL.md',
+  '/abs/path/to/plugins/<plugin>/skills/<layer>/<skill>/reference.md',
+]
+const RUBRIC = '/abs/path/to/plugins/ai-common-toolkit/skills/knowledge/skill-creator/SKILL.md and checklist.md beside it'
+const EXEMPLAR = '/abs/path/to/an/exemplar/skill/in/this/layer/SKILL.md'
+const GROUND_TRUTH = '/abs/path/to/source/code/described/by/the/skill (name the key facts worth checking)'
 
-TARGET (the skill under audit):${TARGET}
-RUBRIC (the bar): ${RUBRIC}
-EXEMPLAR (known-good of this layer): ${EXEMPLAR}
-GROUND TRUTH (what the skill describes): ${GROUND_TRUTH}
-
-Each finding: severity (blocker|major|minor|nit), file:location, the problem, the concrete fix.
-`;
+const LENSES = [
+  { label: 'discovery', brief: 'Does the description alone get the skill loaded? What + when + trigger words + a sample trigger, third person, platform named.' },
+  { label: 'disambiguation', brief: 'Could a router confuse this skill with a sibling? Are the boundaries and hand-offs explicit in the description and header?' },
+  { label: 'triggers', brief: 'Which real phrasings and synonyms a user would type are missing from the description?' },
+  { label: 'clarity', brief: 'Is every instruction unambiguous and actionable, with no step that two readers would execute differently?' },
+  { label: 'layer', brief: 'Does the content match its declared layer (reference descriptive, workflow imperative with a verification gate, tool a thin index over a script)?' },
+  { label: 'leanness', brief: 'Is SKILL.md an index of about one page, with detail one level deep in bundled files and deterministic code in scripts?' },
+  { label: 'tokens', brief: 'Which lines fail the cut test because the model would act correctly without them?' },
+  { label: 'consistency', brief: 'Do terms, names, paths, cross-references and examples agree with each other and with the files that exist?' },
+  { label: 'misfire', brief: 'Where would the model over-apply the skill, skip its gate, invent a path, or produce the wrong output?' },
+  { label: 'redteam', brief: 'Try to make it fail to trigger, trigger wrongly, or break its output; hunt for prose that claims what the scripts do not do.' },
+]
 
 const FINDING = {
-  type: "object",
+  type: 'object',
   additionalProperties: false,
+  required: ['severity', 'location', 'problem', 'fix'],
   properties: {
-    severity: { type: "string", enum: ["blocker", "major", "minor", "nit"] },
-    location: { type: "string" },
-    problem: { type: "string" },
-    fix: { type: "string" },
+    severity: { enum: ['blocker', 'major', 'minor', 'nit'], type: 'string' },
+    location: { type: 'string', description: 'file:line or file#heading' },
+    problem: { type: 'string' },
+    fix: { type: 'string', description: 'a concrete change, not advice' },
   },
-  required: ["severity", "location", "problem", "fix"],
-};
-const LENS_SCHEMA = {
-  type: "object",
-  additionalProperties: false,
-  properties: { lens: { type: "string" }, findings: { type: "array", items: FINDING } },
-  required: ["lens", "findings"],
-};
+}
 
-phase("Audit");
-const results = await parallel(
-  LENSES.map(
-    ([lens, brief]) =>
-      () =>
-        agent(`${COMMON}\n\nYOUR LENS - ${lens}: ${brief}`, {
-          label: `audit:${lens}`,
-          phase: "Audit",
-          schema: LENS_SCHEMA,
-        }),
-  ),
-);
-const all = results.filter(Boolean).flatMap((r) => r.findings.map((f) => ({ ...f, lens: r.lens })));
+const LENS_RESULT = {
+  type: 'object',
+  additionalProperties: false,
+  required: ['lens', 'findings'],
+  properties: {
+    lens: { type: 'string' },
+    findings: { type: 'array', items: FINDING },
+  },
+}
 
-phase("Synthesis");
-const SYNTH = {
-  type: "object",
+const SYNTHESIS = {
+  type: 'object',
   additionalProperties: false,
+  required: ['remaining', 'clearsBar', 'verdict'],
   properties: {
     remaining: {
-      type: "array",
+      type: 'array',
       items: {
-        type: "object",
+        type: 'object',
         additionalProperties: false,
-        properties: { ...FINDING.properties, lenses: { type: "string" } },
-        required: [...FINDING.required, "lenses"],
+        required: ['severity', 'location', 'problem', 'fix', 'lenses'],
+        properties: { lenses: { type: 'string', description: 'every lens that reported it, comma-separated' }, ...FINDING.properties },
       },
     },
-    clearsBar: { type: "boolean" },
-    verdict: { type: "string" },
+    clearsBar: { type: 'boolean' },
+    verdict: { type: 'string', description: 'one paragraph' },
   },
-  required: ["remaining", "clearsBar", "verdict"],
-};
-return await agent(
-  `${COMMON}\n\nYou are the SYNTHESIS architect. Below are ${all.length} raw findings from ${LENSES.length} lenses. Dedup overlaps (note which lenses raised each). DROP false positives - verify against the files before keeping; if you cannot reproduce it, drop it. Classify each survivor; set clearsBar=true ONLY if no blockers and no majors remain. One-paragraph verdict.\n\nRAW:\n${JSON.stringify(all, null, 2)}`,
-  { label: "synthesis", phase: "Synthesis", schema: SYNTH },
-);
+}
+
+const PREAMBLE = [
+  'You are one reviewer on an independent audit panel for a plugin skill. Stay read-only: do not edit any file.',
+  `Read the current state of every target file now; do not rely on anything you saw earlier:\n- ${TARGET.join('\n- ')}`,
+  `Rubric: ${RUBRIC}`,
+  `Exemplar of the same layer: ${EXEMPLAR}`,
+  `Ground truth the skill describes: ${GROUND_TRUTH}`,
+  'Report only genuine findings for your lens that you can reproduce by pointing at a line. Check each one against the rubric and the ground truth before reporting it.',
+  'Each finding needs a severity (blocker, major, minor, nit), a location, the problem, and a concrete fix. If your lens finds nothing, return an empty findings array.',
+].join('\n\n')
+
+phase('Audit')
+const lensOutputs = await parallel(LENSES.map((lensDef) => () =>
+  agent(`${PREAMBLE}\n\nYour lens: ${lensDef.label}. ${lensDef.brief}\nSet "lens" to "${lensDef.label}".`, {
+    label: `audit:${lensDef.label}`,
+    phase: 'Audit',
+    schema: LENS_RESULT,
+  })))
+
+const raw = lensOutputs
+  .filter(Boolean)
+  .flatMap((out) => out.findings.map((item) => ({ ...item, lens: out.lens })))
+log(`${raw.length} raw finding(s) from ${lensOutputs.filter(Boolean).length}/${LENSES.length} lenses`)
+
+phase('Synthesis')
+const synthesis = await agent([
+  'You synthesize an audit panel\'s findings for a plugin skill. Stay read-only.',
+  `Target files:\n- ${TARGET.join('\n- ')}`,
+  `Rubric: ${RUBRIC}`,
+  `Ground truth: ${GROUND_TRUTH}`,
+  `Raw findings (JSON):\n${JSON.stringify(raw, null, 2)}`,
+  'Merge duplicates and record every lens that raised each merged item in "lenses".',
+  'Open the cited file for each finding and drop any you cannot reproduce there.',
+  'Classify what survives as blocker, major, minor or nit.',
+  'Set clearsBar to true only when no blocker and no major remains.',
+  'Write a one-paragraph verdict.',
+].join('\n\n'), { label: 'synthesis', phase: 'Synthesis', schema: SYNTHESIS })
+
+return synthesis

package/pipeline/skills/shared/external/skill-creator/template.md

@@ -1,67 +1,79 @@
-# Skill skeletons
+# Skill templates
 
-Two skeletons - pick by layer. Both stay lean; push detail into one-level-deep bundled files.
+Two skeletons. Copy the one that matches the layer, fill it, then delete every line that fails the cut test. The toolkits use these frontmatter keys: `name`, `description`, `allowed-tools`, `user-invocable`, plus an optional `argument-hint`.
 
-Frontmatter fields used in this toolkit: `name`, `description` (required-quality), `user-invocable`, `argument-hint` (optional), `allowed-tools`.
-
----
-
-## Reference skill (descriptive - knowledge)
-
-No numbered procedures. Principles + pointers + heuristics.
+## Reference skill
 
 ```markdown
 ---
-name: <aspect-noun>          # e.g. navigation - NO platform prefix
-description: <what it covers + when to reach for it + trigger words. Say "iOS/SwiftUI" so the router can disambiguate.>
+name: {aspect-noun}
+description: "{What it covers} for {platform}. {When to reach for it}. Triggers: '{phrase}', '{phrase}', '{phrase}'."
 user-invocable: true
-allowed-tools: Read, Grep, Glob, Bash, Edit, Write
+allowed-tools: Bash, Read, Edit, Write, Glob, Grep
 ---
 
-# <name> - <short title>
+# {name} - {short title}
 
-> **Layer:** reference (knowledge)
-> **Use when:** <trigger>   **Hand off to:** <sibling skill>
+> **Layer:** reference (knowledge). **Use when:** {situation}. **Hand off to:** {sibling skill} for {adjacent concern}.
 
 ## Scope
-In / out, 1-2 lines.
+In: {one line}. Out: {one line}.
 
-## Invariants            <!-- non-negotiables + the why -->
-## Conventions & golden paths   <!-- point to canonical files; don't paste code -->
-## Decision rules        <!-- if X do A; when unsure, ask/verify -->
-## Pitfalls              <!-- known traps -->
-## Escape hatches        <!-- when this doesn't apply → which skill -->
-```
+## Invariants
+- **{Rule}.** {Why it holds; what breaks otherwise.}
 
----
+## Conventions and golden paths
+- {Convention} - see `{path/to/canonical/file}`. Point at the file; do not paste its code.
+
+## Decision rules
+- {Condition} -> {choice}.
 
-## Workflow skill (imperative - action)
+## Pitfalls
+- {Mistake} - {symptom} - {correct move}.
 
-Has a numbered procedure, but only for the deterministic/irreversible spine. Cite reference skills for judgment. Always include a verification gate.
+## Escape hatches
+- {Case this skill does not cover} -> {skill to use}.
+```
+
+## Workflow skill
 
 ```markdown
 ---
-name: <verb-phrase>         # e.g. branch-and-pr
-description: <what it does + when to run it + trigger words (iOS/SwiftUI).>
+name: {verb-phrase}
+description: "{What it does} for {platform}, with {guardrails}. Use when {situation}. Triggers: '{phrase}', '{phrase}'."
 user-invocable: true
-argument-hint: [<step> | ...]
+argument-hint: "{args}"
 allowed-tools: Bash, Read
 ---
 
-# <name> - <short title>
+# {name} - {short title}
 
-> **Layer:** workflow (action)
-> **Use when:** <trigger>   **Don't use when:** <hand off>
+> **Layer:** workflow (action). **Use when:** {situation}. **Don't use when:** {situation} -> {sibling skill}.
 
 ## Scope
-In / out, 1-2 lines.
-
-## Invariants            <!-- hard rules / guardrails -->
-## Conventions & golden paths   <!-- base branch, auth, canonical commands -->
-## Decision rules        <!-- branch when..., escalate when... -->
-## Procedures            <!-- IMPERATIVE: only deterministic/irreversible steps -->
-1. ...
-## Verification          <!-- IMPERATIVE: prove it worked (build/run/test) -->
+In: {one line}. Out: {one line}.
+
+## Invariants
+- **{Rule}.** {Why.}
+
+## Conventions and golden paths
+- Base branch: {branch}. Auth: {how}. Canonical commands: `{command}`.
+
+## Decision rules
+- {Condition} -> {choice}. Judgement calls cite `{reference skill}`.
+
+## Procedures
+1. {Deterministic, imperative step.}
+2. {Next step.}
+
+## Verification
+- Run `{build / test / check}` and confirm {observable result}. The skill is not done until this passes.
+
 ## Pitfalls
+- {Mistake} - {correct move}.
+
 ## Escape hatches
+- {Case outside scope} -> {skill to use}.
 ```
+
+A tool skill follows the workflow skeleton cut down to: when to use, the run command, a sub-command overview, and a link to its bundled `reference.md`.