@botbotgo/kit-builtin 0.0.97 → 0.0.98

package/skills/skill-quality-guard/scripts/audit-skill.mjs

@@ -0,0 +1,427 @@
+import { readFile, readdir, stat, writeFile } from "node:fs/promises";
+import { resolve, join, relative } from "node:path";
+import { parseMarkdownYamlFrontmatter } from "@botbotgo/common/utils";
+
+const NAME_PATTERN = /^[a-z0-9-]+$/;
+const RESERVED_WORDS = ["anthropic", "claude"];
+const REQUIRED_SECTIONS = ["goal", "inputs", "workflow"];
+const DISCOURAGED_FILES = [".DS_Store", "Thumbs.db"];
+const DIRTY_DIRS = ["node_modules", "dist", "build", "output", ".cache", ".git"];
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const token = argv[i];
+    if (!token.startsWith("--")) continue;
+    const key = token.slice(2);
+    const next = argv[i + 1];
+    if (!next || next.startsWith("--")) {
+      out[key] = "true";
+      continue;
+    }
+    out[key] = next;
+    i += 1;
+  }
+  return out;
+}
+
+function toStringValue(v) {
+  if (typeof v === "string") return v;
+  if (typeof v === "number" || typeof v === "boolean") return String(v);
+  if (Array.isArray(v)) return v.map((x) => toStringValue(x)).join("\n");
+  return "";
+}
+
+function addFinding(findings, severity, id, message, suggestion, path) {
+  findings.push({ severity, id, message, suggestion, ...(path ? { path } : {}) });
+}
+
+function validateFrontmatter(frontmatter, findings, filePath) {
+  const name = toStringValue(frontmatter.name).trim();
+  const description = toStringValue(frontmatter.description).trim();
+  const compatibility = toStringValue(frontmatter.compatibility).trim();
+  const allowedTools = toStringValue(frontmatter["allowed-tools"] || frontmatter.allowedTools).trim();
+
+  if (!name) {
+    addFinding(findings, "error", "frontmatter-name-missing", "Missing required frontmatter field: name", "Add a lowercase hyphenated name in YAML frontmatter.", filePath);
+  } else {
+    if (name.length > 64) {
+      addFinding(findings, "error", "frontmatter-name-too-long", `Frontmatter name exceeds 64 chars (${name.length})`, "Shorten the skill name to <=64 characters.", filePath);
+    }
+    if (!NAME_PATTERN.test(name) || name.startsWith("-") || name.endsWith("-") || name.includes("--")) {
+      addFinding(findings, "error", "frontmatter-name-invalid", "Frontmatter name must use lowercase letters, numbers, and single hyphens", "Use a name like `skill-quality-guard`.", filePath);
+    }
+    for (const reserved of RESERVED_WORDS) {
+      if (name.includes(reserved)) {
+        addFinding(findings, "error", "frontmatter-name-reserved", `Frontmatter name cannot include reserved word: ${reserved}`, "Choose a neutral tool name.", filePath);
+      }
+    }
+  }
+
+  if (!description) {
+    addFinding(findings, "error", "frontmatter-description-missing", "Missing required frontmatter field: description", "Add a concise description in frontmatter.", filePath);
+  } else if (description.length > 1024) {
+    addFinding(findings, "error", "frontmatter-description-too-long", `Frontmatter description exceeds 1024 chars (${description.length})`, "Shorten description to <=1024 characters.", filePath);
+  }
+
+  if (compatibility.length > 500) {
+    addFinding(findings, "warning", "frontmatter-compatibility-too-long", `Frontmatter compatibility is very long (${compatibility.length})`, "Keep compatibility short to improve readability.", filePath);
+  }
+
+  if (allowedTools) {
+    const tools = allowedTools
+      .split(/[\s,]+/)
+      .map((x) => x.trim())
+      .filter(Boolean);
+    if (tools.includes("*")) {
+      addFinding(findings, "warning", "allowed-tools-wildcard", "allowed-tools uses wildcard (*)", "Prefer explicit tool allowlist for least privilege.", filePath);
+    }
+  }
+}
+
+function validateSections(instructions, findings, filePath) {
+  const headings = new Set(
+    String(instructions || "")
+      .split("\n")
+      .map((line) => line.trim())
+      .filter((line) => line.startsWith("#"))
+      .map((line) => line.replace(/^#+\s*/, "").toLowerCase()),
+  );
+
+  for (const section of REQUIRED_SECTIONS) {
+    if (!headings.has(section)) {
+      addFinding(findings, "warning", `section-${section}-missing`, `Suggested section missing: ${section}`, `Add a \`${section[0].toUpperCase()}${section.slice(1)}\` section for consistency.`, filePath);
+    }
+  }
+
+  const text = String(instructions || "");
+  if (text.length > 12000) {
+    addFinding(findings, "warning", "instructions-too-long", `Instruction body is very long (${text.length} chars)`, "Move long details into references/*.md and keep SKILL.md concise.", filePath);
+  }
+  if (/\b(TODO|TBD|FIXME)\b/i.test(text)) {
+    addFinding(findings, "warning", "instructions-placeholder", "Instruction body contains TODO/TBD/FIXME placeholders", "Replace placeholders with executable steps.", filePath);
+  }
+  if (/\/(Users|home)\/[\w.-]+\//.test(text) || /[A-Za-z]:\\\\/.test(text)) {
+    addFinding(findings, "warning", "instructions-absolute-path", "Instruction body contains absolute local paths", "Use relative paths so the skill is portable.", filePath);
+  }
+}
+
+function validateStepPrinciples(instructions, findings, filePath) {
+  const text = String(instructions || "");
+  const stepRegex = /^##+\s+Step[^\n]*$/gim;
+  const matches = [...text.matchAll(stepRegex)];
+
+  if (matches.length === 0) {
+    addFinding(
+      findings,
+      "warning",
+      "workflow-step-structure-missing",
+      "Workflow does not define explicit Step sections",
+      "Use explicit Step sections so each step can be audited by execution style.",
+      filePath,
+    );
+    return;
+  }
+
+  for (let i = 0; i < matches.length; i += 1) {
+    const current = matches[i];
+    const next = matches[i + 1];
+    const title = (current[0] || "").trim();
+    const start = current.index + current[0].length;
+    const end = next ? next.index : text.length;
+    const block = text.slice(start, end);
+
+    const hasCommands =
+      /```(?:bash|sh|shell)[\s\S]*?```/i.test(block) ||
+      /(?:^|\n)\s*\d+\.\s*Commands?\b/i.test(block) ||
+      /(?:^|\n)\s*[-*]\s*`[^`\n]+`/.test(block);
+    const hasTools =
+      /(?:^|\n)\s*\d+\.\s*Tools?\b/i.test(block) ||
+      /(?:^|\n)\s*[-*]\s*`(?:execute|read_file|write_file|list_dir|searchText|analyzeProjectRepo|askProjectRepo)`/i.test(block);
+    const hasRequirement = /(?:^|\n)\s*\d+\.\s*Requirement\b/i.test(block);
+    const hasValidation = /(?:^|\n)\s*\d+\.\s*Validation(?:\s+and\s+Retry)?\b/i.test(block);
+    const hasRetry = /\bretry\b/i.test(block);
+    const writesResource = /\b(write|save|create|render|generate)\b/i.test(block)
+      && /\b(file|json|html|artifact|report|output)\b/i.test(block);
+    const hasExplicitLocation =
+      /(?:^|\n)\s*\d+\.\s*Data\b[\s\S]*?(?:\bpath\b|\blocation\b|\boutputPath\b)/i.test(block) ||
+      /(?:^|\n)\s*\d+\.\s*Result\b[\s\S]*?`[^`\n/]+\/[^`\n]+`/i.test(block) ||
+      /(?:^|\n)\s*\d+\.\s*Commands?\b[\s\S]*?(?:--out\s+|--output\s+|output\/|`[^`\n/]+\/[^`\n]+`)/i.test(block) ||
+      /`[^`\n/]+\/[^`\n]+`/.test(block);
+
+    const modeCount = [hasCommands, hasTools, hasRequirement].filter(Boolean).length;
+    if (modeCount === 0) {
+      addFinding(
+        findings,
+        "warning",
+        "step-principle-missing",
+        `Step does not match any execution principle: ${title}`,
+        "Each step should be clearly Command-first, Tool-first, or Requirement-first.",
+        filePath,
+      );
+    }
+
+    const complexHint = /\b(analyze|extract|generate|build|render|workflow|diagram|report)\b/i.test(block);
+    if (complexHint && (!hasValidation || !hasRetry)) {
+      addFinding(
+        findings,
+        "warning",
+        "complex-step-validation-retry-missing",
+        `Complex step should define validation and retry: ${title}`,
+        "Add explicit Validation and Retry rules for complex steps.",
+        filePath,
+      );
+    }
+
+    if (writesResource && !hasExplicitLocation) {
+      addFinding(
+        findings,
+        "warning",
+        "generated-resource-location-missing",
+        `Generated resource step must define explicit output location: ${title}`,
+        "Add a concrete output path (for example `output/report.json`) in Data/Result/Commands.",
+        filePath,
+      );
+    }
+  }
+}
+
+function extractSectionBody(markdown, headingName) {
+  const text = String(markdown || "");
+  const lines = text.split("\n");
+  const startIdx = lines.findIndex((line) => /^##\s+/.test(line.trim())
+    && line.trim().replace(/^##\s+/, "").toLowerCase() === headingName.toLowerCase());
+  if (startIdx < 0) return "";
+  let endIdx = lines.length;
+  for (let i = startIdx + 1; i < lines.length; i += 1) {
+    if (/^##\s+/.test(lines[i].trim())) {
+      endIdx = i;
+      break;
+    }
+  }
+  return lines.slice(startIdx + 1, endIdx).join("\n");
+}
+
+function validateWorkflowOnlySteps(instructions, findings, filePath) {
+  const workflowBody = extractSectionBody(instructions, "Workflow");
+  if (!workflowBody.trim()) return;
+
+  const blocks = workflowBody
+    .split(/\n(?=##+\s+Step\b)/i)
+    .map((x) => x.trim())
+    .filter(Boolean);
+
+  if (blocks.length === 0) {
+    addFinding(
+      findings,
+      "warning",
+      "workflow-only-steps-missing",
+      "Workflow section should contain explicit Step blocks only",
+      "Use `## Step N: ...` blocks under Workflow and remove unrelated narrative text.",
+      filePath,
+    );
+    return;
+  }
+
+  const beforeFirstStep = workflowBody.split(/##+\s+Step\b/i)[0] || "";
+  const noiseLines = beforeFirstStep
+    .split("\n")
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .filter((line) => !line.startsWith("```"))
+    .filter((line) => !/^[-*]\s+/.test(line))
+    .filter((line) => !/^\d+\.\s+/.test(line));
+
+  if (noiseLines.length > 0) {
+    addFinding(
+      findings,
+      "warning",
+      "workflow-nonstep-content",
+      "Workflow section contains non-step content",
+      "Keep Workflow focused on concrete steps only; move other notes to another section.",
+      filePath,
+    );
+  }
+}
+
+async function walk(rootDir, currentDir, result) {
+  const entries = await readdir(currentDir, { withFileTypes: true });
+  for (const entry of entries) {
+    const abs = join(currentDir, entry.name);
+    const rel = relative(rootDir, abs).replaceAll("\\", "/");
+    if (entry.isDirectory()) {
+      result.directories.push(rel || ".");
+      if (DIRTY_DIRS.includes(entry.name)) {
+        continue;
+      }
+      await walk(rootDir, abs, result);
+      continue;
+    }
+    if (entry.isFile()) {
+      const st = await stat(abs);
+      result.files.push({ relPath: rel, size: st.size });
+    }
+  }
+}
+
+function validateCleanliness(walkResult, findings) {
+  const filesByDir = new Map();
+  for (const file of walkResult.files) {
+    const idx = file.relPath.lastIndexOf("/");
+    const dir = idx >= 0 ? file.relPath.slice(0, idx) : ".";
+    filesByDir.set(dir, (filesByDir.get(dir) || 0) + 1);
+
+    const base = file.relPath.split("/").pop() || file.relPath;
+    if (DISCOURAGED_FILES.includes(base)) {
+      addFinding(findings, "warning", "dirty-os-artifact", `Unexpected OS artifact file: ${file.relPath}`, "Remove editor/OS generated files from skill directories.", file.relPath);
+    }
+
+    if (file.relPath.endsWith(".log") || file.relPath.endsWith(".tmp")) {
+      addFinding(findings, "warning", "dirty-temp-file", `Unexpected temporary/log file: ${file.relPath}`, "Remove generated artifacts from skill folders.", file.relPath);
+    }
+
+    if (file.size > 256 * 1024) {
+      addFinding(findings, "warning", "oversized-file", `Large skill asset file: ${file.relPath} (${file.size} bytes)`, "Store heavy data outside skill assets or trim content.", file.relPath);
+    }
+  }
+
+  for (const dir of walkResult.directories) {
+    const leaf = dir.split("/").pop() || dir;
+    if (DIRTY_DIRS.includes(leaf)) {
+      addFinding(findings, "warning", "dirty-build-dir", `Generated/build directory detected inside skill: ${dir}`, "Do not keep build/cache/output folders inside skills.", dir);
+    }
+  }
+
+  for (const [dir, fileCount] of filesByDir.entries()) {
+    if (fileCount > 10) {
+      addFinding(findings, "info", "folder-density-high", `Folder has more than 10 files (${fileCount}): ${dir}`, "Consider splitting files into subfolders for maintainability.", dir);
+    }
+  }
+}
+
+function validateReferences(instructions, walkResult, findings) {
+  const refs = new Set();
+  const text = String(instructions || "");
+  const patterns = [
+    /(?:^|\s)(scripts\/[A-Za-z0-9._\/-]+)/gm,
+    /(?:^|\s)(references\/[A-Za-z0-9._\/-]+)/gm,
+    /(?:^|\s)(assets\/[A-Za-z0-9._\/-]+)/gm,
+  ];
+
+  for (const pattern of patterns) {
+    let m = pattern.exec(text);
+    while (m) {
+      refs.add(m[1].trim());
+      m = pattern.exec(text);
+    }
+  }
+
+  const fileSet = new Set(walkResult.files.map((f) => f.relPath));
+  for (const refPath of refs) {
+    if (!fileSet.has(refPath)) {
+      addFinding(findings, "error", "missing-referenced-asset", `Referenced asset not found: ${refPath}`, "Create the file or remove invalid reference in SKILL.md.", refPath);
+    }
+  }
+}
+
+function summarize(findings) {
+  const counts = { error: 0, warning: 0, info: 0 };
+  for (const f of findings) counts[f.severity] += 1;
+  const score = Math.max(0, 100 - counts.error * 20 - counts.warning * 6 - counts.info * 2);
+  const ok = counts.error === 0;
+  return { ok, score, counts };
+}
+
+export async function auditSkill({ skillPath, strict = false, outputPath } = {}) {
+  if (!skillPath || typeof skillPath !== "string") {
+    throw new Error("skillPath is required");
+  }
+
+  const root = resolve(skillPath);
+  const skillMdPath = join(root, "SKILL.md");
+  const findings = [];
+
+  let skillMd = "";
+  try {
+    skillMd = await readFile(skillMdPath, "utf8");
+  } catch {
+    addFinding(findings, "error", "skill-md-missing", "SKILL.md not found", "Create SKILL.md with YAML frontmatter and workflow instructions.", "SKILL.md");
+    const summary = summarize(findings);
+    const result = {
+      skillPath: root,
+      checkedAt: new Date().toISOString(),
+      ...summary,
+      findings,
+    };
+    if (outputPath) {
+      await writeFile(resolve(outputPath), `${JSON.stringify(result, null, 2)}\n`, "utf8");
+    }
+    if (strict) {
+      throw new Error(`Skill audit failed with ${summary.counts.error} error(s)`);
+    }
+    return result;
+  }
+
+  let frontmatter = {};
+  let instructions = "";
+  try {
+    const parsed = parseMarkdownYamlFrontmatter(skillMd, skillMdPath);
+    frontmatter = parsed.frontmatter || {};
+    instructions = parsed.body || "";
+  } catch (error) {
+    addFinding(findings, "error", "frontmatter-parse-failed", `Failed to parse YAML frontmatter: ${error instanceof Error ? error.message : String(error)}`, "Fix YAML frontmatter format in SKILL.md.", "SKILL.md");
+  }
+
+  validateFrontmatter(frontmatter, findings, "SKILL.md");
+  validateSections(instructions, findings, "SKILL.md");
+  validateWorkflowOnlySteps(instructions, findings, "SKILL.md");
+  validateStepPrinciples(instructions, findings, "SKILL.md");
+
+  const walkResult = { files: [], directories: [] };
+  await walk(root, root, walkResult);
+  validateCleanliness(walkResult, findings);
+  validateReferences(instructions, walkResult, findings);
+
+  const summary = summarize(findings);
+  const result = {
+    skillPath: root,
+    checkedAt: new Date().toISOString(),
+    ...summary,
+    findings,
+    stats: {
+      fileCount: walkResult.files.length,
+      dirCount: walkResult.directories.length,
+    },
+  };
+
+  if (outputPath) {
+    await writeFile(resolve(outputPath), `${JSON.stringify(result, null, 2)}\n`, "utf8");
+  }
+
+  if (strict && !summary.ok) {
+    throw new Error(`Skill audit failed: ${summary.counts.error} error(s), ${summary.counts.warning} warning(s)`);
+  }
+
+  return result;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const skillPath = typeof args.skill === "string" ? args.skill : "";
+  const strict = args.strict === "true";
+  const outputPath = typeof args.out === "string" && args.out.trim() ? args.out.trim() : undefined;
+
+  const report = await auditSkill({ skillPath, strict, outputPath });
+  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+
+  if (strict && !report.ok) {
+    process.exitCode = 1;
+  }
+}
+
+if (process.argv[1] && import.meta.url.endsWith(process.argv[1])) {
+  main().catch((error) => {
+    process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+    process.exit(1);
+  });
+}