novahiz 0.2.2 → 0.2.5

package/skills/secrets-hygiene/scripts/secret_scan.mjs

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+/**
+ * secrets-hygiene: high-signal secret patterns on staged/tracked text files.
+ * Offline regex pass. Not a substitute for gitleaks/TruffleHog in CI.
+ *
+ * Usage: node scripts/secret_scan.mjs <file-or-dir> [...]
+ * Exit 1 if any HIGH finding.
+ */
+
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { join, extname, basename } from "node:path";
+
+const EXTS = new Set([
+  ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".py", ".rb", ".go", ".rs",
+  ".java", ".kt", ".php", ".cs", ".sh", ".bash", ".zsh", ".env", ".yml",
+  ".yaml", ".json", ".toml", ".ini", ".cfg", ".conf", ".properties", ".tf",
+  ".md", ".txt", ".xml", ".html", ".css",
+]);
+
+const IGNORE_DIRS = new Set(["node_modules", ".git", "dist", "build", "target", ".next", "coverage"]);
+
+const RULES = [
+  {
+    id: "aws-access-key-id",
+    level: "HIGH",
+    re: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
+    msg: "AWS access key id",
+  },
+  {
+    id: "github-pat",
+    level: "HIGH",
+    re: /\bghp_[A-Za-z0-9]{36,}\b|\bgithub_pat_[A-Za-z0-9_]{22,}\b/g,
+    msg: "GitHub token",
+  },
+  {
+    id: "slack-token",
+    level: "HIGH",
+    re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g,
+    msg: "Slack token",
+  },
+  {
+    id: "private-key-block",
+    level: "HIGH",
+    re: /-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----/g,
+    msg: "PEM private key block",
+  },
+  {
+    id: "generic-assign-secret",
+    level: "MED",
+    re: /\b(?:api[_-]?key|secret|password|passwd|token)\s*[:=]\s*['"][^'"]{12,}['"]/gi,
+    msg: "Hardcoded secret-like assignment",
+  },
+  {
+    id: "jwt-compact",
+    level: "MED",
+    re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
+    msg: "JWT-looking string",
+  },
+];
+
+const ALLOW_SUBSTR = ["EXAMPLE", "example", "placeholder", "YOUR_", "xxxx", "dummy", "sample", "REPLACE"];
+
+function walk(path, out = []) {
+  const st = statSync(path);
+  if (st.isDirectory()) {
+    if (IGNORE_DIRS.has(basename(path))) return out;
+    for (const name of readdirSync(path)) walk(join(path, name), out);
+  } else if (EXTS.has(extname(path).toLowerCase()) || basename(path).startsWith(".env")) {
+    out.push(path);
+  }
+  return out;
+}
+
+function scanFile(file) {
+  let text;
+  try {
+    text = readFileSync(file, "utf8");
+  } catch {
+    return [];
+  }
+  const findings = [];
+  for (const rule of RULES) {
+    const re = new RegExp(rule.re.source, rule.re.flags.includes("g") ? rule.re.flags : rule.re.flags + "g");
+    let m;
+    const seen = new Set();
+    while ((m = re.exec(text)) !== null) {
+      const match = m[0];
+      if (ALLOW_SUBSTR.some((a) => match.includes(a))) continue;
+      const line = text.slice(0, m.index).split(/\r?\n/).length;
+      const key = `${rule.id}:${line}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      findings.push({ file, line, ...rule });
+      if (findings.length > 50) break;
+    }
+    if (findings.length > 50) break;
+  }
+  return findings;
+}
+
+const roots = process.argv.slice(2);
+if (roots.length === 0) {
+  console.error("usage: node scripts/secret_scan.mjs <file-or-dir> [...]");
+  process.exit(2);
+}
+
+const files = roots.flatMap((r) => walk(r));
+const all = files.flatMap(scanFile);
+const byLevel = { HIGH: [], MED: [], LOW: [] };
+for (const f of all) byLevel[f.level].push(f);
+
+for (const level of ["HIGH", "MED", "LOW"]) {
+  for (const f of byLevel[level]) {
+    console.log(`${level}\t${f.id}\t${f.file}:${f.line}\t${f.msg}`);
+  }
+}
+
+const high = byLevel.HIGH.length;
+console.error(`secret_scan: ${files.length} files, ${all.length} findings (${high} HIGH)`);
+process.exit(high > 0 ? 1 : 0);

package/skills/skill-authoring/SKILL.md

@@ -0,0 +1,93 @@
+---
+name: "skill-authoring"
+description: "Write or rewrite an agent SKILL.md: frontmatter contract, description as routing (what + when + differentiator), progressive disclosure, lean body, references and scripts layout, ship checklist. Use when creating a skill, fixing under/over-triggering, or restructuring a long SKILL.md."
+license: "Apache-2.0"
+metadata:
+  author: Novahiz
+  organization: Novahiz
+  version: "1.0.0"
+  date: September 2026
+---
+
+# skill-authoring
+
+A skill is a folder: `SKILL.md` plus optional `scripts/`, `references/`, `assets/`. Agents load metadata always, body on match, files and scripts only when needed. Write for that loading order.
+
+## Frontmatter contract
+
+```yaml
+---
+name: kebab-case-name          # ≤64, matches folder, [a-z0-9-]
+description: What it does. Use when ... Trigger phrases. Differentiator vs sibling skills.  # ≤1024
+license: Apache-2.0
+metadata:
+  author: ...
+---
+```
+
+- Quote the description if it contains `: ` so strict YAML parsers do not see a nested map.
+- Avoid `<` `>` in frontmatter; they can leak into prompts.
+- Invalid YAML often fails silently (skill never loads).
+
+## Description is the router
+
+The model sees only name + description before deciding to open the file.
+
+| Include | Avoid |
+|---|---|
+| What the skill does | Step-by-step “how” (model will follow the short version and skip the body) |
+| When to use (triggers, file types, user phrasings) | Empty praise (“helps with projects”) |
+| Differentiator from a sibling skill | Competing skills’ full menus |
+
+Pattern: `X via Y. Use for [situations]. [When not / other skill instead if needed].`
+
+If it does not trigger: fix description first (95% of the time). If it triggers wrong: narrow or add negatives. Fix the body only when explicit invocation produces wrong work.
+
+## Progressive disclosure layout
+
+```text
+my-skill/
+  SKILL.md          # workflow, checklists, pointers
+  references/*.md   # deep detail, one level deep
+  scripts/*         # deterministic work; output enters context, source need not
+  assets/*          # templates
+```
+
+- Keep body scannable; when it grows past usefulness, move detail to `references/` and say **when** to open each file.
+- No chains: `SKILL.md → a.md → b.md → c.md`.
+- Long reference: table of contents at the top.
+- Scripts for anything fragile or repeated; prose for judgment calls.
+
+## Body shape
+
+1. What this does / when it applies (one short block).
+2. Ordered workflow with state checks before actions.
+3. Verify → fix → re-verify loop stated explicitly.
+4. Output format if the consumer expects structure.
+5. Pointers: “For X, read `references/x.md`.”
+6. Sources / constraints that must stay visible.
+
+Match strictness to fragility: heuristics for open-ended work; exact steps and scripts for install, migrations, gates.
+
+## Composition
+
+- One skill, one concern.
+- If skill A produces artifacts skill B reads, document the file shape in both.
+- Shared config file beats hidden coupling.
+- Document tool needs (`allowed-tools` only where the host supports it; not universal).
+
+## Ship checklist
+
+- [ ] `name` matches folder; kebab-case.
+- [ ] description: what + when + differentiator; under limit; quoted if needed.
+- [ ] body has workflow, verification loop, failure notes.
+- [ ] references linked with explicit “when to read”.
+- [ ] scripts runnable; errors actionable.
+- [ ] no secrets, no absolute machine paths, no time bombs.
+- [ ] humanizer pass on design-frontend prose (R13); outside design it is optional, not gate-required.
+- [ ] implicit trigger test + explicit execution test (`skill-eval-loop`).
+- [ ] versioned with the repo.
+
+## Sources
+
+Anthropic Agent Skills engineering notes and platform docs (progressive disclosure, description guidance); agentskills.io description optimization; Microsoft Agent Skills structure fields; community skill-authoring guides distilled to routing and disclosure rules. Original Novahiz synthesis; no upstream skill text reused wholesale.

package/skills/skill-authoring/references/skill-template.md

@@ -0,0 +1,59 @@
+# SKILL.md template
+
+Copy into `skills/<name>/SKILL.md`. Fill every bracket. Delete optional blocks you do not need.
+
+```markdown
+---
+name: "<folder-name>"
+description: "<What it does>. Use when <situations and trigger phrases>. <Differentiator vs sibling skill>."
+license: "Apache-2.0"
+metadata:
+  author: Novahiz
+  organization: Novahiz
+  version: "1.0.0"
+  date: September 2026
+---
+
+# <folder-name>
+
+<One or two lines: goal of the skill and when it ends.>
+
+## Workflow
+
+1. <State check before action.>
+2. <Action.>
+3. <Verify what you did.>
+4. <Fix and re-verify if needed.>
+
+## Checklist
+
+- [ ] <Acceptance item>
+- [ ] <Acceptance item>
+
+## References
+
+- For <deep topic>, read `references/<file>.md`.
+- For <repeatable task>, run `scripts/<tool>.mjs`.
+
+## Sources
+
+<Primary sources used. Original synthesis note.>
+```
+
+## Description formula
+
+`<What>. Use when <trigger>. <Differentiator>.`
+
+Keep under 1024 characters. Quote the whole string in YAML if it contains a colon.
+
+## Folder
+
+```text
+skills/<name>/
+  SKILL.md
+  scripts/     # optional
+  references/  # optional, one level deep
+  assets/      # optional templates
+```
+
+Name must match folder. After writing, run `scripts/skill_frontmatter_lint.mjs`.

package/skills/skill-authoring/scripts/skill_frontmatter_lint.mjs

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+/**
+ * skill-authoring: validate SKILL.md frontmatter + body shape.
+ * Usage: node scripts/skill_frontmatter_lint.mjs <path/to/SKILL.md> [...]
+ * Exit 1 if any HIGH finding.
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+
+function parse(text) {
+  const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);
+  if (!m) return { ok: false };
+  const fields = {};
+  for (const line of m[1].split(/\r?\n/)) {
+    const kv = /^([A-Za-z0-9_]+):\s*(.*)$/.exec(line);
+    if (!kv) continue;
+    let v = kv[2].trim();
+    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
+    fields[kv[1]] = v;
+  }
+  return { ok: true, fields, body: text.slice(m[0].length) };
+}
+
+function lint(path) {
+  const out = [];
+  const push = (level, id, msg) => out.push({ level, id, path, msg });
+  if (!existsSync(path)) {
+    push("HIGH", "missing-file", "not found");
+    return out;
+  }
+  const text = readFileSync(path, "utf8");
+  const p = parse(text);
+  if (!p.ok) {
+    push("HIGH", "no-frontmatter", "missing --- frontmatter block");
+    return out;
+  }
+  const f = p.fields;
+  if (!f.name) push("HIGH", "no-name", "missing name");
+  else {
+    if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(f.name)) push("HIGH", "name-format", "name must be lowercase kebab-case");
+    if (f.name.length > 64) push("HIGH", "name-length", "name > 64 chars");
+  }
+  if (!f.description) push("HIGH", "no-description", "missing description");
+  else {
+    if (f.description.length > 1024) push("HIGH", "description-length", `description ${f.description.length} > 1024`);
+    if (/[<>]/.test(f.description)) push("HIGH", "description-xml", "description contains < or >");
+    if (!/use when|use for|when to use|trigger/i.test(f.description)) push("MED", "description-when", "description should say when to use it");
+  }
+  if (/[<>]/.test(text.replace(/^---[\s\S]*?---/, ""))) push("LOW", "body-xml", "body contains angle brackets; verify intentional");
+  if (!/##/.test(p.body)) push("MED", "no-sections", "body has no ## sections");
+  return out;
+}
+
+const files = process.argv.slice(2);
+if (!files.length) {
+  console.error("usage: node scripts/skill_frontmatter_lint.mjs <SKILL.md> [...]");
+  process.exit(2);
+}
+const all = files.flatMap(lint);
+for (const level of ["HIGH", "MED", "LOW"]) {
+  for (const f of all.filter((x) => x.level === level)) console.log(`${level}\t${f.id}\t${f.path}\t${f.msg}`);
+}
+const high = all.filter((f) => f.level === "HIGH").length;
+console.error(`skill_frontmatter_lint: ${files.length} files, ${all.length} findings (${high} HIGH)`);
+process.exit(high > 0 ? 1 : 0);

package/skills/skill-eval-loop/SKILL.md

@@ -0,0 +1,103 @@
+---
+name: "skill-eval-loop"
+description: "Measure whether an agent skill actually helps: prompt sets with should/should-not trigger, with/without skill runs, deterministic checks, rubric grading, CI regressions. Use when hardening a SKILL.md, deciding if a rewrite improved quality, or adding a quality gate for skills."
+license: "Apache-2.0"
+metadata:
+  author: Novahiz
+  organization: Novahiz
+  version: "1.0.0"
+  date: September 2026
+---
+
+# skill-eval-loop
+
+Skill quality is a measured delta, not a feeling. Define done, run with and without the skill, score, and keep the score across versions.
+
+## What an eval is
+
+`prompt → captured run (trace + artifacts) → checks → score you can compare over time`
+
+Three layers, cheapest first:
+
+1. **Deterministic checks**: files exist, commands ran in order, JSONL events, exit codes.
+2. **Rubric / judge**: qualitative structure and style; schema-constrained JSON output; blind to which condition produced the text when comparing A/B.
+3. **Regression diff**: baseline snapshot vs current; fail CI when score or trigger rate drops past a threshold.
+
+## Prompt set
+
+Start with 10-20 rows, grow from real failures.
+
+| Column | Meaning |
+|---|---|
+| id | stable key |
+| prompt | user-style request |
+| expect_trigger | yes / no |
+| must_steps | commands or artifacts that prove success (optional) |
+| must_not | failures that count as regression |
+
+Include negative prompts (nearby topics that must not load the skill). Without negatives, a broader description always looks “better” until production noise appears.
+
+## Trigger testing
+
+- Explicit load: invoke the skill directly; fix the **body** if output is wrong.
+- Implicit load: paraphrase the user ask without naming the skill; fix the **description** if it never fires.
+- Over-trigger: unrelated prompts must not load it; narrow description or add a do-not-use clause.
+
+Description problems look like body problems until you separate these two tests.
+
+## With / without skill
+
+For quality claims, run each prompt twice under the same model and temperature: skill injected vs not. Score both with the same judge and rubric. Report mean, spread, and delta. Overlapping spreads at low N mean “add runs,” not “no effect.”
+
+- Development: 3 runs per prompt.
+- Decision-grade: 10+ runs, 1-3 judges, judge temperature near 0.
+- Judge sees rubric + output; ideally not which arm is which.
+
+## Deterministic traces
+
+Prefer harnesses that emit JSONL events (command executions, file writes). Assert on:
+
+- required commands present and ordered
+- forbidden commands absent
+- artifact paths exist
+- no extra files outside allowed set
+
+These fail fast and explain themselves without another model call.
+
+## Rubric grading
+
+When checks cannot see “is this good structure,” add a second pass that only reads the repo (or the answer) and returns a fixed schema: `overall_pass`, `score`, per-check ids. Same schema every version so you can diff fields, not prose.
+
+## CI gate
+
+```text
+baseline.json  +  current run  →  report diff  →  fail on:
+  - trigger rate drop on should-trigger set
+  - any must_steps regression
+  - composite score drop > agreed epsilon
+```
+
+Keep the baseline in-repo or artifact storage. Update baseline only in the PR that intentionally changes behavior, with a note.
+
+## Loop
+
+1. Manual explore: force the skill, list every miss.
+2. Turn each miss into a row (should or should-not).
+3. Freeze deterministic checks.
+4. Add rubric only where rules cannot.
+5. Record baseline; wire CI.
+6. On every SKILL.md edit, run the set before merge.
+
+## Checklist
+
+- [ ] Definition of done written before prompts.
+- [ ] Mix of trigger and non-trigger prompts.
+- [ ] With/without arm for quality claims.
+- [ ] Deterministic checks cover steps and artifacts.
+- [ ] Rubric JSON schema stable across versions.
+- [ ] Baseline + diff in CI.
+- [ ] New failures always become new rows.
+
+## Sources
+
+OpenAI developer guidance on skill evals (JSONL traces, schema rubrics, small prompt sets); SkillBenchmark / SkillsBench style with-vs-without and blind judging; arXiv framework for skill utility evaluation. Original Novahiz synthesis.

package/skills/skill-eval-loop/references/negative-patterns.md

@@ -0,0 +1,12 @@
+# Negative prompt patterns
+
+Add rows here whenever a false trigger is found in production or eval.
+
+| Skill | Prompt that must NOT trigger | Why it looked related | Fix applied |
+|---|---|---|---|
+| skill-eval-loop | "write a new skill from scratch" | same "skill" word | description steers authoring to skill-authoring |
+| openapi-mcp-server | "explain what MCP is" | MCP keyword | description is build-oriented |
+| browser-session | "install playwright" | playwright keyword | description is session/automation oriented |
+| skill-authoring | "measure skill quality" | skill keyword | description points eval work to skill-eval-loop |
+
+Keep this file next to the prompt set (`prompts.csv`). One row per false positive; re-run the negative set after every description edit.

package/skills/skill-eval-loop/references/run-sheet.md

@@ -0,0 +1,40 @@
+# Eval run sheet
+
+Skill: ______________  Version / commit: ______________  Date: ______________
+
+## Config
+
+- Model / harness:
+- Temperature:
+- Runs per prompt (dev 3 / ship 10+):
+- Judges:
+- Baseline file:
+
+## Prompt set summary
+
+| Class | Count | Pass |
+|---|---|---|
+| should trigger | | |
+| should not | | |
+| deterministic must_steps | | |
+
+## Results
+
+| Metric | Baseline | Current | Delta | Verdict |
+|---|---|---|---|---|
+| Trigger rate (should) | | | | |
+| False trigger rate | | | | |
+| must_steps pass | | | | |
+| Rubric mean (with skill) | | | | |
+| Rubric mean (without) | | | | |
+| Skill delta | | | | |
+
+## Decision
+
+- [ ] Ship (no blocking regression)
+- [ ] Hold (fix description or body first)
+- [ ] Update baseline (intentional behavior change; note why: ________)
+
+## Notes / new rows to add
+
+- 

package/skills/skill-eval-loop/scripts/skill_structure_check.mjs

@@ -0,0 +1,100 @@
+#!/usr/bin/env node
+/**
+ * skill-eval-loop: structural + routing checks for a SKILL.md before eval runs.
+ * Validates frontmatter shape and prints a prompt-set template if missing.
+ *
+ * Usage: node scripts/skill_structure_check.mjs <path/to/SKILL.md> [prompts.csv]
+ * Exit 1 on structural HIGH failures.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+
+function parseFrontmatter(text) {
+  const m = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);
+  if (!m) return { ok: false, error: "missing YAML frontmatter --- block" };
+  const body = m[1];
+  const fields = {};
+  for (const line of body.split(/\r?\n/)) {
+    const kv = /^([A-Za-z0-9_-]+):\s*(.*)$/.exec(line);
+    if (!kv) continue;
+    let v = kv[2].trim();
+    if (
+      (v.startsWith('"') && v.endsWith('"')) ||
+      (v.startsWith("'") && v.endsWith("'"))
+    ) {
+      v = v.slice(1, -1);
+    }
+    fields[kv[1]] = v;
+  }
+  return { ok: true, fields, bodyStart: m[0].length };
+}
+
+const TEMPLATE = `id,prompt,expect_trigger,must_steps,must_not
+t1,"Primary request that should load the skill",yes,"",
+t2,"Paraphrase of the same intent",yes,"",
+t3,"Nearby topic that must NOT trigger",no,"",
+t4,"Edge case discovered in manual testing",yes,"",
+`;
+
+const skillPath = process.argv[2];
+if (!skillPath) {
+  console.error("usage: node scripts/skill_structure_check.mjs <path/to/SKILL.md> [prompts.csv]");
+  process.exit(2);
+}
+if (!existsSync(skillPath)) {
+  console.error(`missing: ${skillPath}`);
+  process.exit(1);
+}
+
+const text = readFileSync(skillPath, "utf8");
+const findings = [];
+const push = (level, id, msg) => findings.push({ level, id, msg });
+
+const fm = parseFrontmatter(text);
+if (!fm.ok) {
+  push("HIGH", "no-frontmatter", fm.error);
+} else {
+  const f = fm.fields;
+  const dir = basename(dirname(skillPath));
+  if (!f.name) push("HIGH", "no-name", "frontmatter missing name");
+  else {
+    if (f.name !== dir) push("HIGH", "name-dir-mismatch", `name "${f.name}" != folder "${dir}"`);
+    if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(f.name)) push("HIGH", "name-format", "name must be lowercase kebab");
+    if (f.name.length > 64) push("HIGH", "name-length", "name > 64 chars");
+  }
+  if (!f.description) push("HIGH", "no-description", "frontmatter missing description");
+  else {
+    if (f.description.length > 1024) push("HIGH", "description-length", `description ${f.description.length} > 1024`);
+    if (/[<>]/.test(f.description)) push("HIGH", "description-xml", "description contains < or >");
+    if (!/use when|when |trigger|for /i.test(f.description)) {
+      push("MED", "description-when", "description should say when to use it");
+    }
+    if (/^(helps with|does stuff)/i.test(f.description.trim())) {
+      push("HIGH", "description-vague", "description too vague to route");
+    }
+  }
+}
+
+const body = text.slice(fm.bodyStart || 0);
+const lines = text.split(/\r?\n/).length;
+if (lines > 500) push("MED", "long-skill", `SKILL.md is ${lines} lines; move detail to references/`);
+if (/\n##\s+(Sources|Quellen)/i.test("\n" + body) === false && !/## Sources/i.test(body)) {
+  // soft only: many skills include Sources
+}
+if (!/##/.test(body)) push("MED", "no-sections", "body has no ## sections; hard to scan");
+
+const promptsArg = process.argv[3];
+if (promptsArg && !existsSync(promptsArg)) {
+  writeFileSync(promptsArg, TEMPLATE, "utf8");
+  console.error(`created prompt template: ${promptsArg}`);
+}
+
+for (const level of ["HIGH", "MED", "LOW"]) {
+  for (const f of findings.filter((x) => x.level === level)) {
+    console.log(`${level}\t${f.id}\t${f.msg}`);
+  }
+}
+const high = findings.filter((f) => f.level === "HIGH").length;
+console.error(`skill_structure_check: ${findings.length} findings (${high} HIGH)`);
+process.exit(high > 0 ? 1 : 0);