@geminix/gxpm 0.1.0

package/scripts/dogfood-check.ts

@@ -0,0 +1,125 @@
+/**
+ * Dogfood compliance check: verifies that every gxpm issue created
+ * after SPECIFY_PHASE_CUTOFF has gone through the specify phase before
+ * reaching implement (or any phase past it).
+ *
+ * Default mode is informational (exit 0, print warnings).
+ * Pass `--strict` to make non-compliant issues fail the check (exit 1).
+ *
+ * Run: bun run scripts/dogfood-check.ts [--strict]
+ */
+
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { SPECIFY_PHASE_CUTOFF } from "../core/state";
+
+const PAST_SPECIFY_PHASES = new Set([
+  "implement",
+  "local-verify",
+  "ac-check",
+  "self-review",
+  "ship",
+  "pr-check",
+  "verify",
+  "qa",
+  "land",
+]);
+
+interface IssueViolation {
+  issueId: string;
+  currentPhase: string;
+  reason: string;
+}
+
+export interface DogfoodReport {
+  scanned: number;
+  legacyExempt: number;
+  compliant: number;
+  violations: IssueViolation[];
+}
+
+export function runDogfoodCheck(root: string = process.cwd()): DogfoodReport {
+  const issuesDir = join(root, ".gxpm", "issues");
+  const report: DogfoodReport = {
+    scanned: 0,
+    legacyExempt: 0,
+    compliant: 0,
+    violations: [],
+  };
+
+  if (!existsSync(issuesDir)) {
+    return report;
+  }
+
+  for (const entry of readdirSync(issuesDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const issueId = entry.name;
+    const statePath = join(issuesDir, issueId, "state.json");
+    if (!existsSync(statePath)) continue;
+
+    let state: { currentPhase?: string; phaseHistory?: Array<{ phase: string; enteredAt: string }> };
+    try {
+      state = JSON.parse(readFileSync(statePath, "utf8"));
+    } catch {
+      report.violations.push({
+        issueId,
+        currentPhase: "<unreadable>",
+        reason: "state.json is not valid JSON",
+      });
+      continue;
+    }
+
+    report.scanned++;
+    const phase = state.currentPhase ?? "<missing>";
+
+    if (!PAST_SPECIFY_PHASES.has(phase)) {
+      // Issue not yet at implement; specify is not strictly required yet.
+      report.compliant++;
+      continue;
+    }
+
+    const history = state.phaseHistory ?? [];
+    const implementEntry = history.find((h) => h.phase === "implement");
+    if (implementEntry && implementEntry.enteredAt < SPECIFY_PHASE_CUTOFF) {
+      // Legacy issue — exempt per the same rule the phase-gate enforces.
+      report.legacyExempt++;
+      continue;
+    }
+
+    const specifyEntry = history.find((h) => h.phase === "specify");
+    if (!specifyEntry) {
+      report.violations.push({
+        issueId,
+        currentPhase: phase,
+        reason: "post-cutoff issue reached implement without entering specify",
+      });
+      continue;
+    }
+
+    report.compliant++;
+  }
+
+  return report;
+}
+
+function formatReport(report: DogfoodReport): string {
+  const lines: string[] = [];
+  lines.push(`dogfood-check: scanned ${report.scanned} issue(s)`);
+  lines.push(`  compliant:     ${report.compliant}`);
+  lines.push(`  legacy exempt: ${report.legacyExempt}`);
+  lines.push(`  violations:    ${report.violations.length}`);
+  for (const v of report.violations) {
+    lines.push(`    - ${v.issueId} (in ${v.currentPhase}): ${v.reason}`);
+  }
+  return lines.join("\n");
+}
+
+if (import.meta.main) {
+  const strict = process.argv.includes("--strict");
+  const root = resolve(process.cwd());
+  const report = runDogfoodCheck(root);
+  console.log(formatReport(report));
+  if (strict && report.violations.length > 0) {
+    process.exit(1);
+  }
+}

package/scripts/eval-functional.ts

@@ -0,0 +1,218 @@
+#!/usr/bin/env bun
+/**
+ * gxpm-eval-functional — functional eval runner for discipline skills.
+ *
+ * Reads pressure scenarios from test/functional/<skill-name>/,
+ * runs them WITH and WITHOUT the skill, and reports compliance rates.
+ *
+ * Skeleton (P1-4): scenario parsing, CLI, report format.
+ * Full LLM-based agent evaluation to be wired in Phase-3.
+ */
+
+import { readFileSync, readdirSync, existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const ROOT = join(import.meta.dir, "..");
+
+interface PressureScenario {
+  name: string;
+  pressures: string[];
+  options: string[];
+  expected: string;
+  rationale: string;
+}
+
+interface ScenarioResult {
+  scenario: string;
+  withSkill: {
+    choice: string;
+    compliant: boolean;
+    citedSections: string[];
+  };
+  withoutSkill: {
+    choice: string;
+    compliant: boolean;
+    rationalizations: string[];
+  };
+  metaTest?: {
+    question: string;
+    response: string;
+    improvement: string;
+  };
+}
+
+interface EvalReport {
+  skill: string;
+  evaluatedAt: string;
+  summary: {
+    total: number;
+    withSkillCompliant: number;
+    withoutSkillNonCompliant: number;
+    citationAccuracy: number;
+    newRationalizations: string[];
+  };
+  results: ScenarioResult[];
+}
+
+function discoverScenarios(skillName: string): PressureScenario[] {
+  const dir = join(ROOT, "test", "functional", skillName);
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) {
+    return [];
+  }
+
+  const scenarios: PressureScenario[] = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (entry.isFile() && entry.name.endsWith(".md")) {
+      const content = readFileSync(join(dir, entry.name), "utf8");
+      const parsed = parseScenario(content);
+      if (parsed) scenarios.push({ ...parsed, name: entry.name.replace(/\.md$/, "") });
+    }
+  }
+  return scenarios;
+}
+
+function parseScenario(content: string): Omit<PressureScenario, "name"> | null {
+  const pressuresMatch = content.match(/## Pressures\n([\s\S]*?)(?=\n## |$)/);
+  const optionsMatch = content.match(/## Options\n([\s\S]*?)(?=\n## |$)/);
+  const expectedMatch = content.match(/## Expected\n([\s\S]*?)(?=\n## |$)/);
+  const rationaleMatch = content.match(/## Rationale\n([\s\S]*?)(?=\n## |$)/);
+
+  if (!pressuresMatch || !optionsMatch || !expectedMatch) return null;
+
+  return {
+    pressures: pressuresMatch[1].trim().split("\n").filter((l) => l.startsWith("- ")).map((l) => l.slice(2).trim()),
+    options: optionsMatch[1].trim().split("\n").filter((l) => /^[A-C]\)/.test(l)).map((l) => l.trim()),
+    expected: expectedMatch[1].trim(),
+    rationale: rationaleMatch ? rationaleMatch[1].trim() : "",
+  };
+}
+
+function runScenario(skillName: string, scenario: PressureScenario): ScenarioResult {
+  // Skeleton: placeholder for LLM agent execution.
+  // Phase-3 will wire actual subagent calls via core/dag-executor.ts.
+  return {
+    scenario: scenario.name,
+    withSkill: {
+      choice: scenario.expected,
+      compliant: true,
+      citedSections: [],
+    },
+    withoutSkill: {
+      choice: "B",
+      compliant: false,
+      rationalizations: [],
+    },
+  };
+}
+
+function buildReport(skillName: string, results: ScenarioResult[]): EvalReport {
+  const withSkillCompliant = results.filter((r) => r.withSkill.compliant).length;
+  const withoutSkillNonCompliant = results.filter((r) => !r.withoutSkill.compliant).length;
+  const totalCitations = results.reduce((s, r) => s + r.withSkill.citedSections.length, 0);
+  const citationAccuracy = results.length > 0 ? totalCitations / results.length : 0;
+  const newRationalizations = results.flatMap((r) => r.withoutSkill.rationalizations);
+
+  return {
+    skill: skillName,
+    evaluatedAt: new Date().toISOString(),
+    summary: {
+      total: results.length,
+      withSkillCompliant,
+      withoutSkillNonCompliant,
+      citationAccuracy,
+      newRationalizations,
+    },
+    results,
+  };
+}
+
+function formatReport(report: EvalReport, asJson: boolean): string {
+  if (asJson) {
+    return JSON.stringify(report, null, 2);
+  }
+
+  const lines: string[] = [];
+  lines.push(`Functional eval: ${report.skill}`);
+  lines.push(`Evaluated at: ${report.evaluatedAt}`);
+  lines.push("");
+  lines.push(`Scenarios: ${report.summary.total}`);
+  lines.push(`With skill compliant: ${report.summary.withSkillCompliant}/${report.summary.total}`);
+  lines.push(`Without skill non-compliant: ${report.summary.withoutSkillNonCompliant}/${report.summary.total}`);
+  lines.push(`Citation accuracy: ${(report.summary.citationAccuracy * 100).toFixed(0)}%`);
+  if (report.summary.newRationalizations.length > 0) {
+    lines.push("");
+    lines.push("New rationalizations found:");
+    for (const r of report.summary.newRationalizations) {
+      lines.push(`  - ${r}`);
+    }
+  }
+  return lines.join("\n");
+}
+
+function discoverDisciplineSkills(): string[] {
+  // Skeleton: discover skills with type=discipline or discipline signals.
+  // For now, hardcode the known discipline skills.
+  return ["gxpm-tdd", "gxpm-triage"];
+}
+
+function usage() {
+  console.log(`gxpm-eval-functional — functional eval runner for discipline skills
+
+Usage:
+  gxpm-eval-functional run --all [--json]
+  gxpm-eval-functional run <skill-name> [--json]
+
+Commands:
+  run --all         Run all discipline skills
+  run <skill-name>  Run a specific skill
+`);
+}
+
+const args = process.argv.slice(2);
+const command = args[0];
+
+if (!command || command === "--help" || command === "-h") {
+  usage();
+  process.exit(0);
+}
+
+const asJson = args.includes("--json");
+
+if (command === "run") {
+  const target = args[1];
+  if (!target) {
+    console.error("Missing target. Use --all or a skill name.");
+    usage();
+    process.exit(1);
+  }
+
+  const skills = target === "--all" ? discoverDisciplineSkills() : [target];
+  const reports: EvalReport[] = [];
+
+  for (const skill of skills) {
+    const scenarios = discoverScenarios(skill);
+    if (scenarios.length === 0) {
+      console.error(`No scenarios found for ${skill} (expected test/functional/${skill}/*.md)`);
+      continue;
+    }
+    const results = scenarios.map((s) => runScenario(skill, s));
+    reports.push(buildReport(skill, results));
+  }
+
+  if (reports.length === 0) {
+    process.exit(1);
+  }
+
+  if (asJson) {
+    console.log(JSON.stringify({ evaluated: reports.length, reports }, null, 2));
+  } else {
+    for (const report of reports) {
+      console.log(formatReport(report, false));
+      console.log("");
+    }
+  }
+} else {
+  console.error(`Unknown command: ${command}`);
+  usage();
+  process.exit(1);
+}

package/scripts/eval.ts

@@ -0,0 +1,246 @@
+#!/usr/bin/env bun
+/**
+ * gxpm-eval — lightweight skill quality eval harness.
+ *
+ * Static analysis of SKILL.md files: structure, frontmatter, triggers, length.
+ * Future iterations can add LLM-based output quality scoring.
+ */
+
+import { readFileSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { discoverTemplates } from "./discover-skills";
+
+interface EvalResult {
+  skill: string;
+  score: number;
+  maxScore: number;
+  checks: EvalCheck[];
+  type: string;
+}
+
+interface EvalCheck {
+  name: string;
+  pass: boolean;
+  message: string;
+}
+
+const ROOT = join(import.meta.dir, "..");
+
+type SkillType = "discipline" | "technique" | "pattern" | "reference" | "unknown";
+
+function detectSkillType(content: string): SkillType {
+  // 1. Frontmatter type field
+  const typeMatch = content.match(/^type:\s*(.+)$/m);
+  if (typeMatch) {
+    const t = typeMatch[1].trim().toLowerCase();
+    if (["discipline", "technique", "pattern", "reference"].includes(t)) {
+      return t as SkillType;
+    }
+  }
+
+  // 2. Content heuristic: strong signals of a discipline skill
+  const disciplineSignals = [
+    /## The Iron Law/i,
+    /## Red Flags/i,
+    /## Common Rationalizations/i,
+    /\*\*No exceptions:\*\*/i,
+    /Violating the letter/i,
+  ];
+  if (disciplineSignals.some((re) => re.test(content))) {
+    return "discipline";
+  }
+
+  return "unknown";
+}
+
+function evaluateSkill(skillPath: string, content: string): EvalResult {
+  const checks: EvalCheck[] = [];
+  const lines = content.split("\n");
+  const skillType = detectSkillType(content);
+
+  // 1. Frontmatter exists
+  const hasFrontmatter = content.startsWith("---");
+  checks.push({
+    name: "frontmatter",
+    pass: hasFrontmatter,
+    message: hasFrontmatter ? "Has YAML frontmatter" : "Missing YAML frontmatter",
+  });
+
+  // 2. Name field
+  const nameMatch = content.match(/^name:\s*(.+)$/m);
+  const hasName = !!nameMatch && nameMatch[1].trim().length > 0;
+  checks.push({
+    name: "name",
+    pass: hasName,
+    message: hasName ? `Name: ${nameMatch![1].trim()}` : "Missing 'name' in frontmatter",
+  });
+
+  // 3. Description field
+  const descMatch = content.match(/^description:\s*(.+)$/m);
+  const desc = descMatch ? descMatch[1].trim() : "";
+  const descOk = desc.length >= 20 && desc.length <= 300;
+  checks.push({
+    name: "description",
+    pass: descOk,
+    message: descOk
+      ? `Description length: ${desc.length}`
+      : `Description length ${desc.length} (want 20-300)`,
+  });
+
+  // 4. Has trigger section
+  const hasTrigger = /## When to trigger|## Commands|## Trigger/i.test(content);
+  checks.push({
+    name: "triggers",
+    pass: hasTrigger,
+    message: hasTrigger ? "Has trigger/commands section" : "Missing trigger/commands section",
+  });
+
+  // 5. Reasonable length
+  const lineCount = lines.length;
+  const lengthOk = lineCount >= 10 && lineCount <= 1000;
+  checks.push({
+    name: "length",
+    pass: lengthOk,
+    message: lengthOk ? `${lineCount} lines` : `${lineCount} lines (want 10-1000)`,
+  });
+
+  // 6. Has references or read next
+  const hasReferences = /## Read Next|## References|## See Also/i.test(content);
+  checks.push({
+    name: "references",
+    pass: hasReferences,
+    message: hasReferences ? "Has references/read-next section" : "Missing references/read-next",
+  });
+
+  // 7–10. Discipline-specific structural checks
+  if (skillType === "discipline") {
+    const hasRationalization = /\|\s*Excuse\s*\|\s*Reality\s*\|/i.test(content);
+    checks.push({
+      name: "rationalization-table",
+      pass: hasRationalization,
+      message: hasRationalization
+        ? "Has rationalization table (Excuse / Reality)"
+        : "Missing rationalization table",
+    });
+
+    const hasRedFlags = /## Red Flags/i.test(content);
+    checks.push({
+      name: "red-flags",
+      pass: hasRedFlags,
+      message: hasRedFlags ? "Has Red Flags section" : "Missing Red Flags section",
+    });
+
+    const hasExplicitNegation = /\*\*No exceptions:\*\*/i.test(content);
+    checks.push({
+      name: "explicit-negation",
+      pass: hasExplicitNegation,
+      message: hasExplicitNegation
+        ? "Has explicit negation (No exceptions)"
+        : "Missing explicit negation clause",
+    });
+
+    const hasFoundationalPrinciple = /Violating the letter/i.test(content);
+    checks.push({
+      name: "foundational-principle",
+      pass: hasFoundationalPrinciple,
+      message: hasFoundationalPrinciple
+        ? "Has foundational principle"
+        : "Missing foundational principle (e.g. 'Violating the letter')",
+    });
+  }
+
+  const passCount = checks.filter((c) => c.pass).length;
+  const maxScore = checks.length * 10;
+  const score = passCount * 10;
+
+  return { skill: skillPath, score, maxScore, checks, type: skillType };
+}
+
+function listSkills(): string[] {
+  const templates = discoverTemplates(ROOT);
+  return templates.map((t) => t.name);
+}
+
+function runEval(skillName?: string): EvalResult[] {
+  const templates = discoverTemplates(ROOT);
+  const toEval = skillName
+    ? templates.filter((t) => t.name === skillName)
+    : templates;
+
+  return toEval.map((t) => {
+    const content = readFileSync(join(ROOT, t.tmpl.endsWith(".tmpl") ? t.output : t.tmpl), "utf8");
+    return evaluateSkill(t.name, content);
+  });
+}
+
+function formatReport(results: EvalResult[], asJson: boolean): string {
+  if (asJson) {
+    return JSON.stringify({ evaluated: results.length, results }, null, 2);
+  }
+
+  const lines: string[] = [];
+  lines.push(`Evaluated ${results.length} skill(s)`);
+  lines.push("");
+
+  for (const r of results) {
+    const pct = Math.round((r.score / r.maxScore) * 100);
+    const icon = pct >= 80 ? "✓" : pct >= 50 ? "~" : "✗";
+    lines.push(`${icon} ${r.skill}: ${r.score}/${r.maxScore} (${pct}%) [type: ${r.type}]`);
+    for (const c of r.checks) {
+      const cicon = c.pass ? "  ✓" : "  ✗";
+      lines.push(`${cicon} ${c.name}: ${c.message}`);
+    }
+    lines.push("");
+  }
+
+  const totalScore = results.reduce((s, r) => s + r.score, 0);
+  const totalMax = results.reduce((s, r) => s + r.maxScore, 0);
+  const totalPct = totalMax > 0 ? Math.round((totalScore / totalMax) * 100) : 0;
+  lines.push(`Overall: ${totalScore}/${totalMax} (${totalPct}%)`);
+
+  return lines.join("\n");
+}
+
+function usage() {
+  console.log(`gxpm-eval — skill quality eval harness
+
+Usage:
+  gxpm-eval list
+  gxpm-eval run [<skill-name>] [--json]
+
+Commands:
+  list              List all discoverable skills
+  run               Run eval on all skills or a specific skill
+`);
+}
+
+const args = process.argv.slice(2);
+const command = args[0];
+
+if (!command || command === "--help" || command === "-h") {
+  usage();
+  process.exit(0);
+}
+
+const asJson = args.includes("--json");
+
+if (command === "list") {
+  const skills = listSkills();
+  if (asJson) {
+    console.log(JSON.stringify({ skills }, null, 2));
+  } else {
+    console.log(skills.join("\n"));
+  }
+} else if (command === "run") {
+  const skillName = args[1]?.startsWith("-") ? undefined : args[1];
+  const results = runEval(skillName);
+  console.log(formatReport(results, asJson));
+  const totalScore = results.reduce((s, r) => s + r.score, 0);
+  const totalMax = results.reduce((s, r) => s + r.maxScore, 0);
+  const totalPct = totalMax > 0 ? Math.round((totalScore / totalMax) * 100) : 0;
+  if (totalPct < 50) process.exit(1);
+} else {
+  console.error(`Unknown command: ${command}`);
+  usage();
+  process.exit(1);
+}