@event4u/agent-config 7.3.0 → 7.5.0

package/src/scripts/check_token_regression.ts

@@ -0,0 +1,266 @@
+#!/usr/bin/env tsx
+/**
+ * Token-regression gate (token-saving Phase 0).
+ *
+ * Fails CI when a tracked always-loaded / projected token surface grows more
+ * than the tolerance (default 5%) over a committed baseline. This is the
+ * RELATIVE guard — distinct from `audit-tokens-budget` (absolute budgets) and
+ * from the cache-prefix byte-stability guard (Phase 5).
+ *
+ * Source of truth is `internal/bench/reports/projection-cost.json`, which is
+ * now produced with EXACT counts via the real tokenizer
+ * (`src/scripts/_lib/token_count.ts` → js-tiktoken cl100k_base). Run
+ * `task audit-tokens` first so the report reflects the working tree.
+ *
+ * Gated metrics (all `tokens_gpt`):
+ *   - eager_rule_load    — thin_projection.eager_gpt (the eager always-on load)
+ *   - thin_rule_load     — thin_projection.thin_gpt
+ *   - skill_descriptions — description_catalog.skills_core_source.tokens_gpt
+ *   - command_descriptions — description_catalog.commands_core_source.tokens_gpt
+ *   - mcp_schemas        — Σ mcp_tool_schemas[*].tokens_gpt
+ *
+ * A baseline captured under a different `token_method` (e.g. proxy → exact)
+ * is a legitimate reset, not a regression — the gate WARNS and asks for a
+ * `--update-baseline` rather than failing on the method switch alone.
+ *
+ * CLI:
+ *   ./scripts-run src/scripts/check_token_regression            # gate (read-only)
+ *   ./scripts-run src/scripts/check_token_regression --json     # machine output
+ *   ./scripts-run src/scripts/check_token_regression --tolerance 0.05
+ *   ./scripts-run src/scripts/check_token_regression --update-baseline  # record current as baseline
+ *
+ * Exit codes:
+ *   0 — within tolerance (or baseline written / warmup: no baseline yet)
+ *   1 — argument / file error
+ *   2 — regression: a metric exceeded baseline by more than the tolerance
+ */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const _HERE = fileURLToPath(import.meta.url);
+// REPO_ROOT = <repo>/src/scripts/<file> → up two.
+const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
+
+const PROJECTION_REPORT = path.join(
+  REPO_ROOT,
+  'internal/bench/reports/projection-cost.json',
+);
+const BASELINE_FILE = path.join(
+  REPO_ROOT,
+  'internal/bench/reports/token-baseline.json',
+);
+const DEFAULT_TOLERANCE = 0.05;
+
+type Json = Record<string, unknown>;
+
+interface Metrics {
+  token_method: string;
+  metrics: Record<string, number>;
+}
+
+function _num(obj: unknown, ...keys: string[]): number {
+  let cur: unknown = obj;
+  for (const k of keys) {
+    if (cur && typeof cur === 'object' && k in (cur as Json)) {
+      cur = (cur as Json)[k];
+    } else {
+      return 0;
+    }
+  }
+  return typeof cur === 'number' ? cur : 0;
+}
+
+/** Extract the flat gated-metric map from a projection-cost report. */
+function extract_metrics(report: Json): Metrics {
+  const mcp = (report['mcp_tool_schemas'] as Json) ?? {};
+  let mcp_total = 0;
+  for (const server of Object.values(mcp)) {
+    mcp_total += _num(server, 'tokens_gpt');
+  }
+  return {
+    token_method: String(report['token_method'] ?? 'unknown'),
+    metrics: {
+      eager_rule_load: _num(report, 'thin_projection', 'eager_gpt'),
+      thin_rule_load: _num(report, 'thin_projection', 'thin_gpt'),
+      skill_descriptions: _num(
+        report,
+        'description_catalog',
+        'skills_core_source',
+        'tokens_gpt',
+      ),
+      command_descriptions: _num(
+        report,
+        'description_catalog',
+        'commands_core_source',
+        'tokens_gpt',
+      ),
+      mcp_schemas: mcp_total,
+    },
+  };
+}
+
+function _readJson(file: string): Json | null {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf-8')) as Json;
+  } catch {
+    return null;
+  }
+}
+
+interface Row {
+  id: string;
+  current: number;
+  baseline: number;
+  pct: number; // signed fraction, e.g. 0.07 = +7%
+  regressed: boolean;
+}
+
+export interface GateResult {
+  status: 'ok' | 'regression' | 'warmup' | 'method-changed';
+  tolerance: number;
+  current_method: string;
+  baseline_method: string | null;
+  rows: Row[];
+}
+
+export function evaluate(
+  current: Metrics,
+  baseline: Metrics | null,
+  tolerance: number,
+): GateResult {
+  if (baseline === null) {
+    return {
+      status: 'warmup',
+      tolerance,
+      current_method: current.token_method,
+      baseline_method: null,
+      rows: [],
+    };
+  }
+  const method_changed = baseline.token_method !== current.token_method;
+  const rows: Row[] = [];
+  for (const [id, cur] of Object.entries(current.metrics)) {
+    const base = baseline.metrics[id] ?? 0;
+    const pct = base > 0 ? (cur - base) / base : 0;
+    rows.push({
+      id,
+      current: cur,
+      baseline: base,
+      pct,
+      // A method switch is a legitimate baseline reset, not a regression.
+      regressed: !method_changed && base > 0 && pct > tolerance,
+    });
+  }
+  rows.sort((a, b) => b.pct - a.pct);
+  const anyRegressed = rows.some((r) => r.regressed);
+  const status: GateResult['status'] = anyRegressed
+    ? 'regression'
+    : method_changed
+      ? 'method-changed'
+      : 'ok';
+  return {
+    status,
+    tolerance,
+    current_method: current.token_method,
+    baseline_method: baseline.token_method,
+    rows,
+  };
+}
+
+function _fmtPct(p: number): string {
+  const sign = p >= 0 ? '+' : '';
+  return `${sign}${(p * 100).toFixed(1)}%`;
+}
+
+function main(argv: string[]): number {
+  let tolerance = DEFAULT_TOLERANCE;
+  let asJson = false;
+  let updateBaseline = false;
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    if (a === '--json') asJson = true;
+    else if (a === '--update-baseline') updateBaseline = true;
+    else if (a === '--tolerance') {
+      const v = Number(argv[(i += 1)]);
+      if (!Number.isFinite(v) || v < 0) {
+        process.stderr.write(`error: invalid --tolerance\n`);
+        return 1;
+      }
+      tolerance = v;
+    } else if (a === '-h' || a === '--help') {
+      process.stdout.write(
+        'usage: check_token_regression [--json] [--tolerance F] [--update-baseline]\n',
+      );
+      return 0;
+    } else {
+      process.stderr.write(`error: unknown argument: ${a}\n`);
+      return 1;
+    }
+  }
+
+  const report = _readJson(PROJECTION_REPORT);
+  if (report === null) {
+    process.stderr.write(
+      `error: cannot read ${path.relative(REPO_ROOT, PROJECTION_REPORT)} — run \`task audit-tokens\` first\n`,
+    );
+    return 1;
+  }
+  const current = extract_metrics(report);
+
+  if (updateBaseline) {
+    const payload = JSON.stringify(current, null, 2) + '\n';
+    fs.writeFileSync(BASELINE_FILE, payload);
+    process.stdout.write(
+      `✅  token baseline written: ${path.relative(REPO_ROOT, BASELINE_FILE)} (method: ${current.token_method})\n`,
+    );
+    return 0;
+  }
+
+  const baselineRaw = _readJson(BASELINE_FILE);
+  const baseline: Metrics | null =
+    baselineRaw === null ? null : (baselineRaw as unknown as Metrics);
+  const result = evaluate(current, baseline, tolerance);
+
+  if (asJson) {
+    process.stdout.write(JSON.stringify(result, null, 2) + '\n');
+  } else if (result.status === 'warmup') {
+    process.stdout.write(
+      `⚠️  no token baseline yet — run \`./scripts-run src/scripts/check_token_regression --update-baseline\` to record one.\n`,
+    );
+  } else {
+    for (const r of result.rows) {
+      const mark = r.regressed ? '❌' : '✅';
+      process.stdout.write(
+        `${mark}  ${r.id}: ${r.current} vs baseline ${r.baseline} (${_fmtPct(r.pct)})\n`,
+      );
+    }
+    if (result.status === 'method-changed') {
+      process.stdout.write(
+        `⚠️  token_method changed (${result.baseline_method} → ${result.current_method}); ` +
+          `baseline reset is legitimate — run --update-baseline to re-anchor.\n`,
+      );
+    } else if (result.status === 'regression') {
+      process.stdout.write(
+        `❌  token regression: a projection grew >${(tolerance * 100).toFixed(0)}% over baseline.\n`,
+      );
+    } else {
+      process.stdout.write(
+        `✅  token surfaces within +${(tolerance * 100).toFixed(0)}% of baseline.\n`,
+      );
+    }
+  }
+
+  return result.status === 'regression' ? 2 : 0;
+}
+
+const _IS_MAIN =
+  process.argv[1] !== undefined &&
+  import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
+
+if (_IS_MAIN) {
+  process.exit(main(process.argv.slice(2)));
+}
+
+export { extract_metrics, main };

package/src/scripts/design_slop_rules.test.ts

@@ -0,0 +1,193 @@
+import { describe, expect, it } from "vitest";
+import { SLOP_RULES, type DesignContext } from "./design_slop_rules.js";
+import { scanFile } from "./lint_design_slop.js";
+
+// Context with no DESIGN.md → all gates OFF, every rule active.
+const NO_CTX: DesignContext = { raw: "", has: () => false };
+const ctxWith = (raw: string): DesignContext => ({
+  raw: raw.toLowerCase(),
+  has: (...ks: string[]) => ks.some((k) => raw.toLowerCase().includes(k.toLowerCase())),
+});
+
+/**
+ * Calibration corpus (Phase 4). One positive (must fire) + one negative (must be
+ * clean) per rule. `ext` drives the engine class. No untested tell — the guard
+ * below asserts every registry rule appears here.
+ */
+const FIXTURES: Record<string, { ext: string; positive: string; negative: string }> = {
+  "slop-v1-side-stripe": {
+    ext: "css",
+    positive: ".card { border-left: 4px solid #6c5ce7; padding: 16px; }",
+    negative: ".card { border-left: 1px solid #eee; padding: 16px; }",
+  },
+  "slop-v3-ghost-card": {
+    ext: "css",
+    positive: ".card { border: 1px solid #ddd; box-shadow: 0 8px 24px rgba(0,0,0,.1); }",
+    negative: ".card { box-shadow: 0 2px 4px rgba(0,0,0,.1); }",
+  },
+  "slop-v6-diagonal-stripes": {
+    ext: "css",
+    positive: ".bg { background: repeating-linear-gradient(45deg,#000,#000 10px,#111 10px,#111 20px); }",
+    negative: ".bg { background: linear-gradient(#fff,#eee); }",
+  },
+  "slop-c2-gradient-text": {
+    ext: "css",
+    positive: ".h { background: linear-gradient(90deg,#f00,#00f); -webkit-background-clip: text; color: transparent; }",
+    negative: ".h { color: #111; }",
+  },
+  "slop-c5-cream-palette": {
+    ext: "css",
+    positive: "body { background: #f5f1ea; } .a { color: #b08947; }",
+    negative: "body { background: #ffffff; } .a { color: #111111; }",
+  },
+  "slop-t4-eyebrow-overuse": {
+    ext: "html",
+    positive:
+      '<section><span class="uppercase">Features</span></section><section><span class="uppercase">Pricing</span></section><section><h2>Contact</h2></section>',
+    negative:
+      '<section><span class="uppercase">Features</span></section><section><h2>Pricing</h2></section><section><h2>Contact</h2></section>',
+  },
+  "slop-t6-crushed-tracking": {
+    ext: "css",
+    positive: ".display { letter-spacing: -0.05em; }",
+    negative: ".display { letter-spacing: -0.02em; }",
+  },
+  "slop-t7-default-fonts": {
+    ext: "css",
+    positive: "body { font-family: 'Inter', sans-serif; }",
+    negative: "body { font-family: 'Söhne', sans-serif; }",
+  },
+  "slop-l4-numbered-markers": {
+    ext: "html",
+    positive: "<div>01</div><div>02</div><div>03</div>",
+    negative: "<div>Discover</div><div>Build</div><div>Ship</div>",
+  },
+  "slop-l8-zindex-magic": {
+    ext: "css",
+    positive: ".modal { z-index: 9999; }",
+    negative: ".modal { z-index: 400; }",
+  },
+  "slop-m4-transition-all": {
+    ext: "css",
+    positive: ".x { transition: all 200ms ease-out; }",
+    negative: ".x { transition: transform 200ms ease-out; }",
+  },
+  "slop-m2-animate-layout": {
+    ext: "css",
+    positive: ".x { transition: width 200ms ease; }",
+    negative: ".x { transition: transform 200ms, opacity 200ms; }",
+  },
+  "slop-cp1-em-dash": {
+    ext: "md",
+    // ≥80 words is the rule floor; positive carries 4 em-dashes (well over the
+    // ~2-per-500 density), negative carries one across the same length.
+    positive:
+      "We build tools — fast, focused tools — that teams genuinely love using every single day of the working week. " +
+      "The product is calm and clear and useful, and people return to it daily because it respects their time — and their attention — and never wastes a single click of effort or a moment of their busy day, and that quiet reliability is the whole point of the thing we set out to make together, and the team keeps shipping the small improvements that matter most to the people who rely on it.",
+    negative:
+      "We build tools, fast and focused tools, that teams genuinely love using every single day of the working week. " +
+      "The product is calm and clear and useful, and people return to it daily because it respects their time and their attention and never wastes a single click of effort or a moment of their busy day — and that quiet reliability is the whole point of the thing we set out to make together, and the team keeps shipping the small improvements that matter most to the people who rely on it.",
+  },
+  "slop-cp2-buzzword": {
+    ext: "md",
+    positive: "We streamline and empower your robust, world-class workflow.",
+    negative: "We cut page load by 40% and remove three steps from checkout.",
+  },
+  "slop-lock-shape": {
+    ext: "css",
+    positive: ".a{border-radius:4px}.b{border-radius:8px}.c{border-radius:12px}.d{border-radius:20px}",
+    negative: ".a{border-radius:8px}.b{border-radius:12px}",
+  },
+  "slop-lock-colour": {
+    ext: "css",
+    positive: ".a{color:#e11d48}.b{background:#2563eb}.c{border-color:#16a34a}",
+    negative: ".a{color:#111827}.b{background:#6b7280}.c{border-color:#2563eb}",
+  },
+  "slop-l9-section-monotony": {
+    ext: "html",
+    // 8 identical media+text sections → 1 distinct layout family (< 4)
+    positive: '<section><div class="grid"><img src="x"><h2>A</h2><p>t</p></div></section>'.repeat(8),
+    // 8 sections, 4 distinct signatures → not monotonous
+    negative: (
+      '<section><div class="grid"><img src="x"><h2>A</h2></div></section>' +
+      "<section><h2>B</h2><p>t</p></section>" +
+      '<section><div class="flex"><p>c</p></div></section>' +
+      '<section><img src="y"></section>'
+    ).repeat(2),
+  },
+  "slop-l10-zigzag": {
+    ext: "html",
+    // 3 consecutive media+text two-column sections → run > 2
+    positive: '<section><div class="grid"><img src="x"><h2>A</h2><p>t</p></div></section>'.repeat(3),
+    // only 2 consecutive → within cap
+    negative: '<section><div class="grid"><img src="x"><h2>A</h2><p>t</p></div></section>'.repeat(2),
+  },
+};
+
+function findingsFor(ruleId: string, content: string, ext: string, ctx = NO_CTX) {
+  return scanFile(content, `fixture.${ext}`, ctx).filter((f) => f.rule === ruleId);
+}
+
+describe("design-slop rule registry", () => {
+  it("every rule has a unique id + a catalog id", () => {
+    const ids = SLOP_RULES.map((r) => r.id);
+    expect(new Set(ids).size).toBe(ids.length);
+    for (const r of SLOP_RULES) {
+      expect(r.id).toMatch(/^slop-/);
+      expect(r.catalogId).toMatch(/^[A-Z]+\d+$/);
+      expect(r.engines.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("no untested tell — every registry rule has a +/- fixture", () => {
+    for (const r of SLOP_RULES) {
+      expect(FIXTURES[r.id], `missing fixture for ${r.id}`).toBeDefined();
+    }
+    // and no orphan fixtures
+    const ids = new Set(SLOP_RULES.map((r) => r.id));
+    for (const k of Object.keys(FIXTURES)) {
+      expect(ids.has(k), `orphan fixture ${k}`).toBe(true);
+    }
+  });
+
+  describe("positive fixtures fire, negative fixtures are clean", () => {
+    for (const r of SLOP_RULES) {
+      const fx = FIXTURES[r.id];
+      if (!fx) continue;
+      it(`${r.id} (${r.catalogId})`, () => {
+        expect(findingsFor(r.id, fx.positive, fx.ext).length, "positive must fire").toBeGreaterThan(0);
+        expect(findingsFor(r.id, fx.negative, fx.ext).length, "negative must be clean").toBe(0);
+      });
+    }
+  });
+});
+
+describe("context gates suppress when DESIGN.md declares intent", () => {
+  it("t7 default-font is suppressed when DESIGN.md adopts Inter", () => {
+    const fx = FIXTURES["slop-t7-default-fonts"]!;
+    expect(findingsFor("slop-t7-default-fonts", fx.positive, "css", NO_CTX).length).toBeGreaterThan(0);
+    expect(
+      findingsFor("slop-t7-default-fonts", fx.positive, "css", ctxWith("Body font: Inter (chosen for variable metrics in a data-dense dashboard)")).length,
+    ).toBe(0);
+  });
+
+  it("c5 cream-palette is suppressed under a declared warm-neutral language", () => {
+    const fx = FIXTURES["slop-c5-cream-palette"]!;
+    expect(findingsFor("slop-c5-cream-palette", fx.positive, "css", NO_CTX).length).toBeGreaterThan(0);
+    expect(
+      findingsFor("slop-c5-cream-palette", fx.positive, "css", ctxWith("Palette: warm-neutral, first-party")).length,
+    ).toBe(0);
+  });
+});
+
+describe("inline-ignore", () => {
+  it("design-slop-disable-next-line suppresses the next line", () => {
+    const css = "/* design-slop-disable-next-line slop-m4-transition-all */\n.x { transition: all 200ms; }";
+    expect(findingsFor("slop-m4-transition-all", css, "css").length).toBe(0);
+  });
+
+  it("file-scope design-slop-disable suppresses the whole file", () => {
+    const css = "/* design-slop-disable slop-l8-zindex-magic */\n.a{z-index:9999}\n.b{z-index:999}";
+    expect(findingsFor("slop-l8-zindex-magic", css, "css").length).toBe(0);
+  });
+});