@event4u/agent-config 8.10.0 → 8.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +148 -0
- package/README.md +5 -5
- package/dist/agent-src/commands/challenge-me/vision.md +121 -32
- package/dist/agent-src/commands/challenge-me/with-docs.md +21 -8
- package/dist/agent-src/commands/commit/in-chunks.md +1 -1
- package/dist/agent-src/commands/commit.md +2 -1
- package/dist/agent-src/commands/cost/report.md +2 -1
- package/dist/agent-src/commands/feature/dev.md +4 -4
- package/dist/agent-src/commands/humanize.md +101 -0
- package/dist/agent-src/commands/jira-ticket.md +2 -2
- package/dist/agent-src/commands/memory/load.md +15 -0
- package/dist/agent-src/commands/optimize/project.md +12 -0
- package/dist/agent-src/commands/roadmap/create.md +12 -11
- package/dist/agent-src/commands/tests/create.md +1 -1
- package/dist/agent-src/commands/worktree/cleanup.md +22 -17
- package/dist/agent-src/commands/worktree/create.md +3 -1
- package/dist/agent-src/commands/worktree/status.md +12 -0
- package/dist/agent-src/contexts/execution/autonomy-mechanics.md +66 -0
- package/dist/agent-src/contexts/execution/orchestration-benchmark-gate.md +1 -0
- package/dist/agent-src/contexts/execution/orchestration-telemetry.md +47 -3
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +14 -0
- package/dist/agent-src/contexts/execution/roadmap-writing-source-derived.md +65 -0
- package/dist/agent-src/contexts/execution/subagent-modes-detail.md +84 -0
- package/dist/agent-src/rules/autonomous-execution.md +7 -37
- package/dist/agent-src/rules/broken-access-control.md +7 -45
- package/dist/agent-src/rules/code-comment-discipline.md +148 -0
- package/dist/agent-src/rules/context-hygiene.md +14 -0
- package/dist/agent-src/rules/delegation-policy.md +10 -8
- package/dist/agent-src/rules/design-fidelity.md +4 -0
- package/dist/agent-src/rules/git-history-discipline.md +4 -79
- package/dist/agent-src/rules/legal-safety-floor.md +11 -55
- package/dist/agent-src/rules/roadmap-progress-sync.md +5 -56
- package/dist/agent-src/rules/think-before-action.md +10 -0
- package/dist/agent-src/rules/verify-before-complete.md +12 -0
- package/dist/agent-src/skills/agent-docs-writing/SKILL.md +13 -12
- package/dist/agent-src/skills/ai-council/SKILL.md +19 -9
- package/dist/agent-src/skills/api-design/SKILL.md +7 -5
- package/dist/agent-src/skills/api-testing/SKILL.md +1 -1
- package/dist/agent-src/skills/authz-review/SKILL.md +59 -2
- package/dist/agent-src/skills/blast-radius-analyzer/SKILL.md +1 -1
- package/dist/agent-src/skills/bug-analyzer/SKILL.md +19 -16
- package/dist/agent-src/skills/code-refactoring/SKILL.md +9 -9
- package/dist/agent-src/skills/content-funnel-design/SKILL.md +2 -0
- package/dist/agent-src/skills/data-flow-mapper/SKILL.md +1 -1
- package/dist/agent-src/skills/defense-in-depth/SKILL.md +2 -2
- package/dist/agent-src/skills/dependency-upgrade/SKILL.md +7 -7
- package/dist/agent-src/skills/design-intelligence/SKILL.md +8 -8
- package/dist/agent-src/skills/design-review/SKILL.md +23 -2
- package/dist/agent-src/skills/doc-coauthoring/SKILL.md +2 -0
- package/dist/agent-src/skills/editorial-calendar/SKILL.md +2 -0
- package/dist/agent-src/skills/fe-design/SKILL.md +7 -9
- package/dist/agent-src/skills/git-workflow/SKILL.md +86 -0
- package/dist/agent-src/skills/humanizer/SKILL.md +147 -0
- package/dist/agent-src/skills/humanizer/data/patterns.md +190 -0
- package/dist/agent-src/skills/humanizer/evals/triggers.json +17 -0
- package/dist/agent-src/skills/laravel/SKILL.md +14 -1
- package/dist/agent-src/skills/launch-readiness/SKILL.md +20 -19
- package/dist/agent-src/skills/legal-practice-profile/SKILL.md +126 -0
- package/dist/agent-src/skills/logging-monitoring/SKILL.md +1 -1
- package/dist/agent-src/skills/mcp/SKILL.md +1 -1
- package/dist/agent-src/skills/memory-consolidation/SKILL.md +14 -1
- package/dist/agent-src/skills/migration-architect/SKILL.md +2 -1
- package/dist/agent-src/skills/performance/SKILL.md +1 -1
- package/dist/agent-src/skills/php-debugging/SKILL.md +17 -0
- package/dist/agent-src/skills/readme-writing/SKILL.md +2 -0
- package/dist/agent-src/skills/readme-writing-package/SKILL.md +2 -0
- package/dist/agent-src/skills/release-comms/SKILL.md +2 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +22 -51
- package/dist/agent-src/skills/security-audit/SKILL.md +6 -9
- package/dist/agent-src/skills/sentry-integration/SKILL.md +2 -2
- package/dist/agent-src/skills/skill-improvement-pipeline/SKILL.md +8 -0
- package/dist/agent-src/skills/source-discovery/SKILL.md +26 -24
- package/dist/agent-src/skills/subagent-orchestration/SKILL.md +55 -46
- package/dist/agent-src/skills/subagent-orchestration/evals/evals.json +42 -0
- package/dist/agent-src/skills/systematic-debugging/SKILL.md +10 -0
- package/dist/agent-src/skills/using-git-worktrees/SKILL.md +10 -0
- package/dist/agent-src/skills/worktree-lifecycle/SKILL.md +27 -20
- package/dist/agent-src/skills/worktree-lifecycle/evals/evals.json +38 -0
- package/dist/agent-src/templates/agent-settings.md +3 -2
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +15 -1
- package/dist/agent-src/templates/roadmaps.md +14 -0
- package/dist/cli/commands/recordTriggerEval.js +4 -14
- package/dist/cli/commands/recordTriggerEval.js.map +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +147 -78
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +6 -6
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +13 -10
- package/dist/discovery/trust-report.md +3 -3
- package/dist/discovery/workspaces.json +14 -9
- package/dist/install/install.mjs +12 -7
- package/dist/mcp/registry-manifest.json +4 -4
- package/dist/router.json +1 -1
- package/dist/scripts/_lib/claude_settings_hooks.js +8 -0
- package/dist/scripts/_lib/claude_settings_hooks.js.map +1 -1
- package/dist/scripts/_lib/trigger_eval_floors.js +24 -0
- package/dist/scripts/_lib/trigger_eval_floors.js.map +1 -0
- package/dist/scripts/condense.js +12 -2
- package/dist/scripts/condense.js.map +1 -1
- package/dist/server/schemas/settings.js +5 -2
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/CLAIMS.md +24 -3
- package/docs/architecture.md +3 -3
- package/docs/benchmark.md +51 -0
- package/docs/catalog.md +8 -5
- package/docs/command-flows.md +2 -1
- package/docs/contracts/ai-council-config.md +92 -0
- package/docs/contracts/command-clusters.md +1 -0
- package/docs/contracts/evidence-based-pruning.md +1 -0
- package/docs/contracts/hook-architecture-v1.md +25 -0
- package/docs/contracts/kernel-membership.md +35 -0
- package/docs/contracts/measurement-baseline.md +1 -0
- package/docs/contracts/release-sizing.md +22 -0
- package/docs/contracts/write-engine.md +38 -1
- package/docs/customization.md +12 -0
- package/docs/decisions/ADR-002-kernel-bucket-overrides.md +11 -0
- package/docs/decisions/ADR-003-flat-cluster-subs-and-colon-syntax.md +11 -0
- package/docs/decisions/ADR-006-skill-tools-python-pilot.md +1 -1
- package/docs/decisions/ADR-028-root-layout.md +1 -1
- package/docs/decisions/ADR-038-canonical-settings-path.md +10 -0
- package/docs/decisions/ADR-092-defer-command-tier-alias-removal.md +29 -5
- package/docs/decisions/ADR-093-ai-council-config-user-global.md +1 -1
- package/docs/decisions/ADR-100-global-knowledge-card-sharing.md +7 -0
- package/docs/decisions/ADR-103-global-knowledge-default-off-until-measured.md +2 -2
- package/docs/decisions/ADR-105-automatic-subagent-orchestration.md +1 -1
- package/docs/decisions/ADR-114-commit-policy-iron-law-override.md +11 -0
- package/docs/decisions/ADR-116-memory-tripwire-activation-path.md +11 -1
- package/docs/decisions/ADR-117-subagents-auto-default-on.md +2 -1
- package/docs/decisions/ADR-118-loop-engineering-boundaries.md +162 -0
- package/docs/decisions/ADR-119-global-knowledge-default-on.md +104 -0
- package/docs/decisions/ADR-200-python-to-typescript-migration.md +11 -1
- package/docs/decisions/ADR-rule-kernel-and-router.md +11 -0
- package/docs/decisions/INDEX.md +11 -9
- package/docs/design/cross-model-parity-eval.md +82 -0
- package/docs/design/length-neutral-judge-rerun.md +92 -0
- package/docs/featured-skills.md +2 -2
- package/docs/getting-started-by-role.md +1 -1
- package/docs/getting-started.md +2 -2
- package/docs/governance-advantage.md +1 -1
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +80 -5
- package/docs/guidelines/code-clarity.md +106 -1
- package/docs/guidelines/design-antipatterns.md +20 -0
- package/docs/guidelines/php/php-coding-patterns.md +12 -2
- package/docs/proof.md +20 -14
- package/docs/wedge/production-validator/production-validator.md +27 -0
- package/package.json +1 -1
- package/src/config/agent-settings.template.yml +29 -9
- package/src/scripts/_cli/cmd_doctor.ts +34 -1
- package/src/scripts/_lib/auto_dispatch.corpus.test.ts +87 -0
- package/src/scripts/_lib/claude_settings_hooks.ts +8 -0
- package/src/scripts/_lib/knowledge_global.ts +6 -2
- package/src/scripts/_lib/knowledge_global_redaction.ts +42 -4
- package/src/scripts/_lib/lexical_index.ts +15 -2
- package/src/scripts/_lib/orchestration_record.ts +14 -0
- package/src/scripts/_lib/trigger_eval_floors.ts +31 -0
- package/src/scripts/ai_council/chairman.ts +90 -0
- package/src/scripts/ai_council/clients.ts +18 -3
- package/src/scripts/ai_council/config.ts +157 -0
- package/src/scripts/ai_council/debate_gates.ts +89 -0
- package/src/scripts/ai_council/orchestrator.ts +65 -2
- package/src/scripts/ai_council/prompts.ts +121 -11
- package/src/scripts/ai_council/replay.ts +17 -0
- package/src/scripts/ai_council/stance_tally.ts +184 -0
- package/src/scripts/ai_tells_rules.ts +433 -0
- package/src/scripts/bench_humanizer_eval.ts +240 -0
- package/src/scripts/build_discovery_manifest.ts +6 -0
- package/src/scripts/build_proof.ts +21 -9
- package/src/scripts/check_claims.ts +40 -0
- package/src/scripts/check_rule_invariants.ts +10 -3
- package/src/scripts/condense.ts +16 -2
- package/src/scripts/consumer_matrix.ts +41 -2
- package/src/scripts/council_cli.ts +147 -0
- package/src/scripts/detect_ai_tells.ts +312 -0
- package/src/scripts/install.ts +12 -0
- package/src/scripts/knowledge_global_cli.ts +1 -1
- package/src/scripts/lint_framework_leakage.ts +88 -7
- package/src/scripts/lint_framework_leakage_allowlist.json +59 -572
- package/src/scripts/lint_knowledge_scale.ts +30 -0
- package/src/scripts/lint_originality_shingles.ts +210 -0
- package/src/scripts/orchestration_record.ts +1 -0
- package/src/scripts/schemas/agent-settings.schema.json +11 -0
- package/src/scripts/self_review_gate.ts +41 -15
- package/src/scripts/skill_eval_coverage.ts +71 -3
- package/src/scripts/skill_linter.ts +1 -0
- package/src/scripts/trigger_eval_rotation.ts +344 -0
- package/src/scripts/update_counts.ts +5 -17
- package/src/scripts/worktree_cleanup_check.ts +263 -0
- package/src/server/schemas/settings.ts +8 -3
- package/src/scripts/analysis_freshness.ts +0 -362
- package/src/scripts/export_replay_corpus.ts +0 -208
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* bench_humanizer_eval — paired eval for the humanizer feature.
|
|
4
|
+
*
|
|
5
|
+
* Two metrics over the before/after corpus in `tests/fixtures/ai-tells/`:
|
|
6
|
+
*
|
|
7
|
+
* 1. OBJECTIVE (free, deterministic): AI-tell reduction per pair via
|
|
8
|
+
* `detect_ai_tells.ts` — hard hits, cluster score/500w, dash density/500w.
|
|
9
|
+
* 2. BLIND PREFERENCE (--judge, billable): an LLM judge sees each pair in
|
|
10
|
+
* randomized A/B order (deterministic LCG seed per pair) and picks which
|
|
11
|
+
* text reads more like a skilled human writer. LENGTH-CONTROLLED: pairs
|
|
12
|
+
* whose word counts differ by more than ±25% are excluded from the
|
|
13
|
+
* preference aggregate (the token program's verbosity-bias finding makes
|
|
14
|
+
* uncontrolled pairs unusable) — they still count for the objective metric.
|
|
15
|
+
*
|
|
16
|
+
* Reports follow docs/benchmarks.md naming:
|
|
17
|
+
* internal/bench/reports/humanizer-v1.{json,md} (canonical pointer)
|
|
18
|
+
* internal/bench/reports/<ISO>-humanizer-v1.{json,md} (immutable trail)
|
|
19
|
+
*
|
|
20
|
+
* Claims discipline: this eval backs `claim:humanizer-tell-reduction` in
|
|
21
|
+
* docs/CLAIMS.md. It measures OUR pattern counts and a blind preference —
|
|
22
|
+
* never third-party "AI detector" outcomes; that claim class stays banned.
|
|
23
|
+
*
|
|
24
|
+
* Usage:
|
|
25
|
+
* npx tsx src/scripts/bench_humanizer_eval.ts # objective only (free)
|
|
26
|
+
* npx tsx src/scripts/bench_humanizer_eval.ts --judge --confirm-spend # + blind preference (billable)
|
|
27
|
+
*
|
|
28
|
+
* `--judge` without `--confirm-spend` estimates the billable call count and
|
|
29
|
+
* halts (exit 2) — the spend is opt-in, never fired implicitly.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
|
|
33
|
+
import { join } from "node:path";
|
|
34
|
+
import { analyzeText, type TellReport } from "./detect_ai_tells.js";
|
|
35
|
+
|
|
36
|
+
const ROOT = process.cwd();
|
|
37
|
+
const FIXTURES = join(ROOT, "tests", "fixtures", "ai-tells");
|
|
38
|
+
const REPORT_DIR = join(ROOT, "internal", "bench", "reports");
|
|
39
|
+
const LENGTH_TOLERANCE = 0.25;
|
|
40
|
+
|
|
41
|
+
interface PairResult {
|
|
42
|
+
name: string;
|
|
43
|
+
language: "en" | "de";
|
|
44
|
+
words_before: number;
|
|
45
|
+
words_after: number;
|
|
46
|
+
length_ratio: number;
|
|
47
|
+
length_controlled: boolean;
|
|
48
|
+
before: Pick<TellReport, "hard_total" | "cluster_score_per_500" | "dash_density_per_500">;
|
|
49
|
+
after: Pick<TellReport, "hard_total" | "cluster_score_per_500" | "dash_density_per_500">;
|
|
50
|
+
judge?: { preferred: "after" | "before"; order: "after-first" | "before-first" };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function slim(r: TellReport) {
|
|
54
|
+
return {
|
|
55
|
+
hard_total: r.hard_total,
|
|
56
|
+
cluster_score_per_500: r.cluster_score_per_500,
|
|
57
|
+
dash_density_per_500: r.dash_density_per_500,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Deterministic per-pair coin flip (LCG) so re-runs keep the same A/B order. */
|
|
62
|
+
function coin(seed: number): boolean {
|
|
63
|
+
return ((seed * 1103515245 + 12345) >>> 16) % 2 === 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function loadPairs(): Array<{ name: string; language: "en" | "de"; before: string; after: string }> {
|
|
67
|
+
const out: Array<{ name: string; language: "en" | "de"; before: string; after: string }> = [];
|
|
68
|
+
for (const language of ["en", "de"] as const) {
|
|
69
|
+
const dir = join(FIXTURES, language);
|
|
70
|
+
let names: string[] = [];
|
|
71
|
+
try {
|
|
72
|
+
names = readdirSync(dir)
|
|
73
|
+
.filter((f) => f.endsWith(".before.md"))
|
|
74
|
+
.map((f) => f.replace(/\.before\.md$/, ""))
|
|
75
|
+
.sort();
|
|
76
|
+
} catch {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
for (const name of names) {
|
|
80
|
+
out.push({
|
|
81
|
+
name,
|
|
82
|
+
language,
|
|
83
|
+
before: readFileSync(join(dir, `${name}.before.md`), "utf8"),
|
|
84
|
+
after: readFileSync(join(dir, `${name}.after.md`), "utf8"),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function judgePair(
|
|
92
|
+
pair: { name: string; before: string; after: string },
|
|
93
|
+
index: number,
|
|
94
|
+
): Promise<{ preferred: "after" | "before"; order: "after-first" | "before-first" }> {
|
|
95
|
+
const { AnthropicClient, load_anthropic_key } = await import("./ai_council/clients.js");
|
|
96
|
+
const client = new AnthropicClient({ api_key: load_anthropic_key() });
|
|
97
|
+
const afterFirst = coin(index + 1);
|
|
98
|
+
const [a, b] = afterFirst ? [pair.after, pair.before] : [pair.before, pair.after];
|
|
99
|
+
const system =
|
|
100
|
+
"You judge writing quality blind. You will see two texts, A and B, covering the same content. " +
|
|
101
|
+
"Pick the one that reads more like it was written by a skilled human writer for a real audience. " +
|
|
102
|
+
"Judge prose quality only — not length, not topic. Reply with exactly one character: A or B.";
|
|
103
|
+
const user = `Text A:\n\n${a}\n\n---\n\nText B:\n\n${b}\n\nWhich reads more like a skilled human writer? Reply A or B.`;
|
|
104
|
+
const resp = client.ask(system, user, 8);
|
|
105
|
+
const text = (resp.text ?? "").trim().toUpperCase();
|
|
106
|
+
const pickedA = text.startsWith("A");
|
|
107
|
+
const preferred = pickedA === afterFirst ? "after" : "before";
|
|
108
|
+
return { preferred, order: afterFirst ? "after-first" : "before-first" };
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function main(): Promise<void> {
|
|
112
|
+
const runJudge = process.argv.includes("--judge");
|
|
113
|
+
const confirmSpend = process.argv.includes("--confirm-spend");
|
|
114
|
+
const pairs = loadPairs();
|
|
115
|
+
if (pairs.length < 20) {
|
|
116
|
+
console.error(`corpus too small: ${pairs.length} pairs (< 20)`);
|
|
117
|
+
process.exit(2);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Spend gate: --judge fires one billable API call per length-controlled
|
|
121
|
+
// pair (via the council client's curl transport). Estimate + halt unless
|
|
122
|
+
// --confirm-spend is passed — mirrors council:run's explicit-confirm model,
|
|
123
|
+
// never an interactive prompt. The objective-only default path is free.
|
|
124
|
+
if (runJudge && !confirmSpend) {
|
|
125
|
+
const controlledCount = pairs.filter((p) => {
|
|
126
|
+
const b = analyzeText(p.before, p.language).words;
|
|
127
|
+
const a = analyzeText(p.after, p.language).words;
|
|
128
|
+
return Math.abs(1 - a / b) <= LENGTH_TOLERANCE;
|
|
129
|
+
}).length;
|
|
130
|
+
process.stderr.write(
|
|
131
|
+
`--judge makes ${controlledCount} billable API call(s) (one per length-controlled pair). ` +
|
|
132
|
+
`Re-run with --confirm-spend to authorize the spend. ` +
|
|
133
|
+
`The objective-only run (no --judge) is free.\n`,
|
|
134
|
+
);
|
|
135
|
+
process.exit(2);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const results: PairResult[] = [];
|
|
139
|
+
for (let i = 0; i < pairs.length; i++) {
|
|
140
|
+
const p = pairs[i]!;
|
|
141
|
+
const before = analyzeText(p.before, p.language);
|
|
142
|
+
const after = analyzeText(p.after, p.language);
|
|
143
|
+
const ratio = after.words / before.words;
|
|
144
|
+
const controlled = Math.abs(1 - ratio) <= LENGTH_TOLERANCE;
|
|
145
|
+
const row: PairResult = {
|
|
146
|
+
name: p.name,
|
|
147
|
+
language: p.language,
|
|
148
|
+
words_before: before.words,
|
|
149
|
+
words_after: after.words,
|
|
150
|
+
length_ratio: Math.round(ratio * 100) / 100,
|
|
151
|
+
length_controlled: controlled,
|
|
152
|
+
before: slim(before),
|
|
153
|
+
after: slim(after),
|
|
154
|
+
};
|
|
155
|
+
if (runJudge && controlled) {
|
|
156
|
+
row.judge = await judgePair(p, i);
|
|
157
|
+
process.stdout.write(` judged ${p.name}: prefers ${row.judge.preferred} (${row.judge.order})\n`);
|
|
158
|
+
}
|
|
159
|
+
results.push(row);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const mean = (xs: number[]) =>
|
|
163
|
+
Math.round((xs.reduce((s, x) => s + x, 0) / Math.max(xs.length, 1)) * 100) / 100;
|
|
164
|
+
const agg = {
|
|
165
|
+
pairs: results.length,
|
|
166
|
+
length_controlled_pairs: results.filter((r) => r.length_controlled).length,
|
|
167
|
+
mean_hard_before: mean(results.map((r) => r.before.hard_total)),
|
|
168
|
+
mean_hard_after: mean(results.map((r) => r.after.hard_total)),
|
|
169
|
+
mean_cluster_before: mean(results.map((r) => r.before.cluster_score_per_500)),
|
|
170
|
+
mean_cluster_after: mean(results.map((r) => r.after.cluster_score_per_500)),
|
|
171
|
+
mean_dash_before: mean(results.map((r) => r.before.dash_density_per_500)),
|
|
172
|
+
mean_dash_after: mean(results.map((r) => r.after.dash_density_per_500)),
|
|
173
|
+
judge_model: undefined as string | undefined,
|
|
174
|
+
judged_pairs: 0,
|
|
175
|
+
prefers_after: 0,
|
|
176
|
+
prefers_before: 0,
|
|
177
|
+
};
|
|
178
|
+
const judged = results.filter((r) => r.judge);
|
|
179
|
+
agg.judged_pairs = judged.length;
|
|
180
|
+
agg.prefers_after = judged.filter((r) => r.judge!.preferred === "after").length;
|
|
181
|
+
agg.prefers_before = judged.filter((r) => r.judge!.preferred === "before").length;
|
|
182
|
+
if (runJudge) {
|
|
183
|
+
const { DEFAULT_ANTHROPIC_MODEL } = await import("./ai_council/clients.js");
|
|
184
|
+
agg.judge_model = DEFAULT_ANTHROPIC_MODEL as string;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const iso = new Date().toISOString().replace(/\.\d+Z$/, "Z").replace(/:/g, "-");
|
|
188
|
+
const payload = { generated: iso, corpus: "tests/fixtures/ai-tells", aggregate: agg, results };
|
|
189
|
+
|
|
190
|
+
const md = [
|
|
191
|
+
"# Humanizer paired eval — v1",
|
|
192
|
+
"",
|
|
193
|
+
`> Generated ${iso} · corpus \`tests/fixtures/ai-tells\` (${agg.pairs} before/after pairs, ` +
|
|
194
|
+
`${agg.length_controlled_pairs} length-controlled ±${LENGTH_TOLERANCE * 100}%).`,
|
|
195
|
+
"",
|
|
196
|
+
"## Objective — AI-tell reduction (deterministic)",
|
|
197
|
+
"",
|
|
198
|
+
"| Metric (mean) | Before | After |",
|
|
199
|
+
"|---|---|---|",
|
|
200
|
+
`| Hard hits | ${agg.mean_hard_before} | ${agg.mean_hard_after} |`,
|
|
201
|
+
`| Cluster score /500w | ${agg.mean_cluster_before} | ${agg.mean_cluster_after} |`,
|
|
202
|
+
`| Dash density /500w | ${agg.mean_dash_before} | ${agg.mean_dash_after} |`,
|
|
203
|
+
"",
|
|
204
|
+
"## Blind preference (length-controlled)",
|
|
205
|
+
"",
|
|
206
|
+
agg.judged_pairs > 0
|
|
207
|
+
? `Judge ${agg.judge_model}: prefers the humanized text in **${agg.prefers_after}/${agg.judged_pairs}** ` +
|
|
208
|
+
`pairs (randomized A/B order, deterministic seed). An honest null here keeps the detector as a ` +
|
|
209
|
+
`hygiene gate; the claim ledger only carries what this table shows.`
|
|
210
|
+
: "_Not run (objective-only invocation)._",
|
|
211
|
+
"",
|
|
212
|
+
"## Scope note",
|
|
213
|
+
"",
|
|
214
|
+
"This eval measures the package's own pattern counts and a blind prose-quality preference.",
|
|
215
|
+
'It never measures third-party "AI detector" outcomes — that claim class is banned',
|
|
216
|
+
"(unfalsifiable from our side; see roadmap non-goals).",
|
|
217
|
+
"",
|
|
218
|
+
"## Open question — real-draft lift is unmeasured",
|
|
219
|
+
"",
|
|
220
|
+
"The `before` fixtures were **deliberately tell-seeded**, so a perfect score",
|
|
221
|
+
"measures seeded-tell removal on a self-constructed corpus — NOT that real",
|
|
222
|
+
"ghostwriter drafts get better. Real-world lift stays **unmeasured** until",
|
|
223
|
+
"write-engine step 4b has processed real `/ghostwriter:write` drafts and those",
|
|
224
|
+
"have been paired-evaluated (the `road-to-humanizer-hardening` live-usage",
|
|
225
|
+
"blocker). The claim ledger is scoped to \"on the fixture corpus\" accordingly.",
|
|
226
|
+
"",
|
|
227
|
+
].join("\n");
|
|
228
|
+
|
|
229
|
+
mkdirSync(REPORT_DIR, { recursive: true });
|
|
230
|
+
writeFileSync(join(REPORT_DIR, "humanizer-v1.json"), JSON.stringify(payload, null, 2));
|
|
231
|
+
writeFileSync(join(REPORT_DIR, "humanizer-v1.md"), md);
|
|
232
|
+
writeFileSync(join(REPORT_DIR, `${iso}-humanizer-v1.json`), JSON.stringify(payload, null, 2));
|
|
233
|
+
writeFileSync(join(REPORT_DIR, `${iso}-humanizer-v1.md`), md);
|
|
234
|
+
process.stdout.write(md + "\n");
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
main().catch((err) => {
|
|
238
|
+
console.error(String(err));
|
|
239
|
+
process.exit(2);
|
|
240
|
+
});
|
|
@@ -419,6 +419,12 @@ function* _iter_artefacts(): Generator<[string, string]> {
|
|
|
419
419
|
// `_classify` subagent branch derives the manifest payload (default-off,
|
|
420
420
|
// packs:[]) — static projection only, never runtime-dispatched.
|
|
421
421
|
for (const p of _collect('subagents', '*.md')) {
|
|
422
|
+
// `_`-prefixed files are injected partials (e.g. _prompt-defense.md),
|
|
423
|
+
// not projectable agents — never discovery artefacts. Mirrors the
|
|
424
|
+
// same filter in condense.generate_claude_subagents.
|
|
425
|
+
if (path.basename(p).startsWith('_')) {
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
422
428
|
yield [p, 'subagent'];
|
|
423
429
|
}
|
|
424
430
|
}
|
|
@@ -27,6 +27,7 @@ import { REPO, LEDGER_REL, load_ledger, pointer_unresolved } from './check_claim
|
|
|
27
27
|
import { collectSkillGaps } from './check_skill_gaps.js';
|
|
28
28
|
import { loadRows as loadComparisonRows } from './check_comparison.js';
|
|
29
29
|
import { computeStatus as domainSoundnessStatus } from './domain_soundness_status.js';
|
|
30
|
+
import { computeCoverage } from './skill_eval_coverage.js';
|
|
30
31
|
|
|
31
32
|
const _FILE = fileURLToPath(import.meta.url);
|
|
32
33
|
const OUT_REL = 'docs/proof.md';
|
|
@@ -108,16 +109,27 @@ function render(): string {
|
|
|
108
109
|
L.push('Benchmark results are published under `docs/` including measured nulls.');
|
|
109
110
|
}
|
|
110
111
|
L.push('');
|
|
112
|
+
const cov = computeCoverage();
|
|
113
|
+
const prio = cov.tiers.priority;
|
|
114
|
+
const other = cov.tiers.other;
|
|
115
|
+
const prioLine =
|
|
116
|
+
prio.covered === prio.total
|
|
117
|
+
? `\`rich\` + routers) are **fully covered (${prio.covered} of ${prio.total})**, the long tail`
|
|
118
|
+
: `\`rich\` + routers, **${prio.covered} of ${prio.total}**) are partially covered; the long tail`;
|
|
111
119
|
L.push('**Behavioural-eval coverage — the honest baseline.** Skill *quality* is only');
|
|
112
|
-
L.push(
|
|
113
|
-
L.push('`evals.json
|
|
114
|
-
L.push(
|
|
115
|
-
L.push(
|
|
116
|
-
L.push(
|
|
117
|
-
L.push('
|
|
118
|
-
L.push('
|
|
119
|
-
L.push('
|
|
120
|
-
L.push('
|
|
120
|
+
L.push(`as good as its measurement. Today **${cov.overall.covered} of ${cov.overall.total}** skills carry a behavioural`);
|
|
121
|
+
L.push('`evals.json`; the highest-traffic / highest-cost tiers (default-surface +');
|
|
122
|
+
L.push(prioLine);
|
|
123
|
+
L.push(`(${other.covered} of ${other.total}) is not. We publish that gap rather than imply`);
|
|
124
|
+
L.push(`"${cov.overall.total} evaluated skills": coverage is measured per tier`);
|
|
125
|
+
L.push('(`./scripts-run src/scripts/skill_eval_coverage`), **CI-ratcheted so it can');
|
|
126
|
+
L.push('only rise**, and the priority tiers carry a hard **tier floor**: every');
|
|
127
|
+
L.push('rich / default-surface / router skill MUST have a behavioural eval or an');
|
|
128
|
+
L.push('explicit exemption-with-reason in `internal/evals/tier-floor-exemptions.json`');
|
|
129
|
+
L.push('— no silent exemptions (the bright line deliberately replaces a');
|
|
130
|
+
L.push('weighted-coverage score). Authoring long-tail evals is gated on per-case');
|
|
131
|
+
L.push('human ratification (a generated assertion that checks the wrong property is');
|
|
132
|
+
L.push('worse than none), so the number grows deliberately, not overnight.');
|
|
121
133
|
L.push('');
|
|
122
134
|
const ds = domainSoundnessStatus();
|
|
123
135
|
L.push('**Non-coding domain soundness — scoped, not proven.** The `finance` /');
|
|
@@ -42,6 +42,13 @@ const SURFACE_ROOTS = ['README.md', 'docs'];
|
|
|
42
42
|
const CLAIM_MARKER = /<!--\s*claim:([A-Za-z0-9._-]+)\s*-->/g;
|
|
43
43
|
const URL_DATED = /^https?:\/\/\S+\(\d{4}-\d{2}-\d{2}\)\s*$/;
|
|
44
44
|
|
|
45
|
+
/** Witness sweep (P1.2) — marketing-class surfaces where a bare number must
|
|
46
|
+
* not appear as a capability claim. docs/proof.md and docs/comparison.yaml
|
|
47
|
+
* are structurally witnessed (pointer columns + their own linters). */
|
|
48
|
+
const WITNESS_SURFACES = ['README.md', 'CAPABILITIES.yaml'];
|
|
49
|
+
/** A percentage or multiplier reading as a measured capability figure. */
|
|
50
|
+
const QUANTIFIED_CLAIM = /\b\d+(?:\.\d+)?\s*(?:%|[x×](?![A-Za-z0-9]))/;
|
|
51
|
+
|
|
45
52
|
class ExitCode extends Error {
|
|
46
53
|
code: number;
|
|
47
54
|
constructor(code: number) {
|
|
@@ -224,6 +231,39 @@ function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
224
231
|
if (un) findings.push({ id: entry.id, file: LEDGER_REL, reason: `backed entry has dangling evidence — ${un}` });
|
|
225
232
|
}
|
|
226
233
|
|
|
234
|
+
// 3. Witness sweep (road-to-opt-subagent-harvest P1.2): a QUANTIFIED
|
|
235
|
+
// capability claim on a marketing-class surface must be witnessed — a
|
|
236
|
+
// `<!-- claim:ID -->` marker on the same line, or an explicit
|
|
237
|
+
// `unverified` annotation. Unmarkered numbers are exactly how an
|
|
238
|
+
// "84.8%"-style figure spreads across surfaces with no methodology
|
|
239
|
+
// (the anti-lesson this check encodes). Fenced code blocks are skipped.
|
|
240
|
+
for (const rel of WITNESS_SURFACES) {
|
|
241
|
+
const abs = path.join(REPO, rel);
|
|
242
|
+
let text: string;
|
|
243
|
+
try {
|
|
244
|
+
text = fs.readFileSync(abs, 'utf-8');
|
|
245
|
+
} catch {
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
let inFence = false;
|
|
249
|
+
const lines = text.split('\n');
|
|
250
|
+
for (let i = 0; i < lines.length; i++) {
|
|
251
|
+
const line = lines[i] as string;
|
|
252
|
+
if (/^\s*(```|~~~)/.test(line)) {
|
|
253
|
+
inFence = !inFence;
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (inFence) continue;
|
|
257
|
+
if (!QUANTIFIED_CLAIM.test(line)) continue;
|
|
258
|
+
if (/<!--\s*claim:/.test(line) || /unverified/i.test(line)) continue;
|
|
259
|
+
findings.push({
|
|
260
|
+
id: '(unmarkered)',
|
|
261
|
+
file: `${rel}:${i + 1}`,
|
|
262
|
+
reason: `quantified claim without a claim marker or 'unverified' annotation — ${line.trim().slice(0, 70)}`,
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
227
267
|
if (findings.length > 0) {
|
|
228
268
|
process.stderr.write(`❌ check_claims: ${findings.length} unbacked/dangling claim(s):\n`);
|
|
229
269
|
for (const f of findings) {
|
|
@@ -64,7 +64,12 @@ function loadEntries(root: string): InvariantEntry[] {
|
|
|
64
64
|
if (!Array.isArray(parsed)) {
|
|
65
65
|
throw new Error(`${INVARIANTS_JSON}: expected a top-level array`);
|
|
66
66
|
}
|
|
67
|
-
|
|
67
|
+
// A leading `{"_meta": ...}` element carries the change-process pointer
|
|
68
|
+
// (JSON has no comments); it is not an invariant entry.
|
|
69
|
+
const entries = (parsed as Array<InvariantEntry | { _meta: unknown }>).filter(
|
|
70
|
+
(e): e is InvariantEntry => !('_meta' in e),
|
|
71
|
+
);
|
|
72
|
+
for (const entry of entries) {
|
|
68
73
|
if (
|
|
69
74
|
typeof entry.rule !== 'string' ||
|
|
70
75
|
typeof entry.file !== 'string' ||
|
|
@@ -77,7 +82,7 @@ function loadEntries(root: string): InvariantEntry[] {
|
|
|
77
82
|
);
|
|
78
83
|
}
|
|
79
84
|
}
|
|
80
|
-
return
|
|
85
|
+
return entries;
|
|
81
86
|
}
|
|
82
87
|
|
|
83
88
|
/** Check one entry against pre-normalised file contents. */
|
|
@@ -144,7 +149,9 @@ function formatText(misses: Miss[], invariantCount: number): string {
|
|
|
144
149
|
lines.push(
|
|
145
150
|
'\nA kernel rule lost a load-bearing clause — likely a merge or condensation ' +
|
|
146
151
|
'dropped it silently (see the #840/#844 incident in this file header). ' +
|
|
147
|
-
'Restore the clause or
|
|
152
|
+
'Restore the clause, or reword it via the documented change process: ' +
|
|
153
|
+
'docs/contracts/kernel-membership.md § 10 — Changing a protected invariant ' +
|
|
154
|
+
'(never "update the gate green").',
|
|
148
155
|
);
|
|
149
156
|
return lines.join('\n');
|
|
150
157
|
}
|
package/src/scripts/condense.ts
CHANGED
|
@@ -1634,10 +1634,20 @@ export function generate_claude_subagents(): number {
|
|
|
1634
1634
|
}
|
|
1635
1635
|
_mkdirp(MODULE_STATE.CLAUDE_AGENTS_DIR);
|
|
1636
1636
|
|
|
1637
|
-
const sources = _rglobSorted(MODULE_STATE.SUBAGENTS_SOURCE, '*.md').filter(
|
|
1637
|
+
const sources = _rglobSorted(MODULE_STATE.SUBAGENTS_SOURCE, '*.md').filter(
|
|
1638
|
+
(p) => _isFile(p) && !path.basename(p).startsWith('_'),
|
|
1639
|
+
);
|
|
1638
1640
|
const current = new Set<string>();
|
|
1639
1641
|
let count = 0;
|
|
1640
1642
|
|
|
1643
|
+
// Prompt-defense preamble partial (road-to-opt-subagent-harvest P1.1):
|
|
1644
|
+
// `_`-prefixed files in src/subagents/ are partials, never agents. The
|
|
1645
|
+
// preamble is injected at the top of every projected body so the whole
|
|
1646
|
+
// fleet carries a uniform injection-defense baseline — each dispatched
|
|
1647
|
+
// subagent is an untrusted-content ingestion point.
|
|
1648
|
+
const defensePath = path.join(MODULE_STATE.SUBAGENTS_SOURCE, '_prompt-defense.md');
|
|
1649
|
+
const defense = _isFile(defensePath) ? _readText(defensePath).trim() : '';
|
|
1650
|
+
|
|
1641
1651
|
for (const src of sources) {
|
|
1642
1652
|
const stem = path.basename(src, '.md');
|
|
1643
1653
|
const text = _readText(src);
|
|
@@ -1661,6 +1671,10 @@ export function generate_claude_subagents(): number {
|
|
|
1661
1671
|
const tier = typeof fm['model_tier'] === 'string' ? (fm['model_tier'] as string) : 'inherit';
|
|
1662
1672
|
const model = tier === 'inherit' ? 'inherit' : (_TIER_TO_CLAUDE_MODEL[tier] ?? 'inherit');
|
|
1663
1673
|
|
|
1674
|
+
const guardedBody =
|
|
1675
|
+
defense !== '' && !body.includes('prompt-defense-preamble')
|
|
1676
|
+
? `\n${defense}\n${body}`
|
|
1677
|
+
: body;
|
|
1664
1678
|
const out =
|
|
1665
1679
|
'---\n' +
|
|
1666
1680
|
`name: ${name}\n` +
|
|
@@ -1668,7 +1682,7 @@ export function generate_claude_subagents(): number {
|
|
|
1668
1682
|
`tools: ${tools}\n` +
|
|
1669
1683
|
`model: ${model}\n` +
|
|
1670
1684
|
'---\n' +
|
|
1671
|
-
|
|
1685
|
+
guardedBody;
|
|
1672
1686
|
|
|
1673
1687
|
const target = path.join(MODULE_STATE.CLAUDE_AGENTS_DIR, `${name}.md`);
|
|
1674
1688
|
_writeText(target, out);
|
|
@@ -52,6 +52,8 @@ interface LegResult {
|
|
|
52
52
|
ok: boolean;
|
|
53
53
|
detail: string;
|
|
54
54
|
durationMs: number;
|
|
55
|
+
/** fn ran to completion (a declared skip or an after-failure skip is NOT executed). */
|
|
56
|
+
executed: boolean;
|
|
55
57
|
}
|
|
56
58
|
|
|
57
59
|
function log(msg: string): void {
|
|
@@ -313,13 +315,20 @@ async function main(): Promise<number> {
|
|
|
313
315
|
if (only && !only.includes(leg.name)) continue;
|
|
314
316
|
// Legs after a hard dependency failure cannot run meaningfully.
|
|
315
317
|
if (failed) {
|
|
316
|
-
ctx.results.push({
|
|
318
|
+
ctx.results.push({
|
|
319
|
+
leg: leg.name,
|
|
320
|
+
ok: false,
|
|
321
|
+
detail: 'skipped (earlier leg failed)',
|
|
322
|
+
durationMs: 0,
|
|
323
|
+
executed: false,
|
|
324
|
+
});
|
|
317
325
|
continue;
|
|
318
326
|
}
|
|
319
327
|
const t0 = Date.now();
|
|
320
328
|
try {
|
|
321
329
|
const detail = await leg.fn(ctx);
|
|
322
|
-
|
|
330
|
+
const executed = !detail.startsWith('skipped');
|
|
331
|
+
ctx.results.push({ leg: leg.name, ok: true, detail, durationMs: Date.now() - t0, executed });
|
|
323
332
|
log(` ✅ ${leg.name} — ${detail}`);
|
|
324
333
|
} catch (err) {
|
|
325
334
|
ctx.results.push({
|
|
@@ -327,12 +336,42 @@ async function main(): Promise<number> {
|
|
|
327
336
|
ok: false,
|
|
328
337
|
detail: (err as Error).message,
|
|
329
338
|
durationMs: Date.now() - t0,
|
|
339
|
+
executed: true,
|
|
330
340
|
});
|
|
331
341
|
log(` ❌ ${leg.name} — ${(err as Error).message}`);
|
|
332
342
|
failed = true;
|
|
333
343
|
}
|
|
334
344
|
}
|
|
335
345
|
|
|
346
|
+
// Leg-completeness gate (road-to-fable-feedback-5 Phase 6a): a full run
|
|
347
|
+
// (no --only) must have EXECUTED every expected leg — a silently-skipped
|
|
348
|
+
// leg can no longer ride a green workflow. The only declared skip is
|
|
349
|
+
// `upgrade` under an explicit --skip-registry flag.
|
|
350
|
+
if (!only && !failed) {
|
|
351
|
+
const declaredSkips = new Set(ctx.skipRegistry ? ['upgrade'] : []);
|
|
352
|
+
const missing = LEGS.filter((l) => {
|
|
353
|
+
const r = ctx.results.find((x) => x.leg === l.name);
|
|
354
|
+
return r === undefined || (!r.executed && !declaredSkips.has(l.name));
|
|
355
|
+
}).map((l) => l.name);
|
|
356
|
+
if (missing.length > 0) {
|
|
357
|
+
log(`❌ leg-completeness gate: expected leg(s) did not execute: ${missing.join(', ')}`);
|
|
358
|
+
failed = true;
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const manifestPath = getOpt('--manifest');
|
|
363
|
+
if (manifestPath) {
|
|
364
|
+
const manifest = {
|
|
365
|
+
ok: !failed,
|
|
366
|
+
node: process.version,
|
|
367
|
+
skipRegistry: ctx.skipRegistry,
|
|
368
|
+
expected: LEGS.map((l) => l.name),
|
|
369
|
+
results: ctx.results,
|
|
370
|
+
};
|
|
371
|
+
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
372
|
+
log(`leg manifest written: ${manifestPath}`);
|
|
373
|
+
}
|
|
374
|
+
|
|
336
375
|
if (argv.includes('--json')) {
|
|
337
376
|
process.stdout.write(`${JSON.stringify({ ok: !failed, results: ctx.results }, null, 2)}\n`);
|
|
338
377
|
}
|