@event4u/agent-config 8.13.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +136 -0
- package/README.md +1 -1
- package/dist/agent-src/commands/agent-handoff.md +35 -0
- package/dist/agent-src/commands/agents/init.md +1 -0
- package/dist/agent-src/commands/agents/optimize.md +1 -0
- package/dist/agent-src/commands/agents/user/accept.md +3 -2
- package/dist/agent-src/commands/agents/user/init.md +5 -4
- package/dist/agent-src/commands/agents/user/update.md +3 -2
- package/dist/agent-src/commands/agents/user.md +1 -0
- package/dist/agent-src/commands/agents.md +1 -0
- package/dist/agent-src/commands/analytics/show.md +1 -0
- package/dist/agent-src/commands/analytics.md +1 -0
- package/dist/agent-src/commands/analyze/decision.md +1 -0
- package/dist/agent-src/commands/analyze/incident.md +1 -0
- package/dist/agent-src/commands/analyze/near-miss.md +1 -0
- package/dist/agent-src/commands/analyze/postmortem.md +1 -0
- package/dist/agent-src/commands/analyze/premortem.md +1 -0
- package/dist/agent-src/commands/analyze/reference-repo.md +1 -0
- package/dist/agent-src/commands/analyze.md +3 -2
- package/dist/agent-src/commands/brand/identity.md +6 -5
- package/dist/agent-src/commands/brand/review.md +6 -5
- package/dist/agent-src/commands/brand/strategy.md +6 -4
- package/dist/agent-src/commands/brand/tokens.md +7 -6
- package/dist/agent-src/commands/brand/voice.md +5 -4
- package/dist/agent-src/commands/brand.md +9 -7
- package/dist/agent-src/commands/bug/fix.md +1 -0
- package/dist/agent-src/commands/bug/investigate.md +1 -0
- package/dist/agent-src/commands/bug.md +2 -1
- package/dist/agent-src/commands/challenge-me/vision.md +1 -0
- package/dist/agent-src/commands/challenge-me/with-docs.md +1 -0
- package/dist/agent-src/commands/challenge-me.md +28 -6
- package/dist/agent-src/commands/chat-history.md +1 -0
- package/dist/agent-src/commands/check-current-md.md +1 -0
- package/dist/agent-src/commands/commit.md +1 -0
- package/dist/agent-src/commands/context/create.md +3 -2
- package/dist/agent-src/commands/context.md +1 -0
- package/dist/agent-src/commands/cost/profile.md +1 -0
- package/dist/agent-src/commands/cost.md +2 -1
- package/dist/agent-src/commands/council/analysis.md +1 -0
- package/dist/agent-src/commands/council/debate.md +1 -0
- package/dist/agent-src/commands/council/default.md +21 -20
- package/dist/agent-src/commands/council/design.md +1 -0
- package/dist/agent-src/commands/council/optimize.md +1 -0
- package/dist/agent-src/commands/council/pr.md +1 -0
- package/dist/agent-src/commands/council.md +1 -0
- package/dist/agent-src/commands/estimate-ticket.md +1 -0
- package/dist/agent-src/commands/feature/dev.md +1 -0
- package/dist/agent-src/commands/feature/explore.md +1 -0
- package/dist/agent-src/commands/feature/plan.md +1 -0
- package/dist/agent-src/commands/feature/refactor.md +1 -0
- package/dist/agent-src/commands/feature/roadmap.md +1 -0
- package/dist/agent-src/commands/feature.md +2 -1
- package/dist/agent-src/commands/fix/pr-comments.md +19 -0
- package/dist/agent-src/commands/fix.md +2 -1
- package/dist/agent-src/commands/ghostwriter/delete.md +1 -0
- package/dist/agent-src/commands/ghostwriter/fetch.md +1 -0
- package/dist/agent-src/commands/ghostwriter/show.md +1 -0
- package/dist/agent-src/commands/ghostwriter/write.md +1 -0
- package/dist/agent-src/commands/ghostwriter.md +1 -0
- package/dist/agent-src/commands/grill-me.md +17 -12
- package/dist/agent-src/commands/humanize.md +1 -0
- package/dist/agent-src/commands/image/analyse.md +2 -1
- package/dist/agent-src/commands/image/create.md +1 -0
- package/dist/agent-src/commands/image/verify.md +1 -0
- package/dist/agent-src/commands/image.md +16 -12
- package/dist/agent-src/commands/implement-ticket.md +1 -0
- package/dist/agent-src/commands/judge/on-diff.md +1 -0
- package/dist/agent-src/commands/judge/solo.md +1 -0
- package/dist/agent-src/commands/judge/steps.md +1 -0
- package/dist/agent-src/commands/judge.md +1 -0
- package/dist/agent-src/commands/knowledge/cross-repo.md +1 -0
- package/dist/agent-src/commands/knowledge/forget.md +1 -0
- package/dist/agent-src/commands/knowledge/ingest.md +1 -0
- package/dist/agent-src/commands/knowledge/list.md +10 -0
- package/dist/agent-src/commands/knowledge.md +1 -0
- package/dist/agent-src/commands/memory/learn-low-impact.md +1 -0
- package/dist/agent-src/commands/memory/mine-session.md +1 -0
- package/dist/agent-src/commands/memory/promote.md +3 -2
- package/dist/agent-src/commands/memory.md +1 -0
- package/dist/agent-src/commands/mission/upgrade.md +45 -24
- package/dist/agent-src/commands/mode.md +1 -0
- package/dist/agent-src/commands/module/create.md +1 -0
- package/dist/agent-src/commands/module/explore.md +1 -0
- package/dist/agent-src/commands/module.md +1 -0
- package/dist/agent-src/commands/optimize/agents-dir.md +1 -0
- package/dist/agent-src/commands/optimize/project.md +1 -0
- package/dist/agent-src/commands/optimize/prompt.md +1 -0
- package/dist/agent-src/commands/optimize.md +2 -1
- package/dist/agent-src/commands/orchestrate.md +1 -0
- package/dist/agent-src/commands/override.md +1 -0
- package/dist/agent-src/commands/package.md +2 -1
- package/dist/agent-src/commands/post-as/ghostwriter.md +1 -0
- package/dist/agent-src/commands/post-as/me.md +1 -0
- package/dist/agent-src/commands/post-as.md +1 -0
- package/dist/agent-src/commands/pr/create/description-only.md +1 -0
- package/dist/agent-src/commands/pr/create.md +19 -17
- package/dist/agent-src/commands/prediction-pool.md +7 -6
- package/dist/agent-src/commands/prepare-for-review.md +1 -0
- package/dist/agent-src/commands/profile/activate.md +1 -0
- package/dist/agent-src/commands/profile/deactivate.md +1 -0
- package/dist/agent-src/commands/profile.md +1 -0
- package/dist/agent-src/commands/project.md +2 -1
- package/dist/agent-src/commands/refine-ticket.md +7 -0
- package/dist/agent-src/commands/research/deep.md +1 -0
- package/dist/agent-src/commands/research.md +1 -0
- package/dist/agent-src/commands/review/routing.md +1 -0
- package/dist/agent-src/commands/review.md +2 -1
- package/dist/agent-src/commands/roadmap/ai-council.md +1 -0
- package/dist/agent-src/commands/roadmap/materialize.md +1 -0
- package/dist/agent-src/commands/roadmap/process-full.md +1 -0
- package/dist/agent-src/commands/roadmap/process-phase.md +1 -0
- package/dist/agent-src/commands/roadmap/process-step.md +1 -0
- package/dist/agent-src/commands/roadmap.md +1 -0
- package/dist/agent-src/commands/security-audit-config.md +2 -1
- package/dist/agent-src/commands/skill/preview.md +1 -0
- package/dist/agent-src/commands/skill.md +1 -0
- package/dist/agent-src/commands/skills/discover.md +1 -0
- package/dist/agent-src/commands/skills.md +1 -0
- package/dist/agent-src/commands/sync/agent-settings.md +1 -0
- package/dist/agent-src/commands/sync.md +2 -1
- package/dist/agent-src/commands/team/adversarial.md +3 -2
- package/dist/agent-src/commands/team/delegate.md +3 -2
- package/dist/agent-src/commands/team/review.md +3 -2
- package/dist/agent-src/commands/team/status.md +49 -18
- package/dist/agent-src/commands/team-knowledge.md +8 -6
- package/dist/agent-src/commands/team.md +21 -16
- package/dist/agent-src/commands/tests/e2e-heal.md +1 -0
- package/dist/agent-src/commands/tests/e2e-plan.md +1 -0
- package/dist/agent-src/commands/tests/execute.md +1 -0
- package/dist/agent-src/commands/tests.md +2 -1
- package/dist/agent-src/commands/update-form-request-messages.md +1 -0
- package/dist/agent-src/commands/upstream-contribute.md +1 -0
- package/dist/agent-src/commands/video/from-script.md +3 -2
- package/dist/agent-src/commands/video/from-song.md +22 -21
- package/dist/agent-src/commands/video/scene.md +1 -0
- package/dist/agent-src/commands/video/stitch.md +2 -1
- package/dist/agent-src/commands/video/storyboard.md +1 -0
- package/dist/agent-src/commands/video.md +1 -0
- package/dist/agent-src/commands/work.md +1 -0
- package/dist/agent-src/commands/worktree/cleanup.md +8 -7
- package/dist/agent-src/commands/worktree/create.md +1 -0
- package/dist/agent-src/commands/worktree/verify.md +1 -0
- package/dist/agent-src/commands/worktree.md +1 -0
- package/dist/agent-src/rules/active-remediation.md +12 -36
- package/dist/agent-src/rules/artifact-drafting-protocol.md +7 -64
- package/dist/agent-src/rules/context-hygiene.md +19 -113
- package/dist/agent-src/rules/design-fidelity.md +4 -60
- package/dist/agent-src/rules/domain-adoption-policy.md +10 -106
- package/dist/agent-src/rules/framework-neutrality-in-generic-skills.md +12 -60
- package/dist/agent-src/rules/improve-before-implement.md +4 -0
- package/dist/agent-src/rules/minimal-safe-diff.md +6 -86
- package/dist/agent-src/rules/size-enforcement.md +10 -0
- package/dist/agent-src/rules/tool-safety.md +17 -3
- package/dist/agent-src/skills/agent-security-review/SKILL.md +12 -1
- package/dist/agent-src/skills/ai-council/SKILL.md +11 -0
- package/dist/agent-src/skills/bug-analyzer/SKILL.md +24 -2
- package/dist/agent-src/skills/code-review/SKILL.md +105 -93
- package/dist/agent-src/skills/code-review/checklists/backend.md +50 -0
- package/dist/agent-src/skills/code-review/checklists/database.md +19 -0
- package/dist/agent-src/skills/code-review/checklists/dependency.md +18 -0
- package/dist/agent-src/skills/code-review/checklists/docs.md +14 -0
- package/dist/agent-src/skills/code-review/checklists/frontend.md +19 -0
- package/dist/agent-src/skills/code-review/checklists/infra.md +17 -0
- package/dist/agent-src/skills/code-review/checklists/producing-the-review.md +62 -0
- package/dist/agent-src/skills/code-review/checklists/test-only.md +13 -0
- package/dist/agent-src/skills/code-review/evals/evals.json +53 -15
- package/dist/agent-src/skills/condense-memory/SKILL.md +3 -2
- package/dist/agent-src/skills/conventional-commits-writing/evals/evals.json +15 -0
- package/dist/agent-src/skills/docker/SKILL.md +10 -0
- package/dist/agent-src/skills/docx-authoring/SKILL.md +102 -0
- package/dist/agent-src/skills/github-ci/evals/evals.json +25 -0
- package/dist/agent-src/skills/humanizer/SKILL.md +79 -3
- package/dist/agent-src/skills/humanizer/references/anti-aiisms.md +112 -0
- package/dist/agent-src/skills/humanizer/references/fixtures.md +70 -0
- package/dist/agent-src/skills/judge-bug-hunter/SKILL.md +7 -1
- package/dist/agent-src/skills/laravel/SKILL.md +10 -0
- package/dist/agent-src/skills/launch-readiness/SKILL.md +35 -0
- package/dist/agent-src/skills/merge-conflicts/SKILL.md +30 -0
- package/dist/agent-src/skills/pdf-tools/SKILL.md +101 -0
- package/dist/agent-src/skills/quality-tools/evals/evals.json +15 -0
- package/dist/agent-src/skills/security-audit/SKILL.md +29 -0
- package/dist/agent-src/skills/security-maturity-assessment/SKILL.md +90 -0
- package/dist/agent-src/skills/security-maturity-assessment/evals/triggers.json +52 -0
- package/dist/agent-src/skills/skill-writing/SKILL.md +79 -0
- package/dist/agent-src/skills/spreadsheet-authoring/SKILL.md +11 -3
- package/dist/agent-src/skills/subagent-orchestration/schemas/team-review-status.json +103 -0
- package/dist/agent-src/skills/terraform/SKILL.md +10 -0
- package/dist/agent-src/skills/test-driven-development/SKILL.md +31 -0
- package/dist/agent-src/skills/threat-modeling/SKILL.md +15 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/scripts/telemetry/aggregator.ts +0 -0
- package/dist/agent-src/templates/scripts/telemetry/engagement.ts +30 -0
- package/dist/agent-src/templates/scripts/telemetry/report_renderer.ts +22 -9
- package/dist/agent-src/templates/scripts/telemetry_record.ts +5 -0
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +256 -190
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +5 -5
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +20 -17
- package/dist/discovery/trust-report.md +3 -3
- package/dist/discovery/workspaces.json +16 -13
- package/dist/install/install.mjs +10 -2
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/router.json +1 -1
- package/dist/scripts/condense.js +5 -0
- package/dist/scripts/condense.js.map +1 -1
- package/dist/server/schemas/settings.js +5 -1
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/CLAIMS.md +8 -1
- package/docs/SKILL_CENSUS.md +8 -0
- package/docs/architecture.md +2 -2
- package/docs/benchmark.md +67 -2
- package/docs/catalog.md +2 -2
- package/docs/contracts/ai-council-config.md +9 -2
- package/docs/contracts/ai-team-config.md +37 -4
- package/docs/contracts/prelaunch-diagnostics.md +151 -0
- package/docs/contracts/release-pr-gating.md +1 -0
- package/docs/decisions/ADR-049-configuration-trust-boundary.md +9 -1
- package/docs/featured-skills.md +2 -2
- package/docs/getting-started-by-role.md +1 -1
- package/docs/getting-started.md +16 -5
- package/docs/governance-advantage.md +1 -1
- package/docs/guidelines/agent-infra/active-remediation-mechanics.md +45 -0
- package/docs/guidelines/agent-infra/agent-interaction-and-decision-quality.md +28 -0
- package/docs/guidelines/agent-infra/artifact-drafting-protocol-mechanics.md +76 -0
- package/docs/guidelines/agent-infra/context-hygiene-mechanics.md +90 -0
- package/docs/guidelines/agent-infra/domain-adoption-gates.md +112 -0
- package/docs/guidelines/agent-infra/framework-neutrality-patterns.md +66 -0
- package/docs/guidelines/agent-infra/minimal-safe-diff-mechanics.md +99 -0
- package/docs/guidelines/agent-infra/skill-quality-checklist.md +13 -0
- package/docs/guidelines/design-fidelity-mechanics.md +76 -0
- package/docs/parity/command-frontmatter.md +60 -0
- package/docs/proof.md +6 -5
- package/package.json +1 -1
- package/src/agent-src/templates/scripts/telemetry/aggregator.ts +0 -0
- package/src/agent-src/templates/scripts/telemetry/engagement.ts +30 -0
- package/src/agent-src/templates/scripts/telemetry/report_renderer.ts +22 -9
- package/src/agent-src/templates/scripts/telemetry_record.ts +5 -0
- package/src/config/agent-settings.template.yml +34 -10
- package/src/config/prelaunch-areas.yml +36 -0
- package/src/scripts/_cli/cmd_doctor.ts +118 -26
- package/src/scripts/_lib/claude_plugin.ts +77 -2
- package/src/scripts/_lib/orchestration_record.ts +5 -0
- package/src/scripts/_lib/tool_adapter_registry.ts +120 -0
- package/src/scripts/ai_team/config.ts +73 -1
- package/src/scripts/ai_team/review_gate.ts +636 -0
- package/src/scripts/ai_team/team_dispatch.ts +612 -0
- package/src/scripts/bench_ab_task_runner.ts +11 -1
- package/src/scripts/bench_parity_count.ts +279 -0
- package/src/scripts/build_linear_digest.ts +3 -1
- package/src/scripts/check_bridge_derivation.ts +172 -0
- package/src/scripts/check_consumer_scope_flip.ts +179 -0
- package/src/scripts/check_council_config_location.ts +115 -15
- package/src/scripts/check_host_loadability.ts +81 -0
- package/src/scripts/condense.ts +5 -0
- package/src/scripts/hook_manifest.yaml +12 -1
- package/src/scripts/lint_handoffs.ts +42 -0
- package/src/scripts/lint_skill_descriptions.ts +228 -0
- package/src/scripts/lint_skill_descriptions_allowlist.json +1 -0
- package/src/scripts/lint_skill_scripts_readonly.ts +145 -0
- package/src/scripts/lint_skill_scripts_readonly_allowlist.json +8 -0
- package/src/scripts/lint_supported_tools_matrix.ts +35 -0
- package/src/scripts/optimize_skill_description.ts +144 -0
- package/src/scripts/orchestration_record.ts +2 -1
- package/src/scripts/prelaunch_diagnostics.ts +248 -0
- package/src/scripts/run_skill_evals.ts +70 -4
- package/src/scripts/schemas/command.schema.json +7 -0
- package/src/scripts/schemas/evals.schema.json +105 -14
- package/src/scripts/schemas/prelaunch-report.schema.json +107 -0
- package/src/scripts/schemas/skill.schema.json +18 -1
- package/src/scripts/schemas/subagent.schema.json +6 -0
- package/src/scripts/skill_linter.ts +43 -3
- package/src/scripts/smoke_host_loadability.sh +73 -0
- package/src/scripts/team_review_gate_hook.ts +115 -0
- package/src/scripts/trigger_eval_grandfather.json +1 -2
- package/src/scripts/utilization_report.ts +225 -0
- package/src/server/schemas/settings.ts +9 -1
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bench_parity_count.ts — cross-model parity eval, count-distribution pass.
|
|
3
|
+
*
|
|
4
|
+
* Design: docs/design/cross-model-parity-eval.md (road-to-opt-measurement-unblock
|
|
5
|
+
* Phase 3). Re-scope: council-transport execution, NOT an in-host subagent
|
|
6
|
+
* harness — each orchestration-corpus task is rendered into a single
|
|
7
|
+
* self-contained prompt (fixture files inlined) asking the model to produce
|
|
8
|
+
* the findings it would surface, dispatched identically to ≥2 vendors via the
|
|
9
|
+
* existing `ExternalAIClient` transport. Output: per-host finding-count
|
|
10
|
+
* distributions + the derived `finding_floor` calibration.
|
|
11
|
+
*
|
|
12
|
+
* Pre-registered BEFORE the first billable call (2026-07-12, maintainer
|
|
13
|
+
* approved "Bauen + Count-Pass", ceiling $8):
|
|
14
|
+
* - VENDORS: anthropic/claude-sonnet-4-5 + openai/gpt-4o (the two council
|
|
15
|
+
* defaults; no new transport, no new auth surface).
|
|
16
|
+
* - REPEATS = 3 identical calls per (task × vendor) — distribution, not
|
|
17
|
+
* a point estimate.
|
|
18
|
+
* - MAX_TOKENS = 2000 per call.
|
|
19
|
+
* - MAX_USD = 8.0 hard abort (exit 2) — the authorized ceiling.
|
|
20
|
+
* - Floor rule: per non-control task, floor = max(1, min over hosts of the
|
|
21
|
+
* median per-host count) — the cross-host lower envelope, so the gate
|
|
22
|
+
* fires on a host/task that under-produces relative to the cross-vendor
|
|
23
|
+
* norm, never relative to one vendor.
|
|
24
|
+
* - pv-02 is a NEGATIVE CONTROL (clean code): its counts are reported but
|
|
25
|
+
* excluded from floor derivation — a well-behaved host surfaces ~0
|
|
26
|
+
* findings there, and a floor > 0 would reward spurious findings.
|
|
27
|
+
* - Counting uses the SAME `_count_findings` as `run_skill_evals.ts`, so
|
|
28
|
+
* calibration and gate measure identically.
|
|
29
|
+
*
|
|
30
|
+
* Usage:
|
|
31
|
+
* npx tsx src/scripts/bench_parity_count.ts --dry-run # render prompts, no API
|
|
32
|
+
* npx tsx src/scripts/bench_parity_count.ts --live # billable
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import * as fs from 'node:fs';
|
|
36
|
+
import * as path from 'node:path';
|
|
37
|
+
|
|
38
|
+
import { _count_findings } from './run_skill_evals.js';
|
|
39
|
+
import { price_usd } from './bench_quality_rerun.js';
|
|
40
|
+
|
|
41
|
+
export const REPEATS = 3;
|
|
42
|
+
export const MAX_TOKENS = 2000;
|
|
43
|
+
export const DEFAULT_MAX_USD = 8.0;
|
|
44
|
+
export const CORPUS_DIR = 'internal/bench/orchestration/corpus';
|
|
45
|
+
export const REPORT_PATH = 'internal/bench/reports/parity-count.json';
|
|
46
|
+
|
|
47
|
+
/** Fixture roots searched, in order, when inlining files a task references. */
|
|
48
|
+
export const FIXTURE_ROOTS = [
|
|
49
|
+
'internal/bench/ab/fixture',
|
|
50
|
+
'internal/bench/ab/fixtures-v2/agL-debug-01',
|
|
51
|
+
'internal/bench/orchestration/fixtures/pv-hollow',
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
/** Static, auditable map: corpus task id → files inlined into its prompt. */
|
|
55
|
+
export const TASK_FILES: Record<string, { files: string[]; control: boolean }> = {
|
|
56
|
+
'orch-01-multifile-analysis': {
|
|
57
|
+
files: ['src/parser.ts', 'src/formatter.ts', 'src/cli.ts', 'src/reducers.mjs'],
|
|
58
|
+
control: false,
|
|
59
|
+
},
|
|
60
|
+
'orch-02-ordered-refactor': {
|
|
61
|
+
files: ['src/parser.ts', 'src/formatter.ts'],
|
|
62
|
+
control: false,
|
|
63
|
+
},
|
|
64
|
+
'orch-03-competitive-impl': {
|
|
65
|
+
files: ['src/parser.ts'],
|
|
66
|
+
control: false,
|
|
67
|
+
},
|
|
68
|
+
'pv-01-hollow-detection': {
|
|
69
|
+
files: ['src/charge.ts', 'tests/charge.test.ts'],
|
|
70
|
+
control: false,
|
|
71
|
+
},
|
|
72
|
+
'pv-02-negative-control': {
|
|
73
|
+
files: ['src/slugify.ts'],
|
|
74
|
+
control: true,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
const SYSTEM_PROMPT = [
|
|
79
|
+
'You are a senior engineer reviewing code for a benchmark. You cannot run',
|
|
80
|
+
'code, modify files, or spawn agents. Work only from the file contents',
|
|
81
|
+
'provided in the message.',
|
|
82
|
+
].join(' ');
|
|
83
|
+
|
|
84
|
+
const OUTPUT_CONTRACT = [
|
|
85
|
+
'',
|
|
86
|
+
'---',
|
|
87
|
+
'OUTPUT CONTRACT (benchmark): Do NOT modify files and do NOT describe how',
|
|
88
|
+
'you would delegate. Output ONLY a numbered markdown list of the concrete',
|
|
89
|
+
'findings / changes you would surface for this task — one finding per',
|
|
90
|
+
'list item, most important first, each with its file reference. If the',
|
|
91
|
+
'code is clean and you have nothing to surface, output exactly:',
|
|
92
|
+
'NO FINDINGS',
|
|
93
|
+
'with no list.',
|
|
94
|
+
].join('\n');
|
|
95
|
+
|
|
96
|
+
/** Extract the first fenced block after a `## Task prompt` heading. */
|
|
97
|
+
export function extract_task_prompt(md: string): string {
|
|
98
|
+
const at = md.indexOf('## Task prompt');
|
|
99
|
+
if (at === -1) throw new Error('no "## Task prompt" heading');
|
|
100
|
+
const rest = md.slice(at);
|
|
101
|
+
const m = rest.match(/```\n([\s\S]*?)```/);
|
|
102
|
+
if (!m || m[1] === undefined) throw new Error('no fenced task-prompt block');
|
|
103
|
+
return m[1].trimEnd();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Resolve a task-referenced file against the fixture roots. */
|
|
107
|
+
export function resolve_fixture(rel: string, repoRoot = '.'): string {
|
|
108
|
+
for (const root of FIXTURE_ROOTS) {
|
|
109
|
+
const p = path.join(repoRoot, root, rel);
|
|
110
|
+
if (fs.existsSync(p)) return p;
|
|
111
|
+
}
|
|
112
|
+
throw new Error(`fixture file not found in any root: ${rel}`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function render_user_prompt(taskId: string, taskPrompt: string, repoRoot = '.'): string {
|
|
116
|
+
const spec = TASK_FILES[taskId];
|
|
117
|
+
if (!spec) throw new Error(`no TASK_FILES entry for ${taskId}`);
|
|
118
|
+
const parts: string[] = ['Here are the project files for this task:', ''];
|
|
119
|
+
for (const rel of spec.files) {
|
|
120
|
+
const body = fs.readFileSync(resolve_fixture(rel, repoRoot), 'utf-8');
|
|
121
|
+
parts.push(`### ${rel}`, '```', body.trimEnd(), '```', '');
|
|
122
|
+
}
|
|
123
|
+
parts.push('## Task', '', taskPrompt, OUTPUT_CONTRACT);
|
|
124
|
+
return parts.join('\n');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function median(xs: number[]): number {
|
|
128
|
+
if (xs.length === 0) return 0;
|
|
129
|
+
const s = [...xs].sort((a, b) => a - b);
|
|
130
|
+
const mid = Math.floor(s.length / 2);
|
|
131
|
+
return s.length % 2 === 1 ? (s[mid] ?? 0) : ((s[mid - 1] ?? 0) + (s[mid] ?? 0)) / 2;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface TaskCounts {
|
|
135
|
+
task: string;
|
|
136
|
+
control: boolean;
|
|
137
|
+
/** host → the REPEATS finding counts */
|
|
138
|
+
counts: Record<string, number[]>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export interface FloorRow {
|
|
142
|
+
task: string;
|
|
143
|
+
control: boolean;
|
|
144
|
+
medians: Record<string, number>;
|
|
145
|
+
/** min over hosts of the per-host median (the lower envelope). */
|
|
146
|
+
envelope: number;
|
|
147
|
+
/** calibrated floor; null for controls (excluded by design). */
|
|
148
|
+
floor: number | null;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Pre-registered floor rule — see header. */
|
|
152
|
+
export function derive_floors(rows: TaskCounts[]): FloorRow[] {
|
|
153
|
+
return rows.map((r) => {
|
|
154
|
+
const medians: Record<string, number> = {};
|
|
155
|
+
for (const [host, xs] of Object.entries(r.counts)) medians[host] = median(xs);
|
|
156
|
+
const vals = Object.values(medians);
|
|
157
|
+
const envelope = vals.length ? Math.min(...vals) : 0;
|
|
158
|
+
return {
|
|
159
|
+
task: r.task,
|
|
160
|
+
control: r.control,
|
|
161
|
+
medians,
|
|
162
|
+
envelope,
|
|
163
|
+
floor: r.control ? null : Math.max(1, Math.floor(envelope)),
|
|
164
|
+
};
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
interface Vendor {
|
|
169
|
+
host: string;
|
|
170
|
+
model: string;
|
|
171
|
+
ask: (system: string, user: string, maxTokens: number) => {
|
|
172
|
+
text: string;
|
|
173
|
+
input_tokens: number;
|
|
174
|
+
output_tokens: number;
|
|
175
|
+
error: string | null;
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function load_vendors(): Promise<Vendor[]> {
|
|
180
|
+
const c = await import('./ai_council/clients.js');
|
|
181
|
+
const anthropic = new c.AnthropicClient({ api_key: c.load_anthropic_key() });
|
|
182
|
+
const openai = new c.OpenAIClient({ api_key: c.load_openai_key() });
|
|
183
|
+
return [
|
|
184
|
+
{
|
|
185
|
+
host: `anthropic/${c.DEFAULT_ANTHROPIC_MODEL}`,
|
|
186
|
+
model: c.DEFAULT_ANTHROPIC_MODEL,
|
|
187
|
+
ask: (s, u, m) => anthropic.ask(s, u, m),
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
host: `openai/${c.DEFAULT_OPENAI_MODEL}`,
|
|
191
|
+
model: c.DEFAULT_OPENAI_MODEL,
|
|
192
|
+
ask: (s, u, m) => openai.ask(s, u, m),
|
|
193
|
+
},
|
|
194
|
+
];
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export async function main(argv: string[]): Promise<number> {
|
|
198
|
+
const live = argv.includes('--live');
|
|
199
|
+
const dry = argv.includes('--dry-run') || !live;
|
|
200
|
+
let maxUsd = DEFAULT_MAX_USD;
|
|
201
|
+
const mi = argv.indexOf('--max-usd');
|
|
202
|
+
if (mi !== -1) maxUsd = Number(argv[mi + 1]);
|
|
203
|
+
|
|
204
|
+
const taskIds = Object.keys(TASK_FILES);
|
|
205
|
+
const prompts: Record<string, string> = {};
|
|
206
|
+
for (const id of taskIds) {
|
|
207
|
+
const md = fs.readFileSync(path.join(CORPUS_DIR, `${id}.md`), 'utf-8');
|
|
208
|
+
prompts[id] = render_user_prompt(id, extract_task_prompt(md));
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (dry) {
|
|
212
|
+
for (const id of taskIds) {
|
|
213
|
+
process.stdout.write(`--- ${id} (${(prompts[id] ?? '').length} chars)\n`);
|
|
214
|
+
}
|
|
215
|
+
process.stdout.write('dry-run: prompts rendered, no API calls.\n');
|
|
216
|
+
return 0;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const vendors = await load_vendors();
|
|
220
|
+
let spent = 0;
|
|
221
|
+
const rows: TaskCounts[] = [];
|
|
222
|
+
const raw: Array<Record<string, unknown>> = [];
|
|
223
|
+
for (const id of taskIds) {
|
|
224
|
+
const counts: Record<string, number[]> = {};
|
|
225
|
+
for (const v of vendors) {
|
|
226
|
+
const hostCounts: number[] = [];
|
|
227
|
+
counts[v.host] = hostCounts;
|
|
228
|
+
for (let rep = 0; rep < REPEATS; rep += 1) {
|
|
229
|
+
if (spent >= maxUsd) {
|
|
230
|
+
process.stderr.write(`ABORT: spent $${spent.toFixed(2)} >= --max-usd ${maxUsd}\n`);
|
|
231
|
+
return 2;
|
|
232
|
+
}
|
|
233
|
+
const resp = v.ask(SYSTEM_PROMPT, prompts[id] ?? '', MAX_TOKENS);
|
|
234
|
+
if (resp.error) {
|
|
235
|
+
process.stderr.write(`ERROR ${id} × ${v.host} rep${rep}: ${resp.error}\n`);
|
|
236
|
+
return 1;
|
|
237
|
+
}
|
|
238
|
+
spent += price_usd(v.model, resp.input_tokens, resp.output_tokens);
|
|
239
|
+
const n = /^\s*NO FINDINGS\s*$/m.test(resp.text) ? 0 : _count_findings(resp.text);
|
|
240
|
+
hostCounts.push(n);
|
|
241
|
+
raw.push({ task: id, host: v.host, rep, count: n, text: resp.text });
|
|
242
|
+
process.stdout.write(`${id} × ${v.host} rep${rep}: ${n} findings ($${spent.toFixed(3)} total)\n`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
rows.push({ task: id, control: TASK_FILES[id]?.control ?? false, counts });
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const floors = derive_floors(rows);
|
|
249
|
+
const report = {
|
|
250
|
+
schema: 'parity-count/1',
|
|
251
|
+
stamp: new Date().toISOString(),
|
|
252
|
+
design: 'docs/design/cross-model-parity-eval.md',
|
|
253
|
+
vendors: vendors.map((v) => v.host),
|
|
254
|
+
repeats: REPEATS,
|
|
255
|
+
max_tokens: MAX_TOKENS,
|
|
256
|
+
floor_rule: 'max(1, floor(min over hosts of median per-host count)); controls excluded',
|
|
257
|
+
counter: 'run_skill_evals._count_findings (identical to the gate)',
|
|
258
|
+
cost_usd_actual: Number(spent.toFixed(4)),
|
|
259
|
+
tasks: rows,
|
|
260
|
+
floors,
|
|
261
|
+
raw,
|
|
262
|
+
};
|
|
263
|
+
fs.mkdirSync(path.dirname(REPORT_PATH), { recursive: true });
|
|
264
|
+
fs.writeFileSync(REPORT_PATH, `${JSON.stringify(report, null, 1)}\n`);
|
|
265
|
+
process.stdout.write(`wrote ${REPORT_PATH} ($${spent.toFixed(2)} actual)\n`);
|
|
266
|
+
return 0;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const isMain = process.argv[1] !== undefined
|
|
270
|
+
&& path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname);
|
|
271
|
+
if (isMain) {
|
|
272
|
+
main(process.argv.slice(2)).then(
|
|
273
|
+
(code) => process.exit(code),
|
|
274
|
+
(err) => {
|
|
275
|
+
process.stderr.write(`${err instanceof Error ? err.stack ?? err.message : String(err)}\n`);
|
|
276
|
+
process.exit(1);
|
|
277
|
+
},
|
|
278
|
+
);
|
|
279
|
+
}
|
|
@@ -41,7 +41,9 @@ export function RuleEntry(name: string, mode = 'as-is', strip_sections: string[]
|
|
|
41
41
|
export const WORKSPACE: RuleEntry[] = [
|
|
42
42
|
RuleEntry('ask-when-uncertain'),
|
|
43
43
|
RuleEntry('commit-conventions'),
|
|
44
|
-
|
|
44
|
+
// P4 Batch B (2026-07-12): the Augment-specific section migrated to
|
|
45
|
+
// guideline:agent-infra/context-hygiene-mechanics — nothing left to strip.
|
|
46
|
+
RuleEntry('context-hygiene', 'degraded'),
|
|
45
47
|
RuleEntry('direct-answers'),
|
|
46
48
|
RuleEntry('markdown-safe-codeblocks'),
|
|
47
49
|
RuleEntry('minimal-safe-diff'),
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* check_bridge_derivation.ts — every generated bridge file hash-derives from
|
|
4
|
+
* its `src/` source (road-to-ecosystem-harvest-skill-quality-gates Phase 4,
|
|
5
|
+
* Source AA).
|
|
6
|
+
*
|
|
7
|
+
* Two deterministic legs:
|
|
8
|
+
*
|
|
9
|
+
* 1. **Symlink leg** — every symlink under a bridge root must resolve to an
|
|
10
|
+
* existing target inside the repo, under one of the derivation sources
|
|
11
|
+
* (`dist/agent-src/`, `src/`, `AGENTS.md`). A broken link or a link
|
|
12
|
+
* escaping the derivation set is a bridge file that does NOT derive.
|
|
13
|
+
* 2. **Regeneration leg** — snapshot the bridge trees (content hash for
|
|
14
|
+
* regular files, target string for symlinks), re-run the generator
|
|
15
|
+
* (`condense --generate-tools`), snapshot again. Any added / removed /
|
|
16
|
+
* changed entry means the tree on disk did not derive from the current
|
|
17
|
+
* source (hand-edit, stale generation, or a nondeterministic generator).
|
|
18
|
+
*
|
|
19
|
+
* Run AFTER `task generate-tools` (CI: the sync-consistency job). Exit 1 on
|
|
20
|
+
* the first drifted artefact, naming it.
|
|
21
|
+
*/
|
|
22
|
+
import { execFileSync } from 'node:child_process';
|
|
23
|
+
import * as crypto from 'node:crypto';
|
|
24
|
+
import * as fs from 'node:fs';
|
|
25
|
+
import * as path from 'node:path';
|
|
26
|
+
import { fileURLToPath } from 'node:url';
|
|
27
|
+
|
|
28
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
29
|
+
const REPO_ROOT = path.join(path.dirname(_HERE), '..', '..');
|
|
30
|
+
|
|
31
|
+
/** Bridge roots covered by the derivation guarantee (repo-relative; dirs or files). */
|
|
32
|
+
export const BRIDGE_ROOTS: readonly string[] = [
|
|
33
|
+
'.claude/rules',
|
|
34
|
+
'.claude/skills',
|
|
35
|
+
'.claude/agents',
|
|
36
|
+
'.claude/personas',
|
|
37
|
+
'.claude/user-types',
|
|
38
|
+
'.cursor/rules',
|
|
39
|
+
'.cursor/commands',
|
|
40
|
+
'.windsurf/rules',
|
|
41
|
+
'.windsurf/workflows',
|
|
42
|
+
'.clinerules',
|
|
43
|
+
'.windsurfrules',
|
|
44
|
+
'GEMINI.md',
|
|
45
|
+
'.augment',
|
|
46
|
+
] as const;
|
|
47
|
+
|
|
48
|
+
/** Repo-relative prefixes a bridge symlink may resolve into. */
|
|
49
|
+
const DERIVATION_SOURCES = ['dist/agent-src', 'src', 'AGENTS.md'] as const;
|
|
50
|
+
|
|
51
|
+
function _walk(p: string, out: string[]): void {
|
|
52
|
+
const st = fs.lstatSync(p);
|
|
53
|
+
if (st.isSymbolicLink() || st.isFile()) {
|
|
54
|
+
out.push(p);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (st.isDirectory()) {
|
|
58
|
+
for (const name of fs.readdirSync(p).sort()) {
|
|
59
|
+
_walk(path.join(p, name), out);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Map of entry → `link:<target>` (symlinks) or `sha256:<hash>` (files) under `root`. */
|
|
65
|
+
export function snapshot_tree(root: string): Record<string, string> {
|
|
66
|
+
const out: Record<string, string> = {};
|
|
67
|
+
if (!fs.existsSync(root) && !_isSymlinkRaw(root)) return out;
|
|
68
|
+
const entries: string[] = [];
|
|
69
|
+
_walk(root, entries);
|
|
70
|
+
for (const p of entries) {
|
|
71
|
+
const rel = path.relative(path.dirname(root), p);
|
|
72
|
+
const st = fs.lstatSync(p);
|
|
73
|
+
if (st.isSymbolicLink()) {
|
|
74
|
+
out[rel] = `link:${fs.readlinkSync(p)}`;
|
|
75
|
+
} else {
|
|
76
|
+
out[rel] = `sha256:${crypto.createHash('sha256').update(fs.readFileSync(p)).digest('hex')}`;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function _isSymlinkRaw(p: string): boolean {
|
|
83
|
+
try {
|
|
84
|
+
return fs.lstatSync(p).isSymbolicLink();
|
|
85
|
+
} catch {
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Human-readable drift lines between two snapshots; empty = identical. */
|
|
91
|
+
export function diff_snapshots(before: Record<string, string>, after: Record<string, string>): string[] {
|
|
92
|
+
const errors: string[] = [];
|
|
93
|
+
for (const k of Object.keys(before).sort()) {
|
|
94
|
+
if (!(k in after)) errors.push(`${k}: present before regeneration, absent after (stale bridge file)`);
|
|
95
|
+
else if (before[k] !== after[k]) errors.push(`${k}: content differs from regenerated output (does not derive from src/)`);
|
|
96
|
+
}
|
|
97
|
+
for (const k of Object.keys(after).sort()) {
|
|
98
|
+
if (!(k in before)) errors.push(`${k}: produced by regeneration but absent before (generation was stale)`);
|
|
99
|
+
}
|
|
100
|
+
return errors;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Every symlink under `root` must resolve inside `repoRoot` to a derivation source. */
|
|
104
|
+
export function validate_symlinks(root: string, repoRoot: string): string[] {
|
|
105
|
+
const errors: string[] = [];
|
|
106
|
+
if (!fs.existsSync(root) && !_isSymlinkRaw(root)) return errors;
|
|
107
|
+
const entries: string[] = [];
|
|
108
|
+
_walk(root, entries);
|
|
109
|
+
for (const p of entries) {
|
|
110
|
+
if (!fs.lstatSync(p).isSymbolicLink()) continue;
|
|
111
|
+
let real: string;
|
|
112
|
+
try {
|
|
113
|
+
real = fs.realpathSync(p);
|
|
114
|
+
} catch {
|
|
115
|
+
errors.push(`${path.relative(repoRoot, p)}: broken symlink (target missing)`);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
const rel = path.relative(fs.realpathSync(repoRoot), real);
|
|
119
|
+
if (rel.startsWith('..')) {
|
|
120
|
+
errors.push(`${path.relative(repoRoot, p)}: symlink escapes the repository (${real})`);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const ok = DERIVATION_SOURCES.some((s) => rel === s || rel.startsWith(`${s}${path.sep}`));
|
|
124
|
+
if (!ok) {
|
|
125
|
+
errors.push(`${path.relative(repoRoot, p)}: symlink target '${rel}' is outside the derivation sources (${DERIVATION_SOURCES.join(', ')})`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return errors;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function run(repoRoot: string): string[] {
|
|
132
|
+
const errors: string[] = [];
|
|
133
|
+
const roots = BRIDGE_ROOTS.map((r) => path.join(repoRoot, r));
|
|
134
|
+
|
|
135
|
+
const before: Record<string, Record<string, string>> = {};
|
|
136
|
+
for (const r of roots) before[r] = snapshot_tree(r);
|
|
137
|
+
|
|
138
|
+
// Regenerate in a child process so MODULE_STATE resolves exactly as the
|
|
139
|
+
// real pipeline does (no partial in-process state). `--generate-tools`
|
|
140
|
+
// covers the host trees; `--project-augment` covers `.augment` (it is
|
|
141
|
+
// projected by the sync path, not by generate-tools).
|
|
142
|
+
for (const flag of ['--generate-tools', '--project-augment']) {
|
|
143
|
+
execFileSync(path.join(repoRoot, 'scripts-run'), ['src/scripts/condense', flag], {
|
|
144
|
+
cwd: repoRoot,
|
|
145
|
+
stdio: 'pipe',
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
for (const r of roots) {
|
|
150
|
+
const rel = path.relative(repoRoot, r);
|
|
151
|
+
for (const e of diff_snapshots(before[r] ?? {}, snapshot_tree(r))) {
|
|
152
|
+
errors.push(`${rel}: ${e}`);
|
|
153
|
+
}
|
|
154
|
+
for (const e of validate_symlinks(r, repoRoot)) {
|
|
155
|
+
errors.push(e);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return errors;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const isMain = process.argv[1] !== undefined && path.resolve(process.argv[1]) === path.resolve(_HERE);
|
|
162
|
+
if (isMain) {
|
|
163
|
+
const idx = process.argv.indexOf('--root');
|
|
164
|
+
const root = idx !== -1 ? (process.argv[idx + 1] ?? REPO_ROOT) : REPO_ROOT;
|
|
165
|
+
const errors = run(root);
|
|
166
|
+
if (errors.length > 0) {
|
|
167
|
+
for (const e of errors) process.stderr.write(`❌ ${e}\n`);
|
|
168
|
+
process.exit(1);
|
|
169
|
+
}
|
|
170
|
+
process.stdout.write('✅ bridge trees derive from src/ (symlink targets valid, regeneration is a fixpoint)\n');
|
|
171
|
+
process.exit(0);
|
|
172
|
+
}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* check_consumer_scope_flip.ts — deterministic held-quality verification for
|
|
3
|
+
* the consumer-scoped rule-projection default flip
|
|
4
|
+
* (road-to-request-scoped-rule-load Phase 1 human gate, approved 2026-07-13).
|
|
5
|
+
*
|
|
6
|
+
* The LLM-paired judge path is CLOSED-BY-DIAGNOSIS (docs/benchmark.md § Length-
|
|
7
|
+
* neutral judge RERUN); the recorded re-open path is deterministic verification.
|
|
8
|
+
* For THIS lever it is exact, not sampled: scoping only REMOVES whole rules —
|
|
9
|
+
* it never edits a shipped body — so output quality on the consumer surface is
|
|
10
|
+
* held by construction iff NO rule the consumer golden set exercises is
|
|
11
|
+
* removed. That is a decidable set inclusion, checked here:
|
|
12
|
+
*
|
|
13
|
+
* 1. The template's flipped `projection.rule_workspaces` default excludes
|
|
14
|
+
* ONLY exclusively-maintainer rules (workspaces == [agent-config-
|
|
15
|
+
* maintainer]) plus the recorded compat exclusion (source-of-truth.md).
|
|
16
|
+
* 2. Every rule tagged by any golden-set task (internal/bench/corpora/
|
|
17
|
+
* token-quality-golden.yaml, 90/90 labelled) still arrives, UNLESS that
|
|
18
|
+
* rule is exclusively-maintainer (then the task is maintainer-scope by
|
|
19
|
+
* the schema's own definition: consumer = kernel + every rule not
|
|
20
|
+
* exclusively maintainer).
|
|
21
|
+
* 3. The before/after evidence (rule count + cl100k tokens via
|
|
22
|
+
* _lib/token_count.ts) is recomputed over the SHIPPED dist rule tree and
|
|
23
|
+
* written to internal/bench/reports/.
|
|
24
|
+
*
|
|
25
|
+
* Exit 0 = flip verified; exit 1 = violation (a consumer-relevant rule would
|
|
26
|
+
* drop — the flip must not ship).
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import * as fs from 'node:fs';
|
|
30
|
+
import * as path from 'node:path';
|
|
31
|
+
|
|
32
|
+
import { parse as parseYaml } from 'yaml';
|
|
33
|
+
|
|
34
|
+
import { excludedRuleBasenames, ruleScopeFromSettings } from '../install/rule_scope.js';
|
|
35
|
+
|
|
36
|
+
export const RULES_DIR = 'dist/agent-src/rules';
|
|
37
|
+
export const TEMPLATE = 'src/config/agent-settings.template.yml';
|
|
38
|
+
export const GOLDEN = 'internal/bench/corpora/token-quality-golden.yaml';
|
|
39
|
+
export const REPORT = 'internal/bench/reports/2026-07-13-consumer-scoped-default-flip.json';
|
|
40
|
+
|
|
41
|
+
/** workspaces: frontmatter list of a rule file ([] when untagged). */
|
|
42
|
+
export function rule_workspaces(rulePath: string): string[] {
|
|
43
|
+
const src = fs.readFileSync(rulePath, 'utf-8');
|
|
44
|
+
const m = src.match(/^---\n([\s\S]*?)\n---/);
|
|
45
|
+
if (!m) return [];
|
|
46
|
+
const meta = parseYaml(m[1] ?? '') as Record<string, unknown>;
|
|
47
|
+
const ws = meta['workspaces'];
|
|
48
|
+
return Array.isArray(ws) ? ws.map((w) => String(w)) : [];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function is_exclusively_maintainer(ws: string[]): boolean {
|
|
52
|
+
return ws.length === 1 && ws[0] === 'agent-config-maintainer';
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface FlipVerdict {
|
|
56
|
+
scope: string[];
|
|
57
|
+
excluded: string[];
|
|
58
|
+
violations: string[]; // excluded rules that are NOT exclusively-maintainer/compat
|
|
59
|
+
golden_rules_total: number;
|
|
60
|
+
golden_rules_dropped: string[]; // golden-tagged rules that would drop (non-maintainer)
|
|
61
|
+
pass: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function verify_flip(repoRoot = '.'): FlipVerdict {
|
|
65
|
+
const template = parseYaml(fs.readFileSync(path.join(repoRoot, TEMPLATE), 'utf-8')) as Record<string, unknown>;
|
|
66
|
+
const scope = ruleScopeFromSettings(template);
|
|
67
|
+
if (scope.workspaces === null) {
|
|
68
|
+
throw new Error('template rule_workspaces is empty — nothing flipped, nothing to verify');
|
|
69
|
+
}
|
|
70
|
+
const rulesDir = path.join(repoRoot, RULES_DIR);
|
|
71
|
+
const excluded = excludedRuleBasenames(rulesDir, scope);
|
|
72
|
+
|
|
73
|
+
const violations: string[] = [];
|
|
74
|
+
for (const name of excluded) {
|
|
75
|
+
if (name === 'source-of-truth.md') continue; // recorded compat exclusion
|
|
76
|
+
const ws = rule_workspaces(path.join(rulesDir, name));
|
|
77
|
+
if (!is_exclusively_maintainer(ws)) {
|
|
78
|
+
violations.push(`${name} [${ws.join(', ')}]`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const golden = parseYaml(fs.readFileSync(path.join(repoRoot, GOLDEN), 'utf-8')) as {
|
|
83
|
+
tasks: Array<{ rules?: string[] }>;
|
|
84
|
+
};
|
|
85
|
+
const goldenRules = new Set<string>();
|
|
86
|
+
for (const t of golden.tasks) for (const r of t.rules ?? []) goldenRules.add(r);
|
|
87
|
+
const excludedIds = new Set(excluded.map((n) => n.replace(/\.md$/, '')));
|
|
88
|
+
const goldenDropped: string[] = [];
|
|
89
|
+
for (const r of goldenRules) {
|
|
90
|
+
if (!excludedIds.has(r)) continue;
|
|
91
|
+
const ws = rule_workspaces(path.join(rulesDir, `${r}.md`));
|
|
92
|
+
if (!is_exclusively_maintainer(ws) && r !== 'source-of-truth') {
|
|
93
|
+
goldenDropped.push(r);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
scope: [...scope.workspaces],
|
|
99
|
+
excluded,
|
|
100
|
+
violations,
|
|
101
|
+
golden_rules_total: goldenRules.size,
|
|
102
|
+
golden_rules_dropped: goldenDropped,
|
|
103
|
+
pass: violations.length === 0 && goldenDropped.length === 0,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async function main(): Promise<number> {
|
|
108
|
+
const v = verify_flip();
|
|
109
|
+
const { gpt_tokens } = await import('./_lib/token_count.js');
|
|
110
|
+
const excludedSet = new Set(v.excluded);
|
|
111
|
+
let legacyRules = 0;
|
|
112
|
+
let legacyTok = 0;
|
|
113
|
+
let scopedRules = 0;
|
|
114
|
+
let scopedTok = 0;
|
|
115
|
+
for (const name of fs.readdirSync(RULES_DIR).sort()) {
|
|
116
|
+
if (!name.endsWith('.md')) continue;
|
|
117
|
+
const body = fs.readFileSync(path.join(RULES_DIR, name), 'utf-8');
|
|
118
|
+
const tok = gpt_tokens(body).tokens;
|
|
119
|
+
// legacy-all baseline mirrors the 2026-07-08 report: compat exclusion applies on both arms
|
|
120
|
+
if (name === 'source-of-truth.md') continue;
|
|
121
|
+
legacyRules += 1;
|
|
122
|
+
legacyTok += tok;
|
|
123
|
+
if (!excludedSet.has(name)) {
|
|
124
|
+
scopedRules += 1;
|
|
125
|
+
scopedTok += tok;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const report = {
|
|
129
|
+
schema_version: 1,
|
|
130
|
+
kind: 'consumer-scoped-default-flip-verification',
|
|
131
|
+
date: '2026-07-13',
|
|
132
|
+
roadmap: 'road-to-request-scoped-rule-load Phase 1 (human gate, approved in-session)',
|
|
133
|
+
tokenizer: 'cl100k_base (js-tiktoken via _lib/token_count.ts)',
|
|
134
|
+
held_quality_arm:
|
|
135
|
+
'deterministic set-inclusion (LLM judging closed-by-diagnosis): scoping removes whole '
|
|
136
|
+
+ 'rules only; verified that every removed rule is exclusively-maintainer (+ the recorded '
|
|
137
|
+
+ 'source-of-truth compat exclusion) and that zero golden-set-exercised consumer rules drop '
|
|
138
|
+
+ '— consumer-surface quality held by construction.',
|
|
139
|
+
scope: { rule_workspaces: v.scope },
|
|
140
|
+
excluded_rules: v.excluded,
|
|
141
|
+
violations: v.violations,
|
|
142
|
+
golden: { rules_tagged: v.golden_rules_total, dropped_non_maintainer: v.golden_rules_dropped },
|
|
143
|
+
legacy_all: { rules: legacyRules, gpt_tokens: legacyTok },
|
|
144
|
+
scoped: { rules: scopedRules, gpt_tokens: scopedTok },
|
|
145
|
+
delta: {
|
|
146
|
+
rules: legacyRules - scopedRules,
|
|
147
|
+
gpt_tokens: legacyTok - scopedTok,
|
|
148
|
+
tokens_pct: Number((((legacyTok - scopedTok) / legacyTok) * 100).toFixed(1)),
|
|
149
|
+
},
|
|
150
|
+
pass: v.pass,
|
|
151
|
+
};
|
|
152
|
+
fs.mkdirSync(path.dirname(REPORT), { recursive: true });
|
|
153
|
+
fs.writeFileSync(REPORT, `${JSON.stringify(report, null, 1)}\n`);
|
|
154
|
+
process.stdout.write(`excluded ${v.excluded.length} rules under scope [${v.scope.join(', ')}]\n`);
|
|
155
|
+
if (v.violations.length > 0) {
|
|
156
|
+
process.stdout.write(`❌ NON-MAINTAINER rules would drop: ${v.violations.join(' · ')}\n`);
|
|
157
|
+
}
|
|
158
|
+
if (v.golden_rules_dropped.length > 0) {
|
|
159
|
+
process.stdout.write(`❌ golden-set rules would drop: ${v.golden_rules_dropped.join(' · ')}\n`);
|
|
160
|
+
}
|
|
161
|
+
process.stdout.write(
|
|
162
|
+
`legacy-all ${legacyRules} rules / ${legacyTok} tok → scoped ${scopedRules} rules / ${scopedTok} tok `
|
|
163
|
+
+ `(−${legacyTok - scopedTok} tok, −${report.delta.tokens_pct}%)\n`,
|
|
164
|
+
);
|
|
165
|
+
process.stdout.write(v.pass ? `✅ flip verified — wrote ${REPORT}\n` : '❌ flip NOT verified\n');
|
|
166
|
+
return v.pass ? 0 : 1;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const isMain = process.argv[1] !== undefined
|
|
170
|
+
&& path.resolve(process.argv[1]) === path.resolve(new URL(import.meta.url).pathname);
|
|
171
|
+
if (isMain) {
|
|
172
|
+
main().then(
|
|
173
|
+
(code) => process.exit(code),
|
|
174
|
+
(err) => {
|
|
175
|
+
process.stderr.write(`${err instanceof Error ? err.stack ?? err.message : String(err)}\n`);
|
|
176
|
+
process.exit(1);
|
|
177
|
+
},
|
|
178
|
+
);
|
|
179
|
+
}
|