@event4u/agent-config 9.14.0 → 9.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +118 -99
- package/README.md +8 -0
- package/dist/agent-src/commands/agents/audit.md +7 -7
- package/dist/agent-src/commands/agents/optimize.md +1 -1
- package/dist/agent-src/commands/analyze/reference-repo.md +1 -1
- package/dist/agent-src/commands/chat-history.md +1 -0
- package/dist/agent-src/commands/context.md +1 -0
- package/dist/agent-src/commands/council.md +1 -1
- package/dist/agent-src/commands/ghostwriter.md +1 -1
- package/dist/agent-src/commands/memory.md +2 -1
- package/dist/agent-src/commands/module.md +1 -0
- package/dist/agent-src/commands/optimize/agents-dir.md +1 -1
- package/dist/agent-src/commands/optimize/deep.md +5 -2
- package/dist/agent-src/commands/optimize/skills.md +5 -5
- package/dist/agent-src/commands/override.md +1 -0
- package/dist/agent-src/commands/prediction-pool.md +4 -4
- package/dist/agent-src/commands/research/deep.md +1 -1
- package/dist/agent-src/commands/roadmap.md +2 -1
- package/dist/agent-src/commands/rule-compliance-audit.md +1 -1
- package/dist/agent-src/commands/tdd.md +1 -0
- package/dist/agent-src/commands/tests.md +1 -0
- package/dist/agent-src/commands/upstream-contribute.md +5 -5
- package/dist/agent-src/commands/video/from-script.md +1 -1
- package/dist/agent-src/commands/video/from-song.md +1 -1
- package/dist/agent-src/commands/video/scene.md +1 -1
- package/dist/agent-src/commands/video/stitch.md +1 -1
- package/dist/agent-src/commands/video/storyboard.md +1 -1
- package/dist/agent-src/commands/worktree/create.md +2 -1
- package/dist/agent-src/contexts/augment-infrastructure.md +1 -1
- package/dist/agent-src/contexts/contracts/artifact-engagement-flow.md +5 -5
- package/dist/agent-src/contexts/contracts/command-suggestion-flow.md +3 -3
- package/dist/agent-src/contexts/contracts/emergency-triage-block.md +2 -2
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +4 -3
- package/dist/agent-src/contexts/execution/subagent-routing.md +19 -0
- package/dist/agent-src/contexts/override-system.md +1 -1
- package/dist/agent-src/ghostwriter/fictional-fixture-v1.md +1 -1
- package/dist/agent-src/rules/augment-edit-discipline.md +1 -1
- package/dist/agent-src/rules/autonomous-execution.md +2 -1
- package/dist/agent-src/rules/brand-consistency.md +1 -0
- package/dist/agent-src/rules/brand-source-of-truth.md +1 -0
- package/dist/agent-src/rules/code-comment-discipline.md +2 -0
- package/dist/agent-src/rules/communication-through-line.md +1 -0
- package/dist/agent-src/rules/content-quoting-floor.md +1 -0
- package/dist/agent-src/rules/cross-source-consistency.md +0 -1
- package/dist/agent-src/rules/downstream-changes.md +1 -0
- package/dist/agent-src/rules/external-code-graph-interop.md +1 -0
- package/dist/agent-src/rules/external-reference-deep-dive.md +1 -0
- package/dist/agent-src/rules/fast-path-marker-visibility.md +1 -0
- package/dist/agent-src/rules/finance-safety-floor.md +0 -1
- package/dist/agent-src/rules/icon-consistency.md +1 -0
- package/dist/agent-src/rules/image-likeness-and-rights.md +1 -0
- package/dist/agent-src/rules/improve-before-implement.md +1 -1
- package/dist/agent-src/rules/invite-challenge.md +1 -0
- package/dist/agent-src/rules/lethal-trifecta-guard.md +1 -0
- package/dist/agent-src/rules/low-impact-corpus-privacy-floor.md +1 -0
- package/dist/agent-src/rules/markdown-safe-codeblocks.md +1 -0
- package/dist/agent-src/rules/media-governance-routing.md +4 -3
- package/dist/agent-src/rules/media-sync-ground-truth.md +1 -0
- package/dist/agent-src/rules/no-attribution-footers.md +1 -0
- package/dist/agent-src/rules/no-decorative-emojis-in-git-surfaces.md +1 -0
- package/dist/agent-src/rules/no-pr-progress-comments.md +1 -0
- package/dist/agent-src/rules/onboarding-gate.md +1 -0
- package/dist/agent-src/rules/output-discipline.md +1 -0
- package/dist/agent-src/rules/prefer-enums-over-literals.md +1 -0
- package/dist/agent-src/rules/preservation-guard.md +1 -0
- package/dist/agent-src/rules/question-not-instruction.md +1 -0
- package/dist/agent-src/rules/roadmap-ci-steps-policy.md +1 -0
- package/dist/agent-src/rules/role-mode-adherence.md +1 -0
- package/dist/agent-src/rules/runtime-safety.md +1 -0
- package/dist/agent-src/rules/security-sensitive-stop.md +1 -0
- package/dist/agent-src/rules/session-canary.md +9 -2
- package/dist/agent-src/rules/source-confidentiality.md +1 -0
- package/dist/agent-src/rules/source-of-truth.md +1 -0
- package/dist/agent-src/rules/spreadsheet-source-quality.md +1 -0
- package/dist/agent-src/rules/think-before-action.md +1 -0
- package/dist/agent-src/rules/token-budget-discipline.md +1 -0
- package/dist/agent-src/rules/token-efficiency.md +1 -0
- package/dist/agent-src/rules/token-optimizer-maintenance.md +5 -5
- package/dist/agent-src/rules/tool-safety.md +1 -0
- package/dist/agent-src/rules/untrusted-input-defense.md +2 -0
- package/dist/agent-src/rules/user-interaction.md +1 -0
- package/dist/agent-src/rules/user-interrupt-priority.md +1 -0
- package/dist/agent-src/skills/adr-create/SKILL.md +1 -1
- package/dist/agent-src/skills/check-refs/SKILL.md +1 -1
- package/dist/agent-src/skills/competitive-positioning/SKILL.md +1 -1
- package/dist/agent-src/skills/condense-memory/SKILL.md +1 -1
- package/dist/agent-src/skills/context-authoring/SKILL.md +1 -1
- package/dist/agent-src/skills/copilot-agents-optimization/SKILL.md +1 -1
- package/dist/agent-src/skills/description-assist/SKILL.md +1 -1
- package/dist/agent-src/skills/github-ci/SKILL.md +1 -0
- package/dist/agent-src/skills/guideline-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/lint-skills/SKILL.md +1 -1
- package/dist/agent-src/skills/readme-reviewer/SKILL.md +1 -1
- package/dist/agent-src/skills/roadmap-management/SKILL.md +8 -1
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +1 -1
- package/dist/agent-src/skills/skill-improvement-pipeline/SKILL.md +1 -1
- package/dist/agent-src/skills/skill-writing/SKILL.md +4 -4
- package/dist/agent-src/skills/token-optimizer/SKILL.md +4 -4
- package/dist/agent-src/skills/upstream-contribute/SKILL.md +6 -6
- package/dist/agent-src/skills/using-git-worktrees/SKILL.md +25 -0
- package/dist/agent-src/templates/AGENTS.md +1 -1
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/command.md +1 -1
- package/dist/agent-src/templates/copilot-instructions.md +3 -3
- package/dist/agent-src/templates/rule.md +2 -2
- package/dist/agent-src/templates/scripts/work_engine/orchestration.ts +1 -1
- package/dist/agent-src/templates/skill-archive-note.md +5 -5
- package/dist/agent-src/templates/skill.md +1 -1
- package/dist/cli/agent-config.js +66 -19
- package/dist/cli/agent-config.js.map +1 -1
- package/dist/cli/main.js +9 -4
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/mcp/content.js +1 -1
- package/dist/cli/mcp/content.js.map +1 -1
- package/dist/cli/python/workspace_drive.js +2 -0
- package/dist/cli/python/workspace_drive.js.map +1 -1
- package/dist/cli/python/workspace_drive_health.js +2 -0
- package/dist/cli/python/workspace_drive_health.js.map +1 -1
- package/dist/cli/python/workspace_hosts.js +2 -0
- package/dist/cli/python/workspace_hosts.js.map +1 -1
- package/dist/cli/python/workspace_sessions.js +2 -0
- package/dist/cli/python/workspace_sessions.js.map +1 -1
- package/dist/cli/registry.js +1 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/cli-delegate/{chunk-A3HJ6NTI.js → chunk-63WSP6YZ.js} +2 -1
- package/dist/cli-delegate/{chunk-EJYIGZ3T.js → chunk-BKO2AZ7Z.js} +1 -1
- package/dist/cli-delegate/{chunk-GCVFLBP2.js → chunk-H574KOUH.js} +338 -494
- package/dist/cli-delegate/chunk-NXFWFAYV.js +339 -0
- package/dist/cli-delegate/{chunk-TEGJU2JX.js → chunk-ZXXJLL5K.js} +2 -2
- package/dist/cli-delegate/cmd_conformance.js +29 -23
- package/dist/cli-delegate/cmd_doctor.js +3 -3
- package/dist/cli-delegate/cmd_export.js +4 -4
- package/dist/cli-delegate/cmd_preflight.js +2 -2
- package/dist/cli-delegate/cmd_prune.js +4 -4
- package/dist/cli-delegate/cmd_settings_migrate.js +4 -4
- package/dist/cli-delegate/cmd_sync.js +4 -4
- package/dist/cli-delegate/cmd_uninstall.js +4 -4
- package/dist/cli-delegate/cmd_validate.js +4 -4
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +127 -95
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +729 -430
- package/dist/install/install.mjs +52 -11
- package/dist/install/rule_scope.js +18 -2
- package/dist/install/rule_scope.js.map +1 -1
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +518 -411
- package/dist/router.json +1 -1
- package/dist/scripts/_lib/claude_settings_hooks.js +30 -5
- package/dist/scripts/_lib/claude_settings_hooks.js.map +1 -1
- package/dist/scripts/_lib/scoped_projection.js +223 -0
- package/dist/scripts/_lib/scoped_projection.js.map +1 -0
- package/dist/scripts/_lib/surface_tiers.js +125 -0
- package/dist/scripts/_lib/surface_tiers.js.map +1 -0
- package/dist/scripts/_lib/token_count.js +166 -0
- package/dist/scripts/_lib/token_count.js.map +1 -0
- package/dist/scripts/_lib/trigger_eval_floors.js +8 -0
- package/dist/scripts/_lib/trigger_eval_floors.js.map +1 -1
- package/dist/scripts/condense.js +139 -9
- package/dist/scripts/condense.js.map +1 -1
- package/dist/scripts/project_thin_rules.js +450 -0
- package/dist/scripts/project_thin_rules.js.map +1 -0
- package/dist/server/routes/workspace.js +3 -1
- package/dist/server/routes/workspace.js.map +1 -1
- package/dist/server/schemas/settings.js +9 -3
- package/dist/server/schemas/settings.js.map +1 -1
- package/dist/shared/userMd/utils.js +1 -1
- package/dist/shared/userMd/utils.js.map +1 -1
- package/dist/ui/assets/index-DmR0mbk4.js +26 -0
- package/dist/ui/assets/index-DmR0mbk4.js.map +1 -0
- package/dist/ui/index.html +1 -1
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +1 -1
- package/package.json +14 -14
- package/src/agent-src/templates/AGENTS.md +1 -1
- package/src/agent-src/templates/scripts/work_engine/orchestration.ts +1 -1
- package/src/config/agent-settings.template.yml +61 -11
- package/src/config/budget-routing.json +7 -0
- package/src/config/ci-local-parity.yml +8 -0
- package/src/config/discovery/unassigned-artefacts.yml +23 -25
- package/src/config/evaluator-budgets.json +35 -5
- package/src/config/gate-coverage.yml +147 -0
- package/src/config/gate-violation-baselines.json +25 -0
- package/src/install/rule_scope.ts +27 -2
- package/src/scripts/_cli/cmd_conformance.ts +13 -24
- package/src/scripts/_dispatch.bash +10 -0
- package/src/scripts/_lib/claude_settings_hooks.ts +30 -4
- package/src/scripts/_lib/gate_baseline.ts +213 -0
- package/src/scripts/_lib/scoped_projection.ts +50 -0
- package/src/scripts/_lib/tier_budget_routing.ts +360 -0
- package/src/scripts/_lib/tool_adapter_registry.ts +8 -1
- package/src/scripts/_lib/trigger_eval_floors.ts +9 -0
- package/src/scripts/_lib/trigger_routers.ts +17 -5
- package/src/scripts/adr/regenerate_index.ts +4 -3
- package/src/scripts/ai-video/lib/parse-blueprint.sh +1 -1
- package/src/scripts/ai_council/compile_corpus.ts +1 -1
- package/src/scripts/anchor_scoring_run.ts +1 -1
- package/src/scripts/audit_cloud_compatibility.ts +37 -2
- package/src/scripts/audit_initial_context.ts +53 -19
- package/src/scripts/audit_likelihood.ts +31 -2
- package/src/scripts/audit_user_type_axis.ts +58 -11
- package/src/scripts/bench_hook_latency.ts +172 -41
- package/src/scripts/bench_persona_placebo.ts +1 -1
- package/src/scripts/bench_quality_run.ts +1 -1
- package/src/scripts/build_mcp_catalog.ts +12 -1
- package/src/scripts/check_always_budget.ts +122 -11
- package/src/scripts/check_augment_description_cap.ts +24 -3
- package/src/scripts/check_ci_strict_superset.ts +150 -0
- package/src/scripts/check_cli_registry_budget_sync.ts +168 -0
- package/src/scripts/check_cluster_patterns.ts +100 -0
- package/src/scripts/check_consumer_scope_flip.ts +26 -4
- package/src/scripts/check_evaluator_budgets.ts +198 -19
- package/src/scripts/check_gate_coverage.ts +338 -2
- package/src/scripts/check_generator_output_coverage.ts +2 -1
- package/src/scripts/check_iron_law_prominence.ts +1 -1
- package/src/scripts/check_kernel_rule_bundle.ts +1 -1
- package/src/scripts/check_no_new_legacy_path.ts +167 -3
- package/src/scripts/check_pack_size.ts +18 -4
- package/src/scripts/check_portability.ts +37 -7
- package/src/scripts/check_references.ts +19 -3
- package/src/scripts/check_release_pr_shape.ts +13 -0
- package/src/scripts/check_token_quality_golden.ts +1 -1
- package/src/scripts/check_token_regression.ts +9 -2
- package/src/scripts/condense.ts +155 -12
- package/src/scripts/cost/budget.mjs +77 -2
- package/src/scripts/evaluator_umbrella.sh +34 -1
- package/src/scripts/generate_capability_matrix.ts +8 -0
- package/src/scripts/generate_router_coverage_from_matrix.ts +155 -0
- package/src/scripts/hook_manifest.yaml +17 -3
- package/src/scripts/hooks/block_config_weakening.ts +337 -0
- package/src/scripts/hooks/concern_registry.ts +2 -0
- package/src/scripts/hooks/probe_throttle.ts +75 -0
- package/src/scripts/install-hooks.sh +24 -2
- package/src/scripts/install.ts +2 -2
- package/src/scripts/lint_agents_md.ts +9 -3
- package/src/scripts/lint_archived_skills.ts +1 -1
- package/src/scripts/lint_artefact_frontmatter.ts +185 -37
- package/src/scripts/lint_command_routing.ts +21 -52
- package/src/scripts/lint_command_verbs.ts +47 -28
- package/src/scripts/lint_context_spine_usage.ts +38 -12
- package/src/scripts/lint_framework_leakage.ts +25 -6
- package/src/scripts/lint_handoffs.ts +49 -2
- package/src/scripts/lint_media_policy_linkage.ts +62 -30
- package/src/scripts/lint_namespace.ts +92 -7
- package/src/scripts/lint_new_skill_gate.ts +39 -31
- package/src/scripts/lint_originality_shingles.ts +7 -1
- package/src/scripts/lint_override_kernel_guard.ts +21 -3
- package/src/scripts/lint_pack_boundaries.ts +461 -91
- package/src/scripts/lint_pack_dependencies.ts +24 -57
- package/src/scripts/lint_persona_governance.ts +1 -1
- package/src/scripts/lint_role_experiences.ts +34 -5
- package/src/scripts/lint_trigger_precision.ts +142 -0
- package/src/scripts/mcp_server/consumer_tool_catalog.json +12 -201
- package/src/scripts/mcp_server/tools.ts +1 -1
- package/src/scripts/prepack-check.mjs +40 -1
- package/src/scripts/prepack_router_targets.d.mts +7 -0
- package/src/scripts/prepack_router_targets.mjs +93 -0
- package/src/scripts/print_required_checks.ts +72 -18
- package/src/scripts/profile_staleness_hook.ts +12 -0
- package/src/scripts/record_evaluator_measurements.mjs +105 -0
- package/src/scripts/recruit_preflight.sh +1 -1
- package/src/scripts/release.ts +214 -7
- package/src/scripts/router_target_paths.d.mts +4 -0
- package/src/scripts/router_target_paths.mjs +69 -0
- package/src/scripts/router_telemetry.ts +52 -1
- package/src/scripts/routing_doctor.ts +560 -0
- package/src/scripts/rule_trigger_eval.ts +403 -0
- package/src/scripts/schemas/rule.schema.json +5 -1
- package/src/scripts/session_canary_hook.ts +96 -5
- package/src/scripts/skill_linter.ts +9 -28
- package/src/scripts/smoke/skills.sh +1 -1
- package/src/scripts/sweep_dead_scan_roots.ts +356 -10
- package/src/scripts/trigger_coverage.ts +13 -2
- package/src/scripts/update_counts.ts +1 -1
- package/src/scripts/validate_telegraph_carveouts.ts +1 -1
- package/src/scripts/wrapper_freshness_hook.ts +14 -0
- package/src/server/schemas/settings.ts +19 -5
- package/src/shared/userMd/utils.ts +1 -1
- package/dist/cli-delegate/chunk-TJIAM5FW.js +0 -133
- package/dist/ui/assets/index-BpSQt5vm.js +0 -40
- package/dist/ui/assets/index-BpSQt5vm.js.map +0 -1
|
@@ -59,9 +59,22 @@ const MONOLITH_TOOLS = ['.windsurfrules'];
|
|
|
59
59
|
// costing?" question as the rule/description surfaces, for the MCP slice. The
|
|
60
60
|
// source-of-truth catalog is the one shipped by this package's MCP server.
|
|
61
61
|
const MCP_CATALOG = path.join(REPO_ROOT, 'src', 'scripts', 'mcp_server', 'consumer_tool_catalog.json');
|
|
62
|
-
//
|
|
63
|
-
//
|
|
62
|
+
// The catalog is served by TWO transports with two different tool lists, so
|
|
63
|
+
// pricing it once produced a number no client ever loads:
|
|
64
|
+
//
|
|
65
|
+
// - stdio (`src/scripts/mcp_server/`) registers only the implemented tools
|
|
66
|
+
// (`REGISTRY` in mcp_server/tools.ts) — the 12 discovery stubs are
|
|
67
|
+
// deliberately unregistered (ADR-132), so a local client sees 19.
|
|
68
|
+
// - worker (`internal/workers/mcp/`) maps EVERY catalog entry into
|
|
69
|
+
// `tools/list` (`stubs.ts::listTools`), tagging stubs `_meta.stub` and
|
|
70
|
+
// answering calls with the not_implemented envelope — so it sees 31.
|
|
71
|
+
//
|
|
72
|
+
// Pricing all 31 under one label reported a hybrid client that cannot exist,
|
|
73
|
+
// and made the over-subscription flag fire on a count no local install pays.
|
|
74
|
+
// Keying by transport keeps the per-server shape and makes each number belong
|
|
75
|
+
// to a real client. `implemented_on` in the catalog is what distinguishes them.
|
|
64
76
|
const MCP_SERVER_NAME = 'agent-config';
|
|
77
|
+
const MCP_TRANSPORTS = ['stdio', 'worker'] as const;
|
|
65
78
|
// Soft advisory ceiling on tool count per server. "Few used" cannot be known at
|
|
66
79
|
// audit time (no runtime call data), so over-subscription is a count-based
|
|
67
80
|
// heuristic: many tools => maintainer should review which earn their cost.
|
|
@@ -396,24 +409,40 @@ export function mcp_tool_schemas(): Record<string, McpServer> {
|
|
|
396
409
|
return {};
|
|
397
410
|
}
|
|
398
411
|
const tools = (catalog.tools as unknown[]) ?? [];
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
.map((t) => _measure_tool_schema(t));
|
|
402
|
-
// priced.sort(key=lambda r: (-int(r["tokens_gpt"]), str(r["name"])))
|
|
403
|
-
priced.sort(
|
|
404
|
-
(a, b) => b.tokens_gpt - a.tokens_gpt || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0),
|
|
412
|
+
const records = tools.filter(
|
|
413
|
+
(t): t is Record<string, unknown> => typeof t === 'object' && t !== null && !Array.isArray(t),
|
|
405
414
|
);
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
+
|
|
416
|
+
/** Tools a given transport actually returns from `tools/list`. */
|
|
417
|
+
const served = (transport: string): Record<string, unknown>[] => {
|
|
418
|
+
// The Worker serves the whole catalog (stubs included, flagged
|
|
419
|
+
// `_meta.stub`); stdio serves only what `implemented_on` names.
|
|
420
|
+
if (transport === 'worker') return records;
|
|
421
|
+
return records.filter(
|
|
422
|
+
(t) => Array.isArray(t.implemented_on) && (t.implemented_on as unknown[]).includes(transport),
|
|
423
|
+
);
|
|
415
424
|
};
|
|
416
|
-
|
|
425
|
+
|
|
426
|
+
const out: Record<string, McpServer> = {};
|
|
427
|
+
for (const transport of MCP_TRANSPORTS) {
|
|
428
|
+
const priced: McpPriced[] = served(transport).map((t) => _measure_tool_schema(t));
|
|
429
|
+
// priced.sort(key=lambda r: (-int(r["tokens_gpt"]), str(r["name"])))
|
|
430
|
+
priced.sort(
|
|
431
|
+
(a, b) => b.tokens_gpt - a.tokens_gpt || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0),
|
|
432
|
+
);
|
|
433
|
+
out[`${MCP_SERVER_NAME} (${transport})`] = {
|
|
434
|
+
tool_count: priced.length,
|
|
435
|
+
chars: priced.reduce((acc, r) => acc + r.chars, 0),
|
|
436
|
+
tokens_gpt: priced.reduce((acc, r) => acc + r.tokens_gpt, 0),
|
|
437
|
+
tokens_claude: priced.reduce((acc, r) => acc + r.tokens_claude, 0),
|
|
438
|
+
tokens_gpt_exact:
|
|
439
|
+
priced.length > 0 ? priced.every((r) => Boolean(r.tokens_gpt_exact)) : true,
|
|
440
|
+
over_subscription: priced.length > MCP_OVERSUBSCRIPTION_TOOL_CAP,
|
|
441
|
+
oversubscription_cap: MCP_OVERSUBSCRIPTION_TOOL_CAP,
|
|
442
|
+
tools: priced.map((r) => ({ name: r.name, tokens_gpt: r.tokens_gpt, chars: r.chars })),
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
return out;
|
|
417
446
|
}
|
|
418
447
|
|
|
419
448
|
export function build(): Record<string, unknown> {
|
|
@@ -678,7 +707,12 @@ export function main(argv: string[] | null = null): number {
|
|
|
678
707
|
const rf = rfVals.length > 0 ? (rfVals[0] as Measure) : ({} as Measure);
|
|
679
708
|
const dc = data.description_catalog as Record<string, Measure>;
|
|
680
709
|
const mcp = (data.mcp_tool_schemas as Record<string, McpServer>) ?? {};
|
|
681
|
-
|
|
710
|
+
// MAX, not sum: the entries are two TRANSPORTS of the same server, and
|
|
711
|
+
// a client connects over exactly one of them. Summing would bill the
|
|
712
|
+
// 19 implemented tools twice and gate on a client that cannot exist —
|
|
713
|
+
// the same conflation the per-transport split exists to remove. The
|
|
714
|
+
// budget is what the worst-case single client actually loads.
|
|
715
|
+
const mcp_gpt = Object.values(mcp).reduce((acc, s) => Math.max(acc, s.tokens_gpt), 0);
|
|
682
716
|
const checks: Record<string, number> = {
|
|
683
717
|
'rules.gpt': rf.tokens_gpt ?? 0,
|
|
684
718
|
// The always-SCANNED skill descriptions (258 source skills) — the
|
|
@@ -30,6 +30,8 @@ import * as fs from 'node:fs';
|
|
|
30
30
|
import * as path from 'node:path';
|
|
31
31
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
32
32
|
|
|
33
|
+
import { DeadScopeError, assertScanned } from './_lib/scan_scope.js';
|
|
34
|
+
|
|
33
35
|
const _HERE = fileURLToPath(import.meta.url);
|
|
34
36
|
// src/scripts/audit_likelihood.py → parent.parent.parent == repo root.
|
|
35
37
|
export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
@@ -38,9 +40,20 @@ export const AUDIT_JSON = path.join(REPORT_DIR, 'auto-rules-audit.json');
|
|
|
38
40
|
export const AUDIT_MD = path.join(REPORT_DIR, 'auto-rules-audit.md');
|
|
39
41
|
export const LIKELIHOOD_JSON = path.join(REPORT_DIR, 'auto-rules-likelihood.json');
|
|
40
42
|
|
|
43
|
+
/**
|
|
44
|
+
* The corpus a rule's trigger terms are scored against.
|
|
45
|
+
*
|
|
46
|
+
* The first two globs named the pre-ADR-051 source container, so skills and
|
|
47
|
+
* commands — by far the largest and most trigger-dense part of the corpus —
|
|
48
|
+
* contributed nothing. The audit still printed a score for every rule, derived
|
|
49
|
+
* from contexts + guidelines alone; that biases every score downward and makes
|
|
50
|
+
* the low-likelihood flag unreliable in the direction that matters (false
|
|
51
|
+
* "nothing triggers this"). Commands are now one `command.md` leaf per verb
|
|
52
|
+
* under `src/domains`.
|
|
53
|
+
*/
|
|
41
54
|
const CORPUS_GLOBS: readonly string[] = [
|
|
42
|
-
'
|
|
43
|
-
'
|
|
55
|
+
'src/skills/**/SKILL.md',
|
|
56
|
+
'src/domains/**/command.md',
|
|
44
57
|
'agents/settings/contexts/**/*.md',
|
|
45
58
|
'docs/guidelines/**/*.md',
|
|
46
59
|
];
|
|
@@ -403,6 +416,22 @@ export function main(): number {
|
|
|
403
416
|
}
|
|
404
417
|
const rules: Json[] = JSON.parse(fs.readFileSync(AUDIT_JSON, 'utf-8'))['rules'];
|
|
405
418
|
const corpus = build_corpus();
|
|
419
|
+
// Scope assertion: an empty corpus scores every rule as low-likelihood
|
|
420
|
+
// while looking exactly like a completed audit.
|
|
421
|
+
try {
|
|
422
|
+
assertScanned({
|
|
423
|
+
gate: 'audit_likelihood',
|
|
424
|
+
scanned: corpus.size,
|
|
425
|
+
units: 'distinct corpus token(s)',
|
|
426
|
+
roots: CORPUS_GLOBS,
|
|
427
|
+
});
|
|
428
|
+
} catch (exc) {
|
|
429
|
+
if (!(exc instanceof DeadScopeError)) {
|
|
430
|
+
throw exc;
|
|
431
|
+
}
|
|
432
|
+
process.stderr.write(`❌ ${exc.message}\n`);
|
|
433
|
+
return 2;
|
|
434
|
+
}
|
|
406
435
|
const scores = rules.map((r) => score(r, corpus));
|
|
407
436
|
fs.writeFileSync(
|
|
408
437
|
LIKELIHOOD_JSON,
|
|
@@ -8,19 +8,22 @@
|
|
|
8
8
|
* split, byte-identical stdout summary, AND byte-identical generated
|
|
9
9
|
* Markdown report. Stdlib-only — no YAML dependency.
|
|
10
10
|
*
|
|
11
|
-
* Two checks across
|
|
11
|
+
* Two checks across `src/skills/`:
|
|
12
12
|
*
|
|
13
13
|
* 1. Orphan values — every `recommended_for_user_types` value must have a
|
|
14
|
-
* `user-types/<value>.yml` config. Orphans are FATAL (exit 1)
|
|
14
|
+
* `user-types/<value>.yml` config. Orphans are FATAL (exit 1), subject to
|
|
15
|
+
* the recorded violation ratchet.
|
|
15
16
|
* 2. Unused configs — every `user-types/*.yml` should be consumed by at
|
|
16
17
|
* least one skill. Unused configs are WARN-only (exit 0).
|
|
17
18
|
*
|
|
18
|
-
* NOTE: the retired Python implementation
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
19
|
+
* NOTE: the retired Python implementation scanned only the pre-ADR-051 source
|
|
20
|
+
* container, and this twin inherited that root verbatim for byte parity — which
|
|
21
|
+
* meant it scanned nothing at all once the container was emptied. Repaired
|
|
22
|
+
* 2026-08-02 to `src/skills`; byte-parity with the .py is deliberately given up
|
|
23
|
+
* here, because parity with a scan of zero files is not a property worth
|
|
24
|
+
* keeping. The report path is `agents/reports/user-type-axis-audit.md`
|
|
25
|
+
* (matching the Python constant, which differs from the docstring's
|
|
26
|
+
* `agents/runtime/reports/` mention — latent bug replicated).
|
|
24
27
|
*
|
|
25
28
|
* Historical quirks are preserved deliberately — tests and downstream consumers pin the exact behaviour.
|
|
26
29
|
*/
|
|
@@ -29,11 +32,21 @@ import * as path from 'node:path';
|
|
|
29
32
|
import process from 'node:process';
|
|
30
33
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
31
34
|
|
|
35
|
+
import { SRC_SKILLS } from './_lib/agent_src.js';
|
|
36
|
+
import { checkRatchet } from './_lib/gate_baseline.js';
|
|
37
|
+
import { DeadScopeError, assertScanned } from './_lib/scan_scope.js';
|
|
38
|
+
|
|
32
39
|
const _HERE = fileURLToPath(import.meta.url);
|
|
33
40
|
|
|
34
41
|
// src/scripts/audit_user_type_axis.ts → parents[2] of the .py file is repo root.
|
|
35
42
|
export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
36
|
-
|
|
43
|
+
/**
|
|
44
|
+
* ADR-051 moved skill authoring to `src/skills`. This still named the retired
|
|
45
|
+
* container, so `_scan_skill_values()` took its `!_isDir` early return and the
|
|
46
|
+
* audit reported `used=0 orphans=0` — a clean verdict derived from reading
|
|
47
|
+
* nothing, with all 7 declared user types written off as "unused".
|
|
48
|
+
*/
|
|
49
|
+
export const SKILLS_ROOT = SRC_SKILLS();
|
|
37
50
|
export const USER_TYPES_ROOT = path.join(REPO_ROOT, 'user-types');
|
|
38
51
|
export const REPORT_PATH = path.join(REPO_ROOT, 'agents', 'reports', 'user-type-axis-audit.md');
|
|
39
52
|
|
|
@@ -228,6 +241,24 @@ export function _render_report(
|
|
|
228
241
|
|
|
229
242
|
export function main(argv: string[]): number {
|
|
230
243
|
const quiet = argv.includes('--quiet');
|
|
244
|
+
// Scope assertion: the orphan/unused verdict is a set difference against
|
|
245
|
+
// the skills corpus, so an unreadable corpus does not yield "no orphans",
|
|
246
|
+
// it yields a vacuous answer that looks identical to a clean one.
|
|
247
|
+
const scannedSkills = _isDir(SKILLS_ROOT) ? _rglobSkillMd(SKILLS_ROOT).length : 0;
|
|
248
|
+
try {
|
|
249
|
+
assertScanned({
|
|
250
|
+
gate: 'audit_user_type_axis',
|
|
251
|
+
scanned: scannedSkills,
|
|
252
|
+
units: 'skill file(s)',
|
|
253
|
+
roots: [_relativeToPosix(SKILLS_ROOT, REPO_ROOT)],
|
|
254
|
+
});
|
|
255
|
+
} catch (exc) {
|
|
256
|
+
if (!(exc instanceof DeadScopeError)) {
|
|
257
|
+
throw exc;
|
|
258
|
+
}
|
|
259
|
+
process.stderr.write(`❌ ${exc.message}\n`);
|
|
260
|
+
return 2;
|
|
261
|
+
}
|
|
231
262
|
const declared = _declared_user_types();
|
|
232
263
|
const byValue = _scan_skill_values();
|
|
233
264
|
const used = new Set<string>(byValue.keys());
|
|
@@ -242,7 +273,8 @@ export function main(argv: string[]): number {
|
|
|
242
273
|
if (!quiet) {
|
|
243
274
|
process.stdout.write(
|
|
244
275
|
`user-type-axis audit — declared=${declared.size} ` +
|
|
245
|
-
`used=${used.size} orphans=${orphans.size} unused=${unused.size}
|
|
276
|
+
`used=${used.size} orphans=${orphans.size} unused=${unused.size} ` +
|
|
277
|
+
`(${scannedSkills} skill file(s) scanned)\n`,
|
|
246
278
|
);
|
|
247
279
|
if (orphans.size > 0) {
|
|
248
280
|
process.stdout.write(' FAIL orphans: ' + _sorted(orphans).join(', ') + '\n');
|
|
@@ -253,7 +285,22 @@ export function main(argv: string[]): number {
|
|
|
253
285
|
process.stdout.write(` report: ${_relativeToPosix(REPORT_PATH, REPO_ROOT)}\n`);
|
|
254
286
|
}
|
|
255
287
|
|
|
256
|
-
|
|
288
|
+
if (orphans.size === 0) {
|
|
289
|
+
return 0;
|
|
290
|
+
}
|
|
291
|
+
const verdict = checkRatchet({
|
|
292
|
+
gate: 'audit_user_type_axis',
|
|
293
|
+
actual: orphans.size,
|
|
294
|
+
repoRoot: REPO_ROOT,
|
|
295
|
+
});
|
|
296
|
+
if (verdict.ok) {
|
|
297
|
+
if (!quiet) {
|
|
298
|
+
process.stdout.write(`⚠️ ${verdict.message}\n`);
|
|
299
|
+
}
|
|
300
|
+
return 0;
|
|
301
|
+
}
|
|
302
|
+
process.stderr.write(`❌ ${verdict.message}\n`);
|
|
303
|
+
return 1;
|
|
257
304
|
}
|
|
258
305
|
|
|
259
306
|
function _isCliEntry(): boolean {
|
|
@@ -12,12 +12,20 @@
|
|
|
12
12
|
* ./scripts-run src/scripts/bench_hook_latency # measure + print
|
|
13
13
|
* ./scripts-run src/scripts/bench_hook_latency --gate # enforce budget
|
|
14
14
|
* ./scripts-run src/scripts/bench_hook_latency --update # record numbers
|
|
15
|
+
* ./scripts-run src/scripts/bench_hook_latency --via-cli # real consumer path
|
|
15
16
|
*
|
|
16
17
|
* Method (pinned so numbers reproduce):
|
|
17
|
-
* - N invocations per hook event (default 50, --runs N)
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* - N invocations per hook event (default 50, --runs N) with a synthetic
|
|
19
|
+
* payload on stdin and a throwaway temp workspace as --project-dir
|
|
20
|
+
* (concern state lands there, never in the repo). Two invocation paths:
|
|
21
|
+
* default — `node dist/hooks/dispatch.js --platform claude
|
|
22
|
+
* --event <e>` (the bare bundle).
|
|
23
|
+
* --via-cli — the EXACT command line `hooks/hooks.json` installs
|
|
24
|
+
* (bash wrapper + project-shim probe +
|
|
25
|
+
* `agent-config dispatch:hook … --native-event …
|
|
26
|
+
* --min-version 1`), i.e. the path consumers actually
|
|
27
|
+
* pay per hook event (road-to-hook-latency-repair
|
|
28
|
+
* Phase 1).
|
|
21
29
|
* - AGENT_CONFIG_REPLAY=1 (feedback-dir writes off — measures dispatch,
|
|
22
30
|
* not audit-trail I/O).
|
|
23
31
|
* - Wall-clock per invocation around spawnSync; p50/p95 per event.
|
|
@@ -25,11 +33,19 @@
|
|
|
25
33
|
* Gate semantics (--gate):
|
|
26
34
|
* - pre_tool_use p95 must be ≤ budgets_ms.pre_tool_use.p95_ci
|
|
27
35
|
* - every event p95 must be ≤ budgets_ms.any_hook_event.p95_ci
|
|
28
|
-
* - when docs/hook-latency.json exists
|
|
29
|
-
*
|
|
36
|
+
* - when docs/hook-latency.json exists AND its recorded invocation_path
|
|
37
|
+
* matches this run's path, a p95 regression beyond
|
|
38
|
+
* regression_gate.max_regression_pct fails (mismatched paths skip the
|
|
39
|
+
* regression net — absolute budgets still bind).
|
|
30
40
|
* Budgets: src/config/hook-latency-budget.json (pre-registered, owner +
|
|
31
41
|
* review date — see the Phase-6 budget-ownership lint).
|
|
32
42
|
*
|
|
43
|
+
* Baseline history (--baseline "<hardware class>"): appends this run to the
|
|
44
|
+
* `history` array in docs/hook-latency.json — invocation path, hardware
|
|
45
|
+
* class, per-event p50/p95 — WITHOUT touching the top-level results the
|
|
46
|
+
* regression net compares against. Used to commit the pre-fix CLI-path
|
|
47
|
+
* numbers before the Phase-2 levers landed.
|
|
48
|
+
*
|
|
33
49
|
* Exit codes: 0 green · 1 budget/regression exceeded · 2 internal error.
|
|
34
50
|
*/
|
|
35
51
|
import { spawnSync } from 'node:child_process';
|
|
@@ -37,9 +53,13 @@ import * as fs from 'node:fs';
|
|
|
37
53
|
import * as os from 'node:os';
|
|
38
54
|
import * as path from 'node:path';
|
|
39
55
|
import { fileURLToPath } from 'node:url';
|
|
56
|
+
import { build_claude_hook_matrix } from './_lib/claude_settings_hooks.js';
|
|
40
57
|
|
|
41
58
|
const REPO_ROOT = path.resolve(fileURLToPath(import.meta.url), '..', '..', '..');
|
|
42
59
|
const BUNDLE = path.join(REPO_ROOT, 'dist', 'hooks', 'dispatch.js');
|
|
60
|
+
const CLI_BIN = path.join(REPO_ROOT, 'dist', 'cli', 'agent-config.js');
|
|
61
|
+
const SHIM = path.join(REPO_ROOT, 'agent-config');
|
|
62
|
+
const MANIFEST_PATH = path.join(REPO_ROOT, 'src', 'scripts', 'hook_manifest.yaml');
|
|
43
63
|
const BUDGET_PATH = path.join(REPO_ROOT, 'src', 'config', 'hook-latency-budget.json');
|
|
44
64
|
const RESULTS_PATH = path.join(REPO_ROOT, 'docs', 'hook-latency.json');
|
|
45
65
|
|
|
@@ -52,6 +72,18 @@ const EVENTS: readonly string[] = [
|
|
|
52
72
|
'session_end',
|
|
53
73
|
];
|
|
54
74
|
|
|
75
|
+
/** snake_case hook event → the platform-native name hooks.json passes. */
|
|
76
|
+
const NATIVE_EVENT: Readonly<Record<string, string>> = {
|
|
77
|
+
pre_tool_use: 'PreToolUse',
|
|
78
|
+
post_tool_use: 'PostToolUse',
|
|
79
|
+
user_prompt_submit: 'UserPromptSubmit',
|
|
80
|
+
stop: 'Stop',
|
|
81
|
+
session_start: 'SessionStart',
|
|
82
|
+
session_end: 'SessionEnd',
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
export type InvocationPath = 'bundle' | 'cli';
|
|
86
|
+
|
|
55
87
|
interface EventResult {
|
|
56
88
|
event: string;
|
|
57
89
|
runs: number;
|
|
@@ -79,29 +111,49 @@ function syntheticPayload(event: string, workspace: string): string {
|
|
|
79
111
|
});
|
|
80
112
|
}
|
|
81
113
|
|
|
82
|
-
|
|
114
|
+
/**
|
|
115
|
+
* The verbatim per-event command `hooks/hooks.json` installs — taken from
|
|
116
|
+
* build_claude_hook_matrix(), the SAME generator that writes hooks.json and
|
|
117
|
+
* the consumer settings entries, so the bench cannot drift from the
|
|
118
|
+
* installed command. The bench sets CLAUDE_PROJECT_DIR to the throwaway
|
|
119
|
+
* workspace (which carries a symlink to the repo shim, exactly the layout a
|
|
120
|
+
* shim-carrying project has), so the probe takes the same branch a consumer
|
|
121
|
+
* pays and concern state stays out of the repo.
|
|
122
|
+
*/
|
|
123
|
+
let _matrixCache: Record<string, string> | null = null;
|
|
124
|
+
function hooksJsonCommand(event: string): string {
|
|
125
|
+
_matrixCache ??= build_claude_hook_matrix(MANIFEST_PATH);
|
|
126
|
+
const native = NATIVE_EVENT[event] ?? event;
|
|
127
|
+
const cmd = _matrixCache[native];
|
|
128
|
+
if (cmd === undefined) {
|
|
129
|
+
throw new Error(`no hook-matrix entry for ${event} (${native}) in ${MANIFEST_PATH}`);
|
|
130
|
+
}
|
|
131
|
+
return cmd;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function benchEvent(
|
|
135
|
+
event: string,
|
|
136
|
+
runs: number,
|
|
137
|
+
workspace: string,
|
|
138
|
+
via: InvocationPath = 'bundle',
|
|
139
|
+
): EventResult {
|
|
83
140
|
const durations: number[] = [];
|
|
84
141
|
const payload = syntheticPayload(event, workspace);
|
|
142
|
+
const command: [string, string[]] =
|
|
143
|
+
via === 'cli'
|
|
144
|
+
? ['bash', ['-c', hooksJsonCommand(event)]]
|
|
145
|
+
: [
|
|
146
|
+
process.execPath,
|
|
147
|
+
[BUNDLE, '--platform', 'claude', '--event', event, '--project-dir', workspace],
|
|
148
|
+
];
|
|
85
149
|
for (let i = 0; i < runs; i += 1) {
|
|
86
150
|
const started = performance.now();
|
|
87
|
-
const proc = spawnSync(
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
'--event',
|
|
94
|
-
event,
|
|
95
|
-
'--project-dir',
|
|
96
|
-
workspace,
|
|
97
|
-
],
|
|
98
|
-
{
|
|
99
|
-
input: payload,
|
|
100
|
-
encoding: 'utf-8',
|
|
101
|
-
env: { ...process.env, AGENT_CONFIG_REPLAY: '1' },
|
|
102
|
-
timeout: 60000,
|
|
103
|
-
},
|
|
104
|
-
);
|
|
151
|
+
const proc = spawnSync(command[0], command[1], {
|
|
152
|
+
input: payload,
|
|
153
|
+
encoding: 'utf-8',
|
|
154
|
+
env: { ...process.env, AGENT_CONFIG_REPLAY: '1', CLAUDE_PROJECT_DIR: workspace },
|
|
155
|
+
timeout: 60000,
|
|
156
|
+
});
|
|
105
157
|
const elapsed = performance.now() - started;
|
|
106
158
|
if (proc.error) {
|
|
107
159
|
throw new Error(`dispatch failed on ${event}: ${proc.error.message}`);
|
|
@@ -134,9 +186,35 @@ interface Budget {
|
|
|
134
186
|
regression_gate: { max_regression_pct: number };
|
|
135
187
|
}
|
|
136
188
|
|
|
189
|
+
interface HistoryEntry {
|
|
190
|
+
recorded_at: string;
|
|
191
|
+
invocation_path: InvocationPath;
|
|
192
|
+
hardware: string;
|
|
193
|
+
node: string;
|
|
194
|
+
platform: string;
|
|
195
|
+
runs_per_event: number;
|
|
196
|
+
results: EventResult[];
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
interface ResultsDoc {
|
|
200
|
+
_comment?: string;
|
|
201
|
+
invocation_path?: InvocationPath;
|
|
202
|
+
results: EventResult[];
|
|
203
|
+
history?: HistoryEntry[];
|
|
204
|
+
[key: string]: unknown;
|
|
205
|
+
}
|
|
206
|
+
|
|
137
207
|
export function main(argv: string[] = process.argv.slice(2)): number {
|
|
138
208
|
const gate = argv.includes('--gate');
|
|
139
209
|
const update = argv.includes('--update');
|
|
210
|
+
const viaCli = argv.includes('--via-cli');
|
|
211
|
+
const via: InvocationPath = viaCli ? 'cli' : 'bundle';
|
|
212
|
+
const baselineIdx = argv.indexOf('--baseline');
|
|
213
|
+
const baselineHardware = baselineIdx >= 0 ? (argv[baselineIdx + 1] ?? null) : null;
|
|
214
|
+
if (baselineIdx >= 0 && (baselineHardware === null || baselineHardware.startsWith('--'))) {
|
|
215
|
+
process.stderr.write('bench_hook_latency: --baseline requires a "<hardware class>" label\n');
|
|
216
|
+
return 2;
|
|
217
|
+
}
|
|
140
218
|
const runsIdx = argv.indexOf('--runs');
|
|
141
219
|
const runs = runsIdx >= 0 ? Number.parseInt(argv[runsIdx + 1] ?? '50', 10) : 50;
|
|
142
220
|
|
|
@@ -146,6 +224,21 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
146
224
|
);
|
|
147
225
|
return 2;
|
|
148
226
|
}
|
|
227
|
+
if (viaCli) {
|
|
228
|
+
// The CLI path exercises shim → dist/cli → bundle; a missing link
|
|
229
|
+
// would make the hooks.json probe silently `exit 0` and the bench
|
|
230
|
+
// would time bash startup instead of a dispatch.
|
|
231
|
+
if (!fs.existsSync(CLI_BIN)) {
|
|
232
|
+
process.stderr.write(
|
|
233
|
+
`bench_hook_latency: CLI missing at ${CLI_BIN} — run \`npm run build:cli\` first\n`,
|
|
234
|
+
);
|
|
235
|
+
return 2;
|
|
236
|
+
}
|
|
237
|
+
if (!fs.existsSync(SHIM)) {
|
|
238
|
+
process.stderr.write(`bench_hook_latency: repo shim missing at ${SHIM}\n`);
|
|
239
|
+
return 2;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
149
242
|
const budget = loadJson<Budget>(BUDGET_PATH);
|
|
150
243
|
if (budget === null) {
|
|
151
244
|
process.stderr.write(`bench_hook_latency: budget file missing/invalid at ${BUDGET_PATH}\n`);
|
|
@@ -155,11 +248,23 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
155
248
|
const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'ac-hook-bench-'));
|
|
156
249
|
const results: EventResult[] = [];
|
|
157
250
|
try {
|
|
251
|
+
if (viaCli) {
|
|
252
|
+
// Stage the workspace like a real consumer project so the
|
|
253
|
+
// installed command's probes take the branch consumers pay:
|
|
254
|
+
// - node_modules/@event4u/agent-config → the package (bundle
|
|
255
|
+
// fast path candidate 1),
|
|
256
|
+
// - ./agent-config shim → the CLI fallback (resolved through
|
|
257
|
+
// the symlink; the shim self-resolves symlinks).
|
|
258
|
+
const scope = path.join(workspace, 'node_modules', '@event4u');
|
|
259
|
+
fs.mkdirSync(scope, { recursive: true });
|
|
260
|
+
fs.symlinkSync(REPO_ROOT, path.join(scope, 'agent-config'));
|
|
261
|
+
fs.symlinkSync(path.join(REPO_ROOT, 'src', 'scripts', 'agent-config'), path.join(workspace, 'agent-config'));
|
|
262
|
+
}
|
|
158
263
|
for (const event of EVENTS) {
|
|
159
|
-
const r = benchEvent(event, runs, workspace);
|
|
264
|
+
const r = benchEvent(event, runs, workspace, via);
|
|
160
265
|
results.push(r);
|
|
161
266
|
process.stdout.write(
|
|
162
|
-
`${event.padEnd(20)} p50 ${String(r.p50_ms).padStart(5)} ms · p95 ${String(r.p95_ms).padStart(5)} ms · max ${String(r.max_ms).padStart(5)} ms (n=${r.runs})\n`,
|
|
267
|
+
`${event.padEnd(20)} p50 ${String(r.p50_ms).padStart(5)} ms · p95 ${String(r.p95_ms).padStart(5)} ms · max ${String(r.max_ms).padStart(5)} ms (n=${r.runs}, via ${via})\n`,
|
|
163
268
|
);
|
|
164
269
|
}
|
|
165
270
|
} finally {
|
|
@@ -167,7 +272,8 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
167
272
|
}
|
|
168
273
|
|
|
169
274
|
let failed = false;
|
|
170
|
-
const prior = loadJson<
|
|
275
|
+
const prior = loadJson<ResultsDoc>(RESULTS_PATH);
|
|
276
|
+
const priorPath: InvocationPath = prior?.invocation_path ?? 'bundle';
|
|
171
277
|
for (const r of results) {
|
|
172
278
|
const anyCap = budget.budgets_ms.any_hook_event.p95_ci;
|
|
173
279
|
const cap = r.event === 'pre_tool_use' ? budget.budgets_ms.pre_tool_use.p95_ci : anyCap;
|
|
@@ -177,7 +283,7 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
177
283
|
);
|
|
178
284
|
failed = true;
|
|
179
285
|
}
|
|
180
|
-
if (gate && prior !== null) {
|
|
286
|
+
if (gate && prior !== null && priorPath === via) {
|
|
181
287
|
const prev = prior.results.find((x) => x.event === r.event);
|
|
182
288
|
if (prev !== undefined && prev.p95_ms > 0) {
|
|
183
289
|
const regressionPct = ((r.p95_ms - prev.p95_ms) / prev.p95_ms) * 100;
|
|
@@ -191,20 +297,45 @@ export function main(argv: string[] = process.argv.slice(2)): number {
|
|
|
191
297
|
}
|
|
192
298
|
}
|
|
193
299
|
}
|
|
300
|
+
if (gate && prior !== null && priorPath !== via) {
|
|
301
|
+
process.stdout.write(
|
|
302
|
+
`ℹ️ regression net skipped — recorded baseline is via ${priorPath}, this run is via ${via}\n`,
|
|
303
|
+
);
|
|
304
|
+
}
|
|
194
305
|
|
|
195
|
-
if (update) {
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
306
|
+
if (update || baselineHardware !== null) {
|
|
307
|
+
const existing = prior ?? ({ results: [] } as ResultsDoc);
|
|
308
|
+
const history: HistoryEntry[] = existing.history ?? [];
|
|
309
|
+
if (baselineHardware !== null) {
|
|
310
|
+
history.push({
|
|
311
|
+
recorded_at: new Date().toISOString(),
|
|
312
|
+
invocation_path: via,
|
|
313
|
+
hardware: baselineHardware,
|
|
314
|
+
node: process.version,
|
|
315
|
+
platform: `${os.platform()}-${os.arch()}`,
|
|
316
|
+
runs_per_event: runs,
|
|
317
|
+
results,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
const doc: ResultsDoc = update
|
|
321
|
+
? {
|
|
322
|
+
_comment:
|
|
323
|
+
'Recorded hook-dispatch latency (bench_hook_latency --update). Method pinned in src/scripts/bench_hook_latency.ts; budgets in src/config/hook-latency-budget.json. Re-run from a fresh clone to reproduce. `history` entries (--baseline) are point-in-time records — the top-level results are the regression base.',
|
|
324
|
+
recorded_at: new Date().toISOString(),
|
|
325
|
+
node: process.version,
|
|
326
|
+
platform: `${os.platform()}-${os.arch()}`,
|
|
327
|
+
hardware_note:
|
|
328
|
+
'see hook-latency-budget.json hardware_reference — CI numbers are the gated ones',
|
|
329
|
+
invocation_path: via,
|
|
330
|
+
runs_per_event: runs,
|
|
331
|
+
results,
|
|
332
|
+
...(history.length > 0 ? { history } : {}),
|
|
333
|
+
}
|
|
334
|
+
: { ...existing, history };
|
|
206
335
|
fs.writeFileSync(RESULTS_PATH, `${JSON.stringify(doc, null, 4)}\n`);
|
|
207
|
-
process.stdout.write(
|
|
336
|
+
process.stdout.write(
|
|
337
|
+
`recorded → ${path.relative(REPO_ROOT, RESULTS_PATH)}${update ? '' : ' (history entry only)'}\n`,
|
|
338
|
+
);
|
|
208
339
|
}
|
|
209
340
|
|
|
210
341
|
if (failed) return 1;
|
|
@@ -29,7 +29,7 @@ import * as path from 'node:path';
|
|
|
29
29
|
import process from 'node:process';
|
|
30
30
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
31
31
|
|
|
32
|
-
import yaml from 'js-yaml';
|
|
32
|
+
import * as yaml from 'js-yaml';
|
|
33
33
|
|
|
34
34
|
import { signTestP } from './second_brain_retrieval.js';
|
|
35
35
|
|
|
@@ -30,7 +30,7 @@ import * as path from 'node:path';
|
|
|
30
30
|
import process from 'node:process';
|
|
31
31
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
32
32
|
|
|
33
|
-
import yaml from 'js-yaml';
|
|
33
|
+
import * as yaml from 'js-yaml';
|
|
34
34
|
|
|
35
35
|
import { build_thin, RULES_SOURCE } from './project_thin_rules.js';
|
|
36
36
|
import { evaluatePair, type JudgeFn, type JudgeVerdict, type PairResult } from './check_quality_regression.js';
|
|
@@ -135,7 +135,18 @@ export function _buildEntries(): CatalogEntryOut[] {
|
|
|
135
135
|
description: STUB_MARKER + stub.description,
|
|
136
136
|
side_effect: stub.side_effect,
|
|
137
137
|
implemented_on: [],
|
|
138
|
-
|
|
138
|
+
// A stub ships an EMPTY schema on purpose. `input_schema` exists to
|
|
139
|
+
// tell a client how to CALL the tool, and no transport permits that
|
|
140
|
+
// for a stub: the Worker answers with the not_implemented envelope
|
|
141
|
+
// and stdio never registers it (`REGISTRY` in mcp_server/tools.ts).
|
|
142
|
+
// A populated schema is therefore always-loaded context that can
|
|
143
|
+
// never be acted on — 665 GPT tok across the 12 stubs, measured
|
|
144
|
+
// 2026-08-02. `stub.input_schema` stays in STUB_TOOLS as the design
|
|
145
|
+
// record for the day the tool is wired; it is simply not shipped
|
|
146
|
+
// until then. The stub-envelope contract's "byte-identical apart
|
|
147
|
+
// from implemented_on" promise still holds — both transports read
|
|
148
|
+
// this same file.
|
|
149
|
+
input_schema: {},
|
|
139
150
|
});
|
|
140
151
|
}
|
|
141
152
|
|