@event4u/agent-config 8.9.0 → 8.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +189 -0
- package/README.md +5 -5
- package/dist/agent-src/commands/challenge-me/vision.md +121 -32
- package/dist/agent-src/commands/challenge-me/with-docs.md +21 -8
- package/dist/agent-src/commands/commit/in-chunks.md +1 -1
- package/dist/agent-src/commands/commit.md +2 -1
- package/dist/agent-src/commands/cost/report.md +14 -3
- package/dist/agent-src/commands/feature/dev.md +4 -4
- package/dist/agent-src/commands/humanize.md +101 -0
- package/dist/agent-src/commands/jira-ticket.md +2 -2
- package/dist/agent-src/commands/memory/load.md +15 -0
- package/dist/agent-src/commands/optimize/project.md +12 -0
- package/dist/agent-src/commands/roadmap/create.md +12 -11
- package/dist/agent-src/commands/tests/create.md +29 -7
- package/dist/agent-src/commands/work.md +33 -0
- package/dist/agent-src/commands/worktree/cleanup.md +83 -0
- package/dist/agent-src/commands/worktree/create.md +64 -0
- package/dist/agent-src/commands/worktree/status.md +75 -0
- package/dist/agent-src/commands/worktree/verify.md +70 -0
- package/dist/agent-src/commands/worktree.md +85 -0
- package/dist/agent-src/contexts/execution/autonomy-mechanics.md +66 -0
- package/dist/agent-src/contexts/execution/orchestration-benchmark-gate.md +1 -0
- package/dist/agent-src/contexts/execution/orchestration-telemetry.md +91 -34
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +14 -0
- package/dist/agent-src/contexts/execution/roadmap-writing-source-derived.md +65 -0
- package/dist/agent-src/contexts/execution/subagent-modes-detail.md +84 -0
- package/dist/agent-src/rules/autonomous-execution.md +7 -37
- package/dist/agent-src/rules/broken-access-control.md +7 -45
- package/dist/agent-src/rules/code-comment-discipline.md +148 -0
- package/dist/agent-src/rules/context-hygiene.md +14 -0
- package/dist/agent-src/rules/delegation-policy.md +10 -8
- package/dist/agent-src/rules/design-fidelity.md +44 -0
- package/dist/agent-src/rules/git-history-discipline.md +4 -79
- package/dist/agent-src/rules/legal-safety-floor.md +11 -55
- package/dist/agent-src/rules/roadmap-progress-sync.md +5 -56
- package/dist/agent-src/rules/spreadsheet-source-quality.md +61 -0
- package/dist/agent-src/rules/think-before-action.md +10 -0
- package/dist/agent-src/rules/verify-before-complete.md +12 -0
- package/dist/agent-src/skills/accessibility-auditor/evals/evals.json +15 -0
- package/dist/agent-src/skills/agent-docs-writing/SKILL.md +13 -12
- package/dist/agent-src/skills/ai-council/SKILL.md +19 -9
- package/dist/agent-src/skills/analysis-skill-router/evals/evals.json +13 -0
- package/dist/agent-src/skills/api-design/SKILL.md +7 -5
- package/dist/agent-src/skills/api-testing/SKILL.md +16 -2
- package/dist/agent-src/skills/authz-review/SKILL.md +59 -2
- package/dist/agent-src/skills/blast-radius-analyzer/SKILL.md +1 -1
- package/dist/agent-src/skills/bug-analyzer/SKILL.md +19 -16
- package/dist/agent-src/skills/character-consistency/evals/evals.json +16 -0
- package/dist/agent-src/skills/code-refactoring/SKILL.md +9 -9
- package/dist/agent-src/skills/command-routing/evals/evals.json +14 -0
- package/dist/agent-src/skills/content-funnel-design/SKILL.md +2 -0
- package/dist/agent-src/skills/dashboard-design/evals/evals.json +16 -0
- package/dist/agent-src/skills/data-flow-mapper/SKILL.md +1 -1
- package/dist/agent-src/skills/dcf-modeling/evals/evals.json +20 -0
- package/dist/agent-src/skills/decision-record/evals/evals.json +20 -0
- package/dist/agent-src/skills/defense-in-depth/SKILL.md +2 -2
- package/dist/agent-src/skills/dependency-upgrade/SKILL.md +7 -7
- package/dist/agent-src/skills/design-intelligence/SKILL.md +8 -8
- package/dist/agent-src/skills/design-intelligence/evals/evals.json +14 -0
- package/dist/agent-src/skills/design-review/SKILL.md +29 -2
- package/dist/agent-src/skills/design-system-capture/SKILL.md +21 -0
- package/dist/agent-src/skills/design-system-capture/evals/evals.json +15 -0
- package/dist/agent-src/skills/developer-like-execution/evals/evals.json +16 -0
- package/dist/agent-src/skills/doc-coauthoring/SKILL.md +29 -0
- package/dist/agent-src/skills/doc-coauthoring/evals/evals.json +16 -0
- package/dist/agent-src/skills/editorial-calendar/SKILL.md +2 -0
- package/dist/agent-src/skills/editorial-calendar/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/editorial-calendar/evals/evals.json +16 -0
- package/dist/agent-src/skills/estimate-ticket/evals/evals.json +16 -0
- package/dist/agent-src/skills/existing-ui-audit/SKILL.md +30 -0
- package/dist/agent-src/skills/existing-ui-audit/evals/triggers.json +47 -0
- package/dist/agent-src/skills/fe-design/SKILL.md +43 -14
- package/dist/agent-src/skills/fe-design/evals/triggers.json +47 -0
- package/dist/agent-src/skills/forecasting/evals/evals.json +20 -0
- package/dist/agent-src/skills/fundraising-narrative/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/fundraising-narrative/evals/evals.json +16 -0
- package/dist/agent-src/skills/git-workflow/SKILL.md +86 -0
- package/dist/agent-src/skills/html-deck/SKILL.md +27 -0
- package/dist/agent-src/skills/humanizer/SKILL.md +147 -0
- package/dist/agent-src/skills/humanizer/data/patterns.md +190 -0
- package/dist/agent-src/skills/humanizer/evals/triggers.json +17 -0
- package/dist/agent-src/skills/iconography/SKILL.md +24 -0
- package/dist/agent-src/skills/incident-commander/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/incident-commander/evals/evals.json +20 -0
- package/dist/agent-src/skills/laravel/SKILL.md +14 -1
- package/dist/agent-src/skills/launch-readiness/SKILL.md +20 -19
- package/dist/agent-src/skills/launch-readiness/evals/evals.json +21 -0
- package/dist/agent-src/skills/legal-practice-profile/SKILL.md +126 -0
- package/dist/agent-src/skills/logging-monitoring/SKILL.md +1 -1
- package/dist/agent-src/skills/logging-monitoring/evals/evals.json +16 -0
- package/dist/agent-src/skills/mcp/SKILL.md +1 -1
- package/dist/agent-src/skills/memory-consolidation/SKILL.md +14 -1
- package/dist/agent-src/skills/messaging-architecture/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/messaging-architecture/evals/evals.json +16 -0
- package/dist/agent-src/skills/migration-architect/SKILL.md +2 -1
- package/dist/agent-src/skills/perf-feedback-craft/evals/evals.json +16 -0
- package/dist/agent-src/skills/performance/SKILL.md +1 -1
- package/dist/agent-src/skills/pest-testing/SKILL.md +1 -1
- package/dist/agent-src/skills/php-debugging/SKILL.md +17 -0
- package/dist/agent-src/skills/playwright-testing/SKILL.md +16 -0
- package/dist/agent-src/skills/prompt-engineering-patterns/evals/evals.json +16 -0
- package/dist/agent-src/skills/readme-writing/SKILL.md +2 -0
- package/dist/agent-src/skills/readme-writing-package/SKILL.md +2 -0
- package/dist/agent-src/skills/refine-prompt/evals/evals.json +16 -0
- package/dist/agent-src/skills/refine-ticket/evals/evals.json +16 -0
- package/dist/agent-src/skills/release-comms/SKILL.md +2 -0
- package/dist/agent-src/skills/release-comms/evals/evals.json +16 -0
- package/dist/agent-src/skills/rice-prioritization/evals/evals.json +20 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +22 -51
- package/dist/agent-src/skills/runway-cognition/evals/evals.json +20 -0
- package/dist/agent-src/skills/scenario-modeling/evals/evals.json +20 -0
- package/dist/agent-src/skills/security-audit/SKILL.md +6 -9
- package/dist/agent-src/skills/sentry-integration/SKILL.md +2 -2
- package/dist/agent-src/skills/skill-improvement-pipeline/SKILL.md +8 -0
- package/dist/agent-src/skills/source-discovery/SKILL.md +26 -24
- package/dist/agent-src/skills/spreadsheet-authoring/SKILL.md +98 -0
- package/dist/agent-src/skills/spreadsheet-authoring/evals/triggers.json +47 -0
- package/dist/agent-src/skills/subagent-orchestration/SKILL.md +55 -46
- package/dist/agent-src/skills/subagent-orchestration/evals/evals.json +42 -0
- package/dist/agent-src/skills/systematic-debugging/SKILL.md +10 -0
- package/dist/agent-src/skills/systematic-debugging/evals/evals.json +20 -0
- package/dist/agent-src/skills/test-case-discovery/SKILL.md +118 -0
- package/dist/agent-src/skills/test-case-discovery/evals/triggers.json +52 -0
- package/dist/agent-src/skills/test-driven-development/SKILL.md +32 -6
- package/dist/agent-src/skills/test-driven-development/evals/evals.json +16 -0
- package/dist/agent-src/skills/testing-anti-patterns/SKILL.md +7 -6
- package/dist/agent-src/skills/threat-modeling/evals/evals.json +21 -0
- package/dist/agent-src/skills/typography-system/evals/evals.json +14 -0
- package/dist/agent-src/skills/unit-economics-modeling/evals/evals.json +24 -0
- package/dist/agent-src/skills/using-git-worktrees/SKILL.md +10 -0
- package/dist/agent-src/skills/verify-completion-evidence/evals/evals.json +16 -0
- package/dist/agent-src/skills/vision-articulation/evals/evals.json +16 -0
- package/dist/agent-src/skills/voice-and-tone-design/evals/evals.json +16 -0
- package/dist/agent-src/skills/worktree-lifecycle/SKILL.md +187 -0
- package/dist/agent-src/skills/worktree-lifecycle/evals/evals.json +38 -0
- package/dist/agent-src/skills/worktree-lifecycle/evals/triggers.json +52 -0
- package/dist/agent-src/templates/agent-settings.md +3 -2
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +15 -1
- package/dist/agent-src/templates/roadmaps.md +14 -0
- package/dist/cli/commands/recordTriggerEval.js +4 -14
- package/dist/cli/commands/recordTriggerEval.js.map +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +374 -88
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +9 -9
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +30 -18
- package/dist/discovery/trust-report.md +5 -5
- package/dist/discovery/workspaces.json +31 -15
- package/dist/install/install.mjs +12 -7
- package/dist/mcp/registry-manifest.json +4 -4
- package/dist/router.json +1 -1
- package/dist/scripts/_lib/claude_settings_hooks.js +8 -0
- package/dist/scripts/_lib/claude_settings_hooks.js.map +1 -1
- package/dist/scripts/_lib/trigger_eval_floors.js +24 -0
- package/dist/scripts/_lib/trigger_eval_floors.js.map +1 -0
- package/dist/scripts/condense.js +12 -2
- package/dist/scripts/condense.js.map +1 -1
- package/dist/server/schemas/settings.js +5 -2
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/CLAIMS.md +24 -3
- package/docs/architecture.md +3 -3
- package/docs/benchmark.md +51 -0
- package/docs/catalog.md +23 -9
- package/docs/command-flows.md +7 -1
- package/docs/contracts/CHANGELOG-conventions.md +6 -0
- package/docs/contracts/ai-council-config.md +92 -0
- package/docs/contracts/command-clusters.md +1 -0
- package/docs/contracts/design-artifact-lifecycle.md +93 -0
- package/docs/contracts/design-artifact-verification.md +30 -1
- package/docs/contracts/evidence-based-pruning.md +1 -0
- package/docs/contracts/hook-architecture-v1.md +25 -0
- package/docs/contracts/kernel-membership.md +35 -0
- package/docs/contracts/measurement-baseline.md +1 -0
- package/docs/contracts/release-pr-gating.md +26 -0
- package/docs/contracts/release-sizing.md +107 -0
- package/docs/contracts/surface-agent-contracts.md +153 -0
- package/docs/contracts/write-engine.md +38 -1
- package/docs/customization.md +12 -0
- package/docs/decisions/ADR-002-kernel-bucket-overrides.md +11 -0
- package/docs/decisions/ADR-003-flat-cluster-subs-and-colon-syntax.md +11 -0
- package/docs/decisions/ADR-006-skill-tools-python-pilot.md +1 -1
- package/docs/decisions/ADR-028-root-layout.md +1 -1
- package/docs/decisions/ADR-038-canonical-settings-path.md +10 -0
- package/docs/decisions/ADR-092-defer-command-tier-alias-removal.md +29 -5
- package/docs/decisions/ADR-093-ai-council-config-user-global.md +1 -1
- package/docs/decisions/ADR-100-global-knowledge-card-sharing.md +7 -0
- package/docs/decisions/ADR-103-global-knowledge-default-off-until-measured.md +2 -2
- package/docs/decisions/ADR-105-automatic-subagent-orchestration.md +1 -1
- package/docs/decisions/ADR-114-commit-policy-iron-law-override.md +11 -0
- package/docs/decisions/ADR-115-command-cluster-consolidation-phase-4.md +1 -1
- package/docs/decisions/ADR-116-memory-tripwire-activation-path.md +11 -1
- package/docs/decisions/ADR-117-subagents-auto-default-on.md +2 -1
- package/docs/decisions/ADR-118-loop-engineering-boundaries.md +162 -0
- package/docs/decisions/ADR-119-global-knowledge-default-on.md +104 -0
- package/docs/decisions/ADR-200-python-to-typescript-migration.md +11 -1
- package/docs/decisions/ADR-rule-kernel-and-router.md +11 -0
- package/docs/decisions/INDEX.md +11 -9
- package/docs/design/cross-model-parity-eval.md +82 -0
- package/docs/design/length-neutral-judge-rerun.md +92 -0
- package/docs/distribution/consumer-matrix.md +96 -0
- package/docs/featured-skills.md +2 -2
- package/docs/getting-started-by-role.md +1 -1
- package/docs/getting-started.md +2 -2
- package/docs/governance-advantage.md +1 -1
- package/docs/guidelines/agent-infra/roadmap-progress-mechanics.md +80 -5
- package/docs/guidelines/code-clarity.md +106 -1
- package/docs/guidelines/design-antipatterns.md +20 -0
- package/docs/guidelines/php/php-coding-patterns.md +12 -2
- package/docs/proof.md +21 -15
- package/docs/release-runbook.md +3 -0
- package/docs/self-review-gate.md +51 -0
- package/docs/wedge/production-validator/production-validator.md +27 -0
- package/package.json +1 -1
- package/src/config/agent-settings.template.yml +29 -9
- package/src/scripts/_cli/cmd_doctor.ts +34 -1
- package/src/scripts/_lib/auto_dispatch.corpus.test.ts +87 -0
- package/src/scripts/_lib/claude_settings_hooks.ts +8 -0
- package/src/scripts/_lib/knowledge_global.ts +6 -2
- package/src/scripts/_lib/knowledge_global_redaction.ts +42 -4
- package/src/scripts/_lib/lexical_index.ts +15 -2
- package/src/scripts/_lib/orchestration_record.ts +27 -0
- package/src/scripts/_lib/orchestration_savings.ts +52 -0
- package/src/scripts/_lib/trigger_eval_floors.ts +31 -0
- package/src/scripts/ai_council/chairman.ts +90 -0
- package/src/scripts/ai_council/clients.ts +18 -3
- package/src/scripts/ai_council/config.ts +157 -0
- package/src/scripts/ai_council/debate_gates.ts +89 -0
- package/src/scripts/ai_council/orchestrator.ts +65 -2
- package/src/scripts/ai_council/prompts.ts +121 -11
- package/src/scripts/ai_council/replay.ts +17 -0
- package/src/scripts/ai_council/stance_tally.ts +184 -0
- package/src/scripts/ai_tells_rules.ts +433 -0
- package/src/scripts/bench_humanizer_eval.ts +240 -0
- package/src/scripts/build_discovery_manifest.ts +6 -0
- package/src/scripts/build_proof.ts +21 -9
- package/src/scripts/check_claims.ts +40 -0
- package/src/scripts/check_release_adjacent_health.ts +121 -0
- package/src/scripts/check_rule_invariants.ts +282 -0
- package/src/scripts/condense.ts +16 -2
- package/src/scripts/consumer_matrix.ts +392 -0
- package/src/scripts/council_cli.ts +147 -0
- package/src/scripts/detect_ai_tells.ts +312 -0
- package/src/scripts/generate_index.ts +3 -2
- package/src/scripts/install.ts +12 -0
- package/src/scripts/knowledge_global_cli.ts +1 -1
- package/src/scripts/lint_changelog_rollback.ts +293 -0
- package/src/scripts/lint_framework_leakage.ts +88 -7
- package/src/scripts/lint_framework_leakage_allowlist.json +59 -572
- package/src/scripts/lint_knowledge_scale.ts +30 -0
- package/src/scripts/lint_originality_shingles.ts +210 -0
- package/src/scripts/orchestration_record.ts +17 -0
- package/src/scripts/orchestration_savings_report.ts +36 -3
- package/src/scripts/schemas/agent-settings.schema.json +11 -0
- package/src/scripts/second_brain_retrieval.ts +44 -2
- package/src/scripts/self_review_gate.test.ts +65 -0
- package/src/scripts/self_review_gate.ts +298 -0
- package/src/scripts/skill_eval_coverage.ts +71 -3
- package/src/scripts/skill_linter.ts +1 -0
- package/src/scripts/trigger_eval_grandfather.json +223 -224
- package/src/scripts/trigger_eval_rotation.ts +344 -0
- package/src/scripts/update_counts.ts +5 -17
- package/src/scripts/worktree_cleanup_check.ts +263 -0
- package/src/server/schemas/settings.ts +8 -3
- package/src/scripts/analysis_freshness.ts +0 -362
- package/src/scripts/export_replay_corpus.ts +0 -208
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
* Health checks (see `CHECK_IDS`):
|
|
29
29
|
* scope · global-binary · claude-plugin · stale-orphans · manifest-integrity ·
|
|
30
30
|
* lockfile-freshness · bridge-drift · mcp-mode · mcp-beta-readiness ·
|
|
31
|
-
* offline-readiness · python-runtime · tier-usage-readiness ·
|
|
31
|
+
* offline-readiness · python-runtime · humanizer-runtime · tier-usage-readiness ·
|
|
32
|
+
* council-cli ·
|
|
32
33
|
* unsupported-combos · wizard-state.
|
|
33
34
|
* Each emits a structured `{id, status, message, remedy}` record with
|
|
34
35
|
* `status` ∈ `ok` / `warn` / `fail` / `skipped` (rendered
|
|
@@ -742,6 +743,7 @@ const CHECK_IDS = [
|
|
|
742
743
|
'mcp-beta-readiness',
|
|
743
744
|
'offline-readiness',
|
|
744
745
|
'python-runtime',
|
|
746
|
+
'humanizer-runtime',
|
|
745
747
|
'tier-usage-readiness',
|
|
746
748
|
'council-cli',
|
|
747
749
|
'unsupported-combos',
|
|
@@ -762,6 +764,7 @@ const GLOBAL_CHECK_IDS: ReadonlySet<string> = new Set([
|
|
|
762
764
|
'mcp-beta-readiness',
|
|
763
765
|
'offline-readiness',
|
|
764
766
|
'python-runtime',
|
|
767
|
+
'humanizer-runtime',
|
|
765
768
|
'tier-usage-readiness',
|
|
766
769
|
'council-cli',
|
|
767
770
|
'wizard-state',
|
|
@@ -1505,6 +1508,33 @@ function _check_python_runtime(): Dict {
|
|
|
1505
1508
|
};
|
|
1506
1509
|
}
|
|
1507
1510
|
|
|
1511
|
+
function _check_humanizer_runtime(): Dict {
|
|
1512
|
+
// Write-engine step 4b (humanize audit) runs `detect_ai_tells.ts` via a
|
|
1513
|
+
// Node/tsx runtime "when available", degrading to a prose-only audit
|
|
1514
|
+
// otherwise. This check surfaces which path a consumer install gets so the
|
|
1515
|
+
// default-on behavior is not a silent surprise. Either state is healthy —
|
|
1516
|
+
// the fallback is graceful by design — so it never fails the run.
|
|
1517
|
+
const node = shutilWhich('node');
|
|
1518
|
+
if (node === null) {
|
|
1519
|
+
return {
|
|
1520
|
+
id: 'humanizer-runtime',
|
|
1521
|
+
status: 'ok',
|
|
1522
|
+
message:
|
|
1523
|
+
'no Node runtime on PATH — write-engine step 4b uses the ' +
|
|
1524
|
+
'prose-only humanize audit (graceful fallback)',
|
|
1525
|
+
remedy:
|
|
1526
|
+
'install Node to enable the mechanical detect_ai_tells.ts pass; ' +
|
|
1527
|
+
'the prose audit runs regardless',
|
|
1528
|
+
};
|
|
1529
|
+
}
|
|
1530
|
+
return {
|
|
1531
|
+
id: 'humanizer-runtime',
|
|
1532
|
+
status: 'ok',
|
|
1533
|
+
message: `Node runtime present (${node}) — step 4b runs the mechanical detect_ai_tells.ts pass`,
|
|
1534
|
+
remedy: '',
|
|
1535
|
+
};
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1508
1538
|
function _check_mcp_beta_readiness(project_root: string): Dict {
|
|
1509
1539
|
// Maintainer-scoped gate: the six artefacts live in the agent-config
|
|
1510
1540
|
// SOURCE repo. In a consumer project the paths can never exist, so the
|
|
@@ -2130,6 +2160,7 @@ function _run_checks(
|
|
|
2130
2160
|
'mcp-beta-readiness': () => _check_mcp_beta_readiness(project_root),
|
|
2131
2161
|
'offline-readiness': () => _check_offline_readiness(),
|
|
2132
2162
|
'python-runtime': () => _check_python_runtime(),
|
|
2163
|
+
'humanizer-runtime': () => _check_humanizer_runtime(),
|
|
2133
2164
|
'tier-usage-readiness': () => _check_tier_usage_readiness(project_root),
|
|
2134
2165
|
'council-cli': () => _check_council_cli(project_root),
|
|
2135
2166
|
'unsupported-combos': () => _check_unsupported_combos(manifest),
|
|
@@ -2200,6 +2231,7 @@ function _run_checks_no_manifest(
|
|
|
2200
2231
|
'mcp-beta-readiness': () => _check_mcp_beta_readiness(project_root),
|
|
2201
2232
|
'offline-readiness': () => _check_offline_readiness(),
|
|
2202
2233
|
'python-runtime': () => _check_python_runtime(),
|
|
2234
|
+
'humanizer-runtime': () => _check_humanizer_runtime(),
|
|
2203
2235
|
'tier-usage-readiness': () => _check_tier_usage_readiness(project_root),
|
|
2204
2236
|
'council-cli': () => _check_council_cli(project_root),
|
|
2205
2237
|
'unsupported-combos': () => _skipped_manifest_check('unsupported-combos'),
|
|
@@ -2687,6 +2719,7 @@ export {
|
|
|
2687
2719
|
_check_offline_readiness,
|
|
2688
2720
|
_check_stale_orphans,
|
|
2689
2721
|
_check_python_runtime,
|
|
2722
|
+
_check_humanizer_runtime,
|
|
2690
2723
|
_check_mcp_beta_readiness,
|
|
2691
2724
|
_check_tier_usage_readiness,
|
|
2692
2725
|
_check_council_cli,
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic `parallelizable:` classifier recall + false-positive measurement
|
|
3
|
+
* against the orchestration corpus (road-to-orchestration-scope-decision Phase 2
|
|
4
|
+
* / road-to-subagent-value-realization-followup Phase 1 Step 3).
|
|
5
|
+
*
|
|
6
|
+
* The corpus tasks (`internal/bench/orchestration/corpus/*.md`) each document a
|
|
7
|
+
* "Classification signal" and an expected dispatch mode. This encodes those
|
|
8
|
+
* documented signals as `TaskSignals` and asserts `classifyTask` (the pure v1
|
|
9
|
+
* classifier) fires the expected mode — recall on the modes v1 covers
|
|
10
|
+
* (`do-in-parallel`, `do-in-steps`), FP=0 on the negative control, and the
|
|
11
|
+
* documented scope gap (v1 has no `do-competitively` / verdict mode).
|
|
12
|
+
*
|
|
13
|
+
* This is the durable, reproducible "record actual hit/miss counts" deliverable
|
|
14
|
+
* + a regression guard for the classifier. Deterministic — no spend, no agents.
|
|
15
|
+
*/
|
|
16
|
+
import { describe, expect, it } from 'vitest';
|
|
17
|
+
|
|
18
|
+
import { classifyTask, type ActivationInputs, type TaskSignals } from './auto_dispatch.js';
|
|
19
|
+
|
|
20
|
+
const ON: ActivationInputs = { enabled: true, auto: 'on', subagent_spawn: true };
|
|
21
|
+
|
|
22
|
+
/** Corpus signals transcribed from each task's documented "Classification signal". */
|
|
23
|
+
const CORPUS: Array<{
|
|
24
|
+
id: string;
|
|
25
|
+
signals: TaskSignals;
|
|
26
|
+
expectMode: 'do-in-parallel' | 'do-in-steps' | null;
|
|
27
|
+
note: string;
|
|
28
|
+
}> = [
|
|
29
|
+
{
|
|
30
|
+
id: 'orch-01',
|
|
31
|
+
signals: { parallelizable: 'files', independent_slices: 4, size_estimate: 4, ordered_plan: false },
|
|
32
|
+
expectMode: 'do-in-parallel',
|
|
33
|
+
note: 'N independent files, same analysis shape',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
id: 'orch-02',
|
|
37
|
+
signals: { ordered_plan: true, size_estimate: 5, independent_slices: 0 },
|
|
38
|
+
expectMode: 'do-in-steps',
|
|
39
|
+
note: 'explicit ordered numbered plan',
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
id: 'orch-03',
|
|
43
|
+
signals: { size_estimate: 3, independent_slices: 0, ordered_plan: false },
|
|
44
|
+
expectMode: null,
|
|
45
|
+
note: 'do-competitively — NOT a v1 DispatchMode (documented scope gap)',
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: 'pv-01',
|
|
49
|
+
signals: { size_estimate: 2, independent_slices: 0, ordered_plan: false },
|
|
50
|
+
expectMode: null,
|
|
51
|
+
note: 'verdict dispatch — out of the v1 parallel/steps scope',
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
id: 'pv-02',
|
|
55
|
+
signals: { size_estimate: 1, independent_slices: 0, ordered_plan: false },
|
|
56
|
+
expectMode: null,
|
|
57
|
+
note: 'negative control — a clean one-file task MUST NOT dispatch',
|
|
58
|
+
},
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
describe('parallelizable classifier — corpus recall', () => {
|
|
62
|
+
for (const c of CORPUS) {
|
|
63
|
+
it(`${c.id}: ${c.note}`, () => {
|
|
64
|
+
const r = classifyTask(c.signals, ON);
|
|
65
|
+
expect(r.mode).toBe(c.expectMode);
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
it('recall on v1-covered modes is 2/2 (orch-01 parallel, orch-02 steps)', () => {
|
|
70
|
+
const covered = CORPUS.filter((c) => c.expectMode !== null);
|
|
71
|
+
const hits = covered.filter((c) => classifyTask(c.signals, ON).mode === c.expectMode);
|
|
72
|
+
expect(covered).toHaveLength(2);
|
|
73
|
+
expect(hits).toHaveLength(2);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('false-positive rate is 0 — the negative control (pv-02) never dispatches', () => {
|
|
77
|
+
const pv02 = CORPUS.find((c) => c.id === 'pv-02')!;
|
|
78
|
+
expect(classifyTask(pv02.signals, ON).action).not.toBe('dispatch');
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('documents the v1 scope gap: do-competitively (orch-03) + verdict (pv-01) are not classified', () => {
|
|
82
|
+
for (const id of ['orch-03', 'pv-01']) {
|
|
83
|
+
const c = CORPUS.find((x) => x.id === id)!;
|
|
84
|
+
expect(classifyTask(c.signals, ON).mode).toBeNull();
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
});
|
|
@@ -108,8 +108,16 @@ export function build_claude_hook_matrix(manifest_path: string): ClaudeHookMatri
|
|
|
108
108
|
if (!concerns || (Array.isArray(concerns) && concerns.length === 0)) continue;
|
|
109
109
|
const native = ac_to_native[ac_event];
|
|
110
110
|
if (native === undefined) continue;
|
|
111
|
+
// Hook-resilience shim (road-to-opt-subagent-harvest P1.4):
|
|
112
|
+
// local binary → PATH fallback → SILENT exit 0 when neither exists
|
|
113
|
+
// (a hook must never error-spam the loop on an uninstalled repo).
|
|
114
|
+
// The dispatcher's own exit code PROPAGATES when it runs — a
|
|
115
|
+
// fail_closed concern (e.g. block-no-verify) still denies; only the
|
|
116
|
+
// missing-binary case degrades open, which is the honest degrade
|
|
117
|
+
// (an absent dispatcher cannot evaluate anything).
|
|
111
118
|
matrix[native] =
|
|
112
119
|
'BIN="$CLAUDE_PROJECT_DIR/agent-config"; [ -x "$BIN" ] || BIN=agent-config; ' +
|
|
120
|
+
'command -v "$BIN" >/dev/null 2>&1 || exit 0; ' +
|
|
113
121
|
`"$BIN" dispatch:hook --platform claude --event ${ac_event} ` +
|
|
114
122
|
`--native-event ${native} --project-dir "$CLAUDE_PROJECT_DIR" ` +
|
|
115
123
|
`--min-version ${String(hook_spec)}`;
|
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* Python→TS migration). The public API and CLI contract mirror the Python
|
|
7
7
|
* original EXACTLY — same exported snake_case names, same semantics, same
|
|
8
8
|
* tier-classification, provenance footer, and config-merge behavior, same
|
|
9
|
-
* exit codes and byte-identical stdout. No behaviour changes
|
|
9
|
+
* exit codes and byte-identical stdout. No behaviour changes at port time;
|
|
10
|
+
* one deliberate post-port change: DEFAULT_ALLOWED_TIERS narrowed to
|
|
11
|
+
* ['public'] per ADR-119 (2026-07-12).
|
|
10
12
|
*
|
|
11
13
|
* Structure-grounding v2 (ADR-100 / road-to-structure-grounding-v2). Promotes
|
|
12
14
|
* *expensive* (remote) project-local knowledge cards to a per-user, file-first
|
|
@@ -51,8 +53,10 @@ export const TIERS: readonly string[] = ['public', 'vendor', 'proprietary'];
|
|
|
51
53
|
/**
|
|
52
54
|
* Default tiers eligible to auto-cross a project boundary under default-on.
|
|
53
55
|
* `proprietary` is excluded by design — manual-only regardless of `enabled`.
|
|
56
|
+
* Narrowed to `public` only per ADR-119 (2026-07-12, council-validated
|
|
57
|
+
* flip): `vendor` is a deliberate opt-in widening, not the shipped default.
|
|
54
58
|
*/
|
|
55
|
-
export const DEFAULT_ALLOWED_TIERS: readonly string[] = ['public'
|
|
59
|
+
export const DEFAULT_ALLOWED_TIERS: readonly string[] = ['public'];
|
|
56
60
|
|
|
57
61
|
/**
|
|
58
62
|
* Hard defaults applied when the setting block is absent. Default is ON for
|
|
@@ -7,7 +7,10 @@
|
|
|
7
7
|
* the tier gate are matched byte-for-byte against the retired Python implementation. The
|
|
8
8
|
* public API and CLI contract mirror Python exactly — same exported
|
|
9
9
|
* snake_case names, same `GateResult` shape + `summary()` text, same exit
|
|
10
|
-
* codes and stdout. No behaviour changes
|
|
10
|
+
* codes and stdout. No behaviour changes at port time; one ADDITIVE
|
|
11
|
+
* hardening landed after the port (2026-07-12, ADR-103 flip validation):
|
|
12
|
+
* hidden-unicode detection + strip-then-rescan in `redaction_scan` — inputs
|
|
13
|
+
* without invisible characters are unaffected.
|
|
11
14
|
*
|
|
12
15
|
* Structure-grounding v2, Phase 1 — the privacy crux (ADR-100 /
|
|
13
16
|
* road-to-structure-grounding-v2). Before any project-local card may cross a
|
|
@@ -121,11 +124,26 @@ function _compileIgnoreCase(pattern: string): RegExp {
|
|
|
121
124
|
return new RegExp(pattern, 'i');
|
|
122
125
|
}
|
|
123
126
|
|
|
127
|
+
// Invisible / control characters usable to smuggle an identifier past the
|
|
128
|
+
// pattern scan (zero-width class, bidi controls, soft hyphen, BOM, Unicode
|
|
129
|
+
// Tag block) — the encoding-smuggling attack class from the ADR-103
|
|
130
|
+
// council re-evaluation (2026-07-11). Presence is itself a violation:
|
|
131
|
+
// a knowledge card has no legitimate need for invisible characters, and
|
|
132
|
+
// halt-and-surface (never silent-rewrite) is this gate's contract.
|
|
133
|
+
const _HIDDEN_UNICODE_RE =
|
|
134
|
+
/[----\u{E0000}-\u{E007F}]/gu;
|
|
135
|
+
|
|
124
136
|
/**
|
|
125
137
|
* Combined privacy-floor + source-confidentiality scan over `text`.
|
|
126
138
|
*
|
|
127
139
|
* Returns the list of violations (empty = clean). Reused by the Phase-4
|
|
128
140
|
* linter as the CI net that redaction actually fired on committed cards.
|
|
141
|
+
*
|
|
142
|
+
* Hidden-unicode hardening (additive, 2026-07-12): invisible characters are
|
|
143
|
+
* flagged as their own violation class AND the pattern scan runs on the
|
|
144
|
+
* stripped text, so a zero-width-broken key/email is still caught by its
|
|
145
|
+
* own category. Inputs without invisible characters behave byte-identically
|
|
146
|
+
* to the pre-hardening scan.
|
|
129
147
|
*/
|
|
130
148
|
export function redaction_scan(
|
|
131
149
|
text: string,
|
|
@@ -136,16 +154,36 @@ export function redaction_scan(
|
|
|
136
154
|
sql_identifiers?: readonly string[];
|
|
137
155
|
} = {},
|
|
138
156
|
): RedactionViolation[] {
|
|
139
|
-
const
|
|
157
|
+
const violations: RedactionViolation[] = [];
|
|
158
|
+
_HIDDEN_UNICODE_RE.lastIndex = 0;
|
|
159
|
+
const hidden = text.match(_HIDDEN_UNICODE_RE);
|
|
160
|
+
let scanText = text;
|
|
161
|
+
if (hidden !== null && hidden.length > 0) {
|
|
162
|
+
const counts = new Map<string, number>();
|
|
163
|
+
for (const ch of hidden) {
|
|
164
|
+
const cp = `U+${(ch.codePointAt(0) as number).toString(16).toUpperCase().padStart(4, '0')}`;
|
|
165
|
+
counts.set(cp, (counts.get(cp) ?? 0) + 1);
|
|
166
|
+
}
|
|
167
|
+
const summary = [...counts.entries()]
|
|
168
|
+
.map(([cp, n]) => (n > 1 ? `${cp}×${n}` : cp))
|
|
169
|
+
.join(' ');
|
|
170
|
+
violations.push({
|
|
171
|
+
category: 'hidden_unicode',
|
|
172
|
+
snippet: _pySliceHead(summary, 40),
|
|
173
|
+
note: 'invisible characters — possible identifier smuggling; pattern scan ran on the stripped text',
|
|
174
|
+
});
|
|
175
|
+
scanText = text.replace(_HIDDEN_UNICODE_RE, '');
|
|
176
|
+
}
|
|
177
|
+
const floor = redact_low_impact_entry(scanText, {
|
|
140
178
|
repoRoot: options.repo_root ?? null,
|
|
141
179
|
privateDomains: options.private_domains ?? [],
|
|
142
180
|
customerNames: options.customer_names ?? [],
|
|
143
181
|
sqlIdentifiers: options.sql_identifiers ?? [],
|
|
144
182
|
});
|
|
145
|
-
|
|
183
|
+
violations.push(...floor.violations);
|
|
146
184
|
for (const rx of _load_denylist_patterns()) {
|
|
147
185
|
rx.lastIndex = 0;
|
|
148
|
-
const m = rx.exec(
|
|
186
|
+
const m = rx.exec(scanText);
|
|
149
187
|
if (m) {
|
|
150
188
|
violations.push({
|
|
151
189
|
category: 'external_source',
|
|
@@ -90,11 +90,20 @@ export class LexicalIndex {
|
|
|
90
90
|
return Math.max(0, Math.log(1 + (this.n - df + 0.5) / (df + 0.5)));
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
/**
|
|
93
|
+
/**
|
|
94
|
+
* BM25 score of one document against the tokenised query terms, weighted by
|
|
95
|
+
* **term coverage**: the raw BM25 sum is multiplied by `coverage²`, where
|
|
96
|
+
* `coverage = distinct-query-terms-matched / distinct-query-terms`. This
|
|
97
|
+
* stops a single generic, high-TF term match from outranking a document that
|
|
98
|
+
* matches several rarer query terms once each (Phase 1 refinement). A
|
|
99
|
+
* single-term query has `coverage == 1`, so its score is unchanged — the
|
|
100
|
+
* weighting only re-orders genuinely multi-term queries.
|
|
101
|
+
*/
|
|
94
102
|
score(queryTerms: readonly string[], docId: string): number {
|
|
95
103
|
const doc = this.docs.get(docId);
|
|
96
104
|
if (doc === undefined || doc.len === 0) return 0;
|
|
97
105
|
let s = 0;
|
|
106
|
+
const matched = new Set<string>();
|
|
98
107
|
for (const term of queryTerms) {
|
|
99
108
|
const tf = doc.tf.get(term);
|
|
100
109
|
if (tf === undefined) continue;
|
|
@@ -102,8 +111,12 @@ export class LexicalIndex {
|
|
|
102
111
|
if (idf === 0) continue;
|
|
103
112
|
const denom = tf + BM25_K1 * (1 - BM25_B + (BM25_B * doc.len) / (this.avgdl || 1));
|
|
104
113
|
s += (idf * (tf * (BM25_K1 + 1))) / denom;
|
|
114
|
+
matched.add(term);
|
|
105
115
|
}
|
|
106
|
-
return
|
|
116
|
+
if (s === 0) return 0;
|
|
117
|
+
const distinctQuery = new Set(queryTerms).size;
|
|
118
|
+
const coverage = distinctQuery > 0 ? matched.size / distinctQuery : 1;
|
|
119
|
+
return s * coverage * coverage;
|
|
107
120
|
}
|
|
108
121
|
|
|
109
122
|
/**
|
|
@@ -24,6 +24,17 @@ export type TierSource = 'static' | 'inferred' | 'inherit';
|
|
|
24
24
|
export type Band = 'low' | 'medium' | 'high';
|
|
25
25
|
export type LinePhase = 'refine' | 'memory' | 'analyze' | 'plan' | 'implement' | 'test' | 'verify' | 'report';
|
|
26
26
|
export type LineOutcome = 'success' | 'blocked' | 'skipped' | 'error';
|
|
27
|
+
/** The orchestration form the form-gate selected (road-to-opt-subagent-harvest P2). */
|
|
28
|
+
export type DispatchModeId =
|
|
29
|
+
| 'do-and-judge'
|
|
30
|
+
| 'do-and-judge-two-stage'
|
|
31
|
+
| 'do-in-steps'
|
|
32
|
+
| 'do-in-parallel'
|
|
33
|
+
| 'do-competitively'
|
|
34
|
+
| 'judge-with-debate'
|
|
35
|
+
| 'do-in-worktrees'
|
|
36
|
+
| 'do-with-live-app-judge'
|
|
37
|
+
| 'none';
|
|
27
38
|
|
|
28
39
|
/** Inputs the agent supplies about one auto-dispatch. */
|
|
29
40
|
export interface RecordInput {
|
|
@@ -37,6 +48,8 @@ export interface RecordInput {
|
|
|
37
48
|
tier_chosen?: TierChosen | null | undefined;
|
|
38
49
|
tier_source?: TierSource | null | undefined;
|
|
39
50
|
task_class?: string | null | undefined;
|
|
51
|
+
/** Form-gate outcome: which of the seven modes (or 'none') the gate selected. */
|
|
52
|
+
dispatch_mode?: DispatchModeId | null | undefined;
|
|
40
53
|
task_size_estimate?: number | undefined;
|
|
41
54
|
wall_clock_ms?: number | undefined;
|
|
42
55
|
/** Absolute measured tokens the dispatched slice consumed (feeds the modeled cost-%). */
|
|
@@ -45,6 +58,10 @@ export interface RecordInput {
|
|
|
45
58
|
session_tier?: string | null | undefined;
|
|
46
59
|
dispatch_outcome?: DispatchOutcome | undefined;
|
|
47
60
|
verify_mode?: VerifyMode | undefined;
|
|
61
|
+
/** QUALITY: subagent return adopted without parent rework. Optional boolean (null = not measured). */
|
|
62
|
+
first_pass_success?: boolean | undefined;
|
|
63
|
+
/** QUALITY: slice retried on a higher tier after a verification failure. Optional boolean (null = not measured). */
|
|
64
|
+
escalated?: boolean | undefined;
|
|
48
65
|
// Audit-log envelope (sensible defaults for a dispatch record)
|
|
49
66
|
phase?: LinePhase | undefined;
|
|
50
67
|
outcome?: LineOutcome | undefined;
|
|
@@ -127,6 +144,13 @@ export function buildOrchestrationLine(input: RecordInput): BuiltLine {
|
|
|
127
144
|
const phase = input.phase ?? 'implement';
|
|
128
145
|
if (!PHASES.includes(phase)) errors.push(`phase must be one of ${PHASES.join(' | ')}`);
|
|
129
146
|
|
|
147
|
+
if (input.first_pass_success !== undefined && typeof input.first_pass_success !== 'boolean') {
|
|
148
|
+
errors.push('first_pass_success must be a boolean (true = return adopted without parent rework) or omitted');
|
|
149
|
+
}
|
|
150
|
+
if (input.escalated !== undefined && typeof input.escalated !== 'boolean') {
|
|
151
|
+
errors.push('escalated must be a boolean (true = retried on a higher tier after verification failure) or omitted');
|
|
152
|
+
}
|
|
153
|
+
|
|
130
154
|
if (!input.ts) errors.push('ts (ISO-8601 UTC) is required');
|
|
131
155
|
if (!input.id) errors.push('id (ULID or content hash) is required');
|
|
132
156
|
|
|
@@ -142,10 +166,13 @@ export function buildOrchestrationLine(input: RecordInput): BuiltLine {
|
|
|
142
166
|
outcome: dOutcome,
|
|
143
167
|
verify_mode: vMode,
|
|
144
168
|
task_class: input.task_class ?? null,
|
|
169
|
+
dispatch_mode: input.dispatch_mode ?? null,
|
|
145
170
|
tier_chosen: input.tier_chosen ?? null,
|
|
146
171
|
tier_source: input.tier_source ?? null,
|
|
147
172
|
dispatch_tokens: input.dispatch_tokens ?? null,
|
|
148
173
|
session_tier: input.session_tier ?? null,
|
|
174
|
+
first_pass_success: input.first_pass_success ?? null,
|
|
175
|
+
escalated: input.escalated ?? null,
|
|
149
176
|
};
|
|
150
177
|
|
|
151
178
|
const line: Record<string, unknown> = {
|
|
@@ -39,6 +39,10 @@ export interface OrchestrationRecord {
|
|
|
39
39
|
dispatch_tokens?: number | null;
|
|
40
40
|
/** The orchestrator's own tier — the baseline the downshift is measured against. */
|
|
41
41
|
session_tier?: string | null;
|
|
42
|
+
/** QUALITY: subagent return adopted without parent rework. null/absent = pre-extension line / not measured. */
|
|
43
|
+
first_pass_success?: boolean | null;
|
|
44
|
+
/** QUALITY: slice retried on a higher tier after a verification failure. null/absent = pre-extension line / not measured. */
|
|
45
|
+
escalated?: boolean | null;
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
/** A parsed audit-log-v1 line (only the fields this aggregator touches). */
|
|
@@ -65,6 +69,21 @@ export interface ModeledCost {
|
|
|
65
69
|
weights: TierWeights;
|
|
66
70
|
}
|
|
67
71
|
|
|
72
|
+
export interface QualityStats {
|
|
73
|
+
/** Dispatches carrying at least one of the two quality booleans (the n behind the ≥20 pairing gate). */
|
|
74
|
+
quality_lines: number;
|
|
75
|
+
/** Dispatches carrying a boolean first_pass_success. */
|
|
76
|
+
first_pass_lines: number;
|
|
77
|
+
first_pass_success_count: number;
|
|
78
|
+
/** first_pass_success_count / first_pass_lines, 0..1; null when no line carries the field. */
|
|
79
|
+
first_pass_success_rate: number | null;
|
|
80
|
+
/** Dispatches carrying a boolean escalated. */
|
|
81
|
+
escalated_lines: number;
|
|
82
|
+
escalated_count: number;
|
|
83
|
+
/** escalated_count / escalated_lines, 0..1; null when no line carries the field. */
|
|
84
|
+
escalation_rate: number | null;
|
|
85
|
+
}
|
|
86
|
+
|
|
68
87
|
export interface SavingsReport {
|
|
69
88
|
/** Lines carrying an orchestration object with spawn_count > 0. */
|
|
70
89
|
dispatches: number;
|
|
@@ -84,6 +103,8 @@ export interface SavingsReport {
|
|
|
84
103
|
measured_share: number;
|
|
85
104
|
/** Modeled downshift cost-% (tier-weight based). Distinct from the measured token counts. */
|
|
86
105
|
modeled_cost: ModeledCost;
|
|
106
|
+
/** QUALITY dimension — paired with cost per council verdict; never report savings alone. */
|
|
107
|
+
quality: QualityStats;
|
|
87
108
|
/** Honest caveats about what the number does and does not mean. */
|
|
88
109
|
notes: string[];
|
|
89
110
|
}
|
|
@@ -115,6 +136,15 @@ export function aggregateOrchestrationSavings(
|
|
|
115
136
|
by_task_class: {},
|
|
116
137
|
measured_share: 0,
|
|
117
138
|
modeled_cost: { covered_dispatches: 0, baseline_cost_units: 0, delegated_cost_units: 0, cost_reduction_pct: null, weights },
|
|
139
|
+
quality: {
|
|
140
|
+
quality_lines: 0,
|
|
141
|
+
first_pass_lines: 0,
|
|
142
|
+
first_pass_success_count: 0,
|
|
143
|
+
first_pass_success_rate: null,
|
|
144
|
+
escalated_lines: 0,
|
|
145
|
+
escalated_count: 0,
|
|
146
|
+
escalation_rate: null,
|
|
147
|
+
},
|
|
118
148
|
notes: [],
|
|
119
149
|
};
|
|
120
150
|
|
|
@@ -154,6 +184,21 @@ export function aggregateOrchestrationSavings(
|
|
|
154
184
|
report.modeled_cost.baseline_cost_units += dt * wSession;
|
|
155
185
|
report.modeled_cost.delegated_cost_units += dt * wChosen;
|
|
156
186
|
}
|
|
187
|
+
|
|
188
|
+
// Quality dimension — only booleans count; null/absent = pre-extension
|
|
189
|
+
// line / not measured (never a silent false).
|
|
190
|
+
const q = report.quality;
|
|
191
|
+
const fps = typeof o.first_pass_success === 'boolean';
|
|
192
|
+
const esc = typeof o.escalated === 'boolean';
|
|
193
|
+
if (fps || esc) q.quality_lines += 1;
|
|
194
|
+
if (fps) {
|
|
195
|
+
q.first_pass_lines += 1;
|
|
196
|
+
if (o.first_pass_success === true) q.first_pass_success_count += 1;
|
|
197
|
+
}
|
|
198
|
+
if (esc) {
|
|
199
|
+
q.escalated_lines += 1;
|
|
200
|
+
if (o.escalated === true) q.escalated_count += 1;
|
|
201
|
+
}
|
|
157
202
|
}
|
|
158
203
|
|
|
159
204
|
report.measured_share = report.dispatches === 0 ? 0 : report.by_provenance.measured.dispatches / report.dispatches;
|
|
@@ -161,6 +206,10 @@ export function aggregateOrchestrationSavings(
|
|
|
161
206
|
const mc = report.modeled_cost;
|
|
162
207
|
mc.cost_reduction_pct = mc.baseline_cost_units > 0 ? (mc.baseline_cost_units - mc.delegated_cost_units) / mc.baseline_cost_units : null;
|
|
163
208
|
|
|
209
|
+
const q = report.quality;
|
|
210
|
+
q.first_pass_success_rate = q.first_pass_lines > 0 ? q.first_pass_success_count / q.first_pass_lines : null;
|
|
211
|
+
q.escalation_rate = q.escalated_lines > 0 ? q.escalated_count / q.escalated_lines : null;
|
|
212
|
+
|
|
164
213
|
// Honest caveats — always attached so a consumer never over-reads the number.
|
|
165
214
|
if (report.dispatches === 0) {
|
|
166
215
|
report.notes.push('No orchestration telemetry yet (0 dispatches). Data accrues as `subagents.auto: on` delegates real slices (recorded via orchestration_record).');
|
|
@@ -174,6 +223,9 @@ export function aggregateOrchestrationSavings(
|
|
|
174
223
|
} else {
|
|
175
224
|
report.notes.push('MODELED cost-% unavailable: no dispatch yet carries dispatch_tokens + session_tier + tier_chosen. Record those (orchestration_record --dispatch-tokens/--session-tier/--tier-chosen) to populate it.');
|
|
176
225
|
}
|
|
226
|
+
if (q.quality_lines === 0) {
|
|
227
|
+
report.notes.push('QUALITY columns empty: no dispatch yet carries first_pass_success / escalated. Record them (orchestration_record --first-pass-success/--escalated) — cost and quality are reported as a PAIR, never savings alone.');
|
|
228
|
+
}
|
|
177
229
|
}
|
|
178
230
|
|
|
179
231
|
return report;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared trigger-eval precision/recall floors — single source of truth.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `src/cli/commands/recordTriggerEval.ts` so the weekly
|
|
5
|
+
* canary rotation (`src/scripts/trigger_eval_rotation.ts`, ADR-118 §4) and
|
|
6
|
+
* the `eval:record` provenance recorder enforce the SAME per-domain floors.
|
|
7
|
+
* Values are unchanged from the recorder's originals (Phase D council
|
|
8
|
+
* verdict — tuned to the trigger pattern each skill shows, not a single
|
|
9
|
+
* global pair): a reference task wants near-perfect precision; a judgment
|
|
10
|
+
* task tolerates a looser one.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export interface TriggerEvalFloor {
|
|
14
|
+
minRecall: number;
|
|
15
|
+
minPrecision: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Universal fallback floor when a skill carries no domain-specific entry. */
|
|
19
|
+
export const DEFAULT_FLOOR: TriggerEvalFloor = { minRecall: 1.0, minPrecision: 0.8 };
|
|
20
|
+
|
|
21
|
+
/** Domain-specific floors; skills absent here fall back to {@link DEFAULT_FLOOR}. */
|
|
22
|
+
export const DOMAIN_FLOORS: Readonly<Record<string, TriggerEvalFloor>> = {
|
|
23
|
+
'image-generation': { minRecall: 1.0, minPrecision: 0.85 },
|
|
24
|
+
iconography: { minRecall: 1.0, minPrecision: 0.9 },
|
|
25
|
+
'brand-strategy': { minRecall: 0.9, minPrecision: 0.7 },
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** Resolve the effective floor for a skill (domain entry, else default). */
|
|
29
|
+
export function floor_for(skill: string | null | undefined): TriggerEvalFloor {
|
|
30
|
+
return (skill !== null && skill !== undefined && DOMAIN_FLOORS[skill]) || DEFAULT_FLOOR;
|
|
31
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Chairman selection (road-to-opt-council-deliberation Phase 2).
|
|
2
|
+
//
|
|
3
|
+
// The council's Iron Law argues the host, having framed the artefact, cannot
|
|
4
|
+
// independently judge it — so synthesis by a *non-deliberating* member removes
|
|
5
|
+
// that bias. This module is the pure SELECTION decision (who chairs, and the
|
|
6
|
+
// visible verdict annotation); the billable dispatch of the chairman's synthesis
|
|
7
|
+
// call lives in the CLI run path, gated behind the same config.
|
|
8
|
+
//
|
|
9
|
+
// `auto` policy — decided by the contested-design council pass (anthropic/
|
|
10
|
+
// claude-sonnet-4-5 + openai/gpt-4o, 2-round debate, 2026-07-12):
|
|
11
|
+
// **provider-family difference is primary** — independence from the
|
|
12
|
+
// deliberators' priors is the binding constraint for a judge, and provider
|
|
13
|
+
// diversity is the one structural guarantee available without an LLM call.
|
|
14
|
+
// The **tie-break is the capability signal**: an explicit optional
|
|
15
|
+
// `members.<name>.tier` when present (higher = stronger), else deterministic
|
|
16
|
+
// config order. In this engine each member IS one provider (the
|
|
17
|
+
// one-advisor-per-provider invariant), so every enabled member that did not
|
|
18
|
+
// deliberate is provider-different from every deliberator by construction.
|
|
19
|
+
|
|
20
|
+
export interface ChairmanCandidate {
|
|
21
|
+
/** Member name (== provider key in this engine). */
|
|
22
|
+
name: string;
|
|
23
|
+
/** Optional capability rank from `members.<name>.tier`; higher = stronger. */
|
|
24
|
+
tier: number | null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface ChairmanSelection {
|
|
28
|
+
/** The chosen chairman member name, or `null` → host synthesis (today's path). */
|
|
29
|
+
member: string | null;
|
|
30
|
+
/** Visible annotation for the synthesis verdict — never a silent substitution. */
|
|
31
|
+
annotation: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Pure chairman selection.
|
|
36
|
+
*
|
|
37
|
+
* @param mode one of `host` | `member` | `auto` (validated at config load)
|
|
38
|
+
* @param member the configured member name when `mode === 'member'`
|
|
39
|
+
* @param deliberated member names that produced a real (non-error, non-empty) response this session
|
|
40
|
+
* @param candidates enabled members in **config order** (the engine's only trusted ordering)
|
|
41
|
+
*/
|
|
42
|
+
export function select_chairman(
|
|
43
|
+
mode: string,
|
|
44
|
+
member: string | null,
|
|
45
|
+
deliberated: ReadonlySet<string>,
|
|
46
|
+
candidates: readonly ChairmanCandidate[],
|
|
47
|
+
): ChairmanSelection {
|
|
48
|
+
if (mode === 'host') {
|
|
49
|
+
return { member: null, annotation: 'Chairman: host' };
|
|
50
|
+
}
|
|
51
|
+
const enabledNames = new Set(candidates.map((c) => c.name));
|
|
52
|
+
if (mode === 'member') {
|
|
53
|
+
if (member === null || !enabledNames.has(member)) {
|
|
54
|
+
return {
|
|
55
|
+
member: null,
|
|
56
|
+
annotation: `Chairman: host (member ${_q(member)} unavailable — host fallback)`,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
if (deliberated.has(member)) {
|
|
60
|
+
// A member that argued in the debate cannot independently judge it.
|
|
61
|
+
return {
|
|
62
|
+
member: null,
|
|
63
|
+
annotation: `Chairman: host (member ${_q(member)} deliberated — cannot self-judge, host fallback)`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
return { member, annotation: `Chairman: ${member}` };
|
|
67
|
+
}
|
|
68
|
+
// mode === 'auto' — provider-family difference primary (non-deliberating ⇒
|
|
69
|
+
// provider-different by construction here); tier tie-break; config-order final.
|
|
70
|
+
const pool = candidates.filter((c) => !deliberated.has(c.name));
|
|
71
|
+
if (pool.length === 0) {
|
|
72
|
+
return {
|
|
73
|
+
member: null,
|
|
74
|
+
annotation: 'Chairman: host (no non-panel member available)',
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
let best = pool[0] as ChairmanCandidate;
|
|
78
|
+
for (const c of pool.slice(1)) {
|
|
79
|
+
const bestTier = best.tier ?? Number.NEGATIVE_INFINITY;
|
|
80
|
+
const cTier = c.tier ?? Number.NEGATIVE_INFINITY;
|
|
81
|
+
if (cTier > bestTier) {
|
|
82
|
+
best = c; // strictly higher tier wins; equal/lower keeps config order
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return { member: best.name, annotation: `Chairman: ${best.name} (auto)` };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function _q(v: string | null): string {
|
|
89
|
+
return v === null ? 'null' : `'${v}'`;
|
|
90
|
+
}
|
|
@@ -71,6 +71,21 @@ export const DEFAULT_GEMINI_MODEL = 'gemini-2.5-pro';
|
|
|
71
71
|
export const DEFAULT_XAI_MODEL = 'grok-4';
|
|
72
72
|
export const DEFAULT_PERPLEXITY_MODEL = 'sonar-pro';
|
|
73
73
|
|
|
74
|
+
// Vendor CLI (subscription transport) default models. Omitting `model` in a
|
|
75
|
+
// CLI client's options pins THESE values — a pin, not "latest"; the vendor
|
|
76
|
+
// CLI's own default may be newer. Bump deliberately; never assume drift.
|
|
77
|
+
//
|
|
78
|
+
// These are SEPARATE from the API-transport `DEFAULT_*_MODEL` constants above
|
|
79
|
+
// because API and CLI defaults can legitimately diverge — openai proves it
|
|
80
|
+
// (API `gpt-4o` vs CLI `gpt-5`). The anthropic/gemini CLI values currently
|
|
81
|
+
// coincide with their API defaults, but stay pinned independently so a future
|
|
82
|
+
// divergence is a one-line change, not a hunt through inline literals.
|
|
83
|
+
// (`xai` / `perplexity` CLI reuse their API constants: their CLIs are
|
|
84
|
+
// community wrappers around the same paid API, so the values do not diverge.)
|
|
85
|
+
export const DEFAULT_OPENAI_CLI_MODEL = 'gpt-5';
|
|
86
|
+
export const DEFAULT_ANTHROPIC_CLI_MODEL = 'claude-sonnet-4-5';
|
|
87
|
+
export const DEFAULT_GEMINI_CLI_MODEL = 'gemini-2.5-pro';
|
|
88
|
+
|
|
74
89
|
// OpenAI-API-compatible endpoints. xAI and Perplexity both expose the
|
|
75
90
|
// `/v1/chat/completions` shape, so their clients reuse the `openai` SDK with a
|
|
76
91
|
// custom `base_url`. Gemini has its own SDK (`google-genai`).
|
|
@@ -1171,7 +1186,7 @@ export class AnthropicCliClient extends CliClient {
|
|
|
1171
1186
|
|
|
1172
1187
|
constructor(opts: CliClientOptions = {}) {
|
|
1173
1188
|
super({
|
|
1174
|
-
model: opts.model ??
|
|
1189
|
+
model: opts.model ?? DEFAULT_ANTHROPIC_CLI_MODEL,
|
|
1175
1190
|
binary: opts.binary,
|
|
1176
1191
|
timeout_seconds: opts.timeout_seconds,
|
|
1177
1192
|
max_calls_per_day: opts.max_calls_per_day,
|
|
@@ -1271,7 +1286,7 @@ export class OpenAICliClient extends CliClient {
|
|
|
1271
1286
|
|
|
1272
1287
|
constructor(opts: CliClientOptions = {}) {
|
|
1273
1288
|
super({
|
|
1274
|
-
model: opts.model ??
|
|
1289
|
+
model: opts.model ?? DEFAULT_OPENAI_CLI_MODEL,
|
|
1275
1290
|
binary: opts.binary,
|
|
1276
1291
|
timeout_seconds: opts.timeout_seconds,
|
|
1277
1292
|
max_calls_per_day: opts.max_calls_per_day,
|
|
@@ -1394,7 +1409,7 @@ export class GeminiCliClient extends CliClient {
|
|
|
1394
1409
|
|
|
1395
1410
|
constructor(opts: CliClientOptions = {}) {
|
|
1396
1411
|
super({
|
|
1397
|
-
model: opts.model ??
|
|
1412
|
+
model: opts.model ?? DEFAULT_GEMINI_CLI_MODEL,
|
|
1398
1413
|
binary: opts.binary,
|
|
1399
1414
|
timeout_seconds: opts.timeout_seconds,
|
|
1400
1415
|
max_calls_per_day: opts.max_calls_per_day,
|