@event4u/agent-config 14.20.0 → 14.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.augment-plugin/marketplace.json +3 -3
- package/.augment-plugin/plugin.json +1 -1
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +441 -0
- package/CREDITS.md +37 -0
- package/README.md +1 -1
- package/agents/templates/.ai-council.yml.example +10 -0
- package/dist/agent-src/commands/analyze/inbox.md +381 -6
- package/dist/agent-src/commands/analyze/{reference-repo.md → repo.md} +251 -34
- package/dist/agent-src/commands/analyze/roadmap-repos.md +129 -0
- package/dist/agent-src/commands/analyze.md +12 -7
- package/dist/agent-src/commands/feature/plan.md +55 -16
- package/dist/agent-src/commands/feature/refactor.md +8 -3
- package/dist/agent-src/commands/humanize.md +48 -4
- package/dist/agent-src/contexts/communication/rules-auto/token-efficiency-mechanics.md +69 -0
- package/dist/agent-src/contexts/execution/auto-dispatch-classification.md +5 -5
- package/dist/agent-src/contexts/execution/contract-decision-sheet.md +22 -1
- package/dist/agent-src/contexts/execution/host-capability-manifest.md +4 -1
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +35 -2
- package/dist/agent-src/contexts/execution/terminal-states.md +4 -1
- package/dist/agent-src/rules/evaluator-independence.md +4 -4
- package/dist/agent-src/rules/external-code-graph-interop.md +19 -14
- package/dist/agent-src/rules/external-reference-deep-dive.md +2 -2
- package/dist/agent-src/rules/token-efficiency.md +26 -3
- package/dist/agent-src/scripts/_cli_entry.ts +51 -0
- package/dist/agent-src/scripts/archive_completed_roadmaps.ts +14 -23
- package/dist/agent-src/scripts/roadmap_gates.ts +4 -20
- package/dist/agent-src/scripts/roadmap_set_step.ts +5 -2
- package/dist/agent-src/scripts/stubs_due.ts +4 -18
- package/dist/agent-src/scripts/update_roadmap_progress.ts +7 -24
- package/dist/agent-src/skills/code-intelligence/SKILL.md +25 -0
- package/dist/agent-src/skills/competitive-positioning/SKILL.md +2 -2
- package/dist/agent-src/skills/humanizer/references/anti-aiisms.md +27 -11
- package/dist/agent-src/skills/learning-to-rule-or-skill/SKILL.md +1 -1
- package/dist/agent-src/skills/markitdown/SKILL.md +1 -1
- package/dist/agent-src/skills/repomix-packer/SKILL.md +1 -1
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +17 -0
- package/dist/agent-src/skills/technical-specification/SKILL.md +4 -0
- package/dist/agent-src/templates/agent-settings.md +0 -1
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/features.md +5 -0
- package/dist/agent-src/templates/roadmaps.md +15 -0
- package/dist/cli-delegate/_cli_entry.js +8 -0
- package/dist/cli-delegate/_lib/pin_resolver.js +170 -0
- package/dist/cli-delegate/archival_sweep.js +8 -0
- package/dist/cli-delegate/archive_completed_roadmaps.js +494 -0
- package/dist/cli-delegate/check_update_banner.js +923 -0
- package/dist/cli-delegate/{chunk-KPBWFOPZ.js → chunk-2FTMENJL.js} +194 -37
- package/dist/cli-delegate/{chunk-PI2GU3DP.js → chunk-2TAQ2K4V.js} +4 -1
- package/dist/cli-delegate/chunk-5EK5WQN6.js +28 -0
- package/dist/cli-delegate/chunk-7XGD2OEU.js +96 -0
- package/dist/cli-delegate/{chunk-C6IPZRDA.js → chunk-BIXEDQPZ.js} +7 -1
- package/dist/cli-delegate/chunk-DBH4AHDK.js +155 -0
- package/dist/cli-delegate/{chunk-NUMQHLS5.js → chunk-J3RFNKG4.js} +21 -2
- package/dist/cli-delegate/chunk-JO25G5H7.js +20 -0
- package/dist/cli-delegate/chunk-LTYVIY76.js +7961 -0
- package/dist/cli-delegate/chunk-QAJS5YWY.js +999 -0
- package/dist/cli-delegate/{chunk-ZVBOHSVF.js → chunk-TGIPZBCW.js} +1 -1
- package/dist/cli-delegate/{chunk-YS6XTLEK.js → chunk-TYSY33MD.js} +56 -6
- package/dist/cli-delegate/chunk-ULET4BYS.js +328 -0
- package/dist/cli-delegate/chunk-YDQSHTD7.js +187 -0
- package/dist/cli-delegate/chunk-YWVXLR44.js +48 -0
- package/dist/cli-delegate/chunk-ZZQBUHFK.js +273 -0
- package/dist/cli-delegate/cmd_conformance.js +19 -5
- package/dist/cli-delegate/cmd_doctor.js +3 -3
- package/dist/cli-delegate/cmd_export.js +2 -2
- package/dist/cli-delegate/cmd_handoff.js +27 -3
- package/dist/cli-delegate/cmd_prune.js +2 -2
- package/dist/cli-delegate/cmd_session_recycle.js +2 -2
- package/dist/cli-delegate/cmd_settings_migrate.js +2 -2
- package/dist/cli-delegate/cmd_sync.js +2 -2
- package/dist/cli-delegate/cmd_uninstall.js +2 -2
- package/dist/cli-delegate/cmd_validate.js +2 -2
- package/dist/cli-delegate/dashboard_mode.js +16 -0
- package/dist/cli-delegate/gate_budget.js +24 -0
- package/dist/cli-delegate/gate_execute.js +25 -0
- package/dist/cli-delegate/guarded_baseline.js +18 -0
- package/dist/cli-delegate/resume_probe.js +22 -0
- package/dist/cli-delegate/roadmap_gates.js +638 -0
- package/dist/cli-delegate/roadmap_set_step.js +173 -0
- package/dist/cli-delegate/stub_queue.js +47 -0
- package/dist/cli-delegate/stubs_due.js +38 -0
- package/dist/cli-delegate/ticket_bundles.js +59 -0
- package/dist/cli-delegate/update_roadmap_progress.js +60 -0
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +60 -29
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +3 -3
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +7 -6
- package/dist/discovery/trust-report.md +2 -2
- package/dist/discovery/workspaces.json +6 -5
- package/dist/hooks/dispatch.js +1522 -1008
- package/dist/install/claudeLayerCarriage.js +237 -0
- package/dist/install/claudePathsPlan.js +79 -0
- package/dist/install/hostLayerFingerprint.js +1 -1
- package/dist/install/install.mjs +10 -1
- package/dist/install/partitionEligibility.js +231 -117
- package/dist/install/ruleLayerPartition.js +10 -7
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +39376 -38902
- package/dist/scripts/ai_council/content_ceiling.js +63 -0
- package/dist/scripts/ai_council/seat_policy.js +106 -0
- package/dist/scripts/ai_council/transport_resolver.js +21 -0
- package/dist/scripts/condense.js +32 -25
- package/docs/MIGRATION.md +49 -1
- package/docs/contracts/settings-classes.md +1 -1
- package/docs/guidelines/agent-infra/minimal-safe-diff-mechanics.md +73 -0
- package/llms.txt +14 -14
- package/package.json +7 -4
- package/src/agent-src/scripts/_cli_entry.ts +51 -0
- package/src/agent-src/scripts/archive_completed_roadmaps.ts +14 -23
- package/src/agent-src/scripts/roadmap_gates.ts +4 -20
- package/src/agent-src/scripts/roadmap_set_step.ts +5 -2
- package/src/agent-src/scripts/stubs_due.ts +4 -18
- package/src/agent-src/scripts/update_roadmap_progress.ts +7 -24
- package/src/config/continuity-surface.json +509 -0
- package/src/config/estate-count-budget.json +1 -1
- package/src/config/gate-coverage.yml +29 -8
- package/src/config/gate-violation-baselines.json +8 -8
- package/src/config/gitignore-block.txt +19 -0
- package/src/config/host-injection-effect.json +73 -0
- package/src/config/packed-binary-manifest.json +33 -0
- package/src/config/preamble-payload-budget.json +112 -105
- package/src/config/publish-surface.json +4 -2
- package/src/config/release-gate-locality.yml +4 -3
- package/src/install/claudeLayerCarriage.ts +273 -0
- package/src/install/claudePathsPlan.ts +76 -0
- package/src/install/hostLayerFingerprint.ts +1 -1
- package/src/install/partitionEligibility.ts +261 -127
- package/src/install/ruleLayerPartition.ts +19 -23
- package/src/scripts/_dispatch.bash +77 -27
- package/src/scripts/_lib/auto_dispatch.ts +2 -2
- package/src/scripts/_lib/blocked_by_marker.ts +189 -0
- package/src/scripts/_lib/catalogue_layer_parity.ts +146 -0
- package/src/scripts/_lib/continuity_surface.ts +296 -0
- package/src/scripts/_lib/corpus_manifest.ts +26 -9
- package/src/scripts/_lib/hook_settings.ts +1 -1
- package/src/scripts/_lib/host_capability.ts +38 -1
- package/src/scripts/_lib/injection_effect.ts +151 -0
- package/src/scripts/_lib/layer_overlap_notice.ts +50 -6
- package/src/scripts/_lib/packed_binary_predicate.ts +225 -0
- package/src/scripts/_lib/pin_resolver.ts +13 -0
- package/src/scripts/_lib/regression_neighbourhood.ts +1 -1
- package/src/scripts/_lib/release_highlights.ts +19 -211
- package/src/scripts/_lib/release_material.ts +27 -77
- package/src/scripts/_lib/retired_status.ts +74 -0
- package/src/scripts/_lib/skill_catalogue.ts +156 -21
- package/src/scripts/_lib/structured_ask.ts +117 -0
- package/src/scripts/_lib/subagent_capsule.ts +117 -8
- package/src/scripts/_lib/tty_prompt.ts +76 -0
- package/src/scripts/ai_council/api_transport.ts +168 -0
- package/src/scripts/ai_council/clients.ts +58 -63
- package/src/scripts/ai_council/config.ts +13 -0
- package/src/scripts/ai_council/content_ceiling.ts +79 -0
- package/src/scripts/ai_council/deliberation_metrics.ts +353 -0
- package/src/scripts/ai_council/jury_aggregate.ts +178 -0
- package/src/scripts/ai_council/prompts.ts +9 -0
- package/src/scripts/ai_council/seat_policy.ts +137 -0
- package/src/scripts/ai_council/status_surface.ts +79 -0
- package/src/scripts/ai_council/synthesis_disagreement.ts +206 -0
- package/src/scripts/ai_council/transport_resolver.ts +59 -1
- package/src/scripts/ai_tells_rules.ts +446 -5
- package/src/scripts/ask_block_census.ts +499 -0
- package/src/scripts/bench_humanizer_eval.ts +264 -40
- package/src/scripts/bench_prose_tells_fp.ts +198 -0
- package/src/scripts/capture_skill_catalogue.ts +24 -11
- package/src/scripts/check_branch_work_committed.ts +234 -0
- package/src/scripts/check_candidate_lines.ts +499 -0
- package/src/scripts/check_continuity_surface.ts +475 -0
- package/src/scripts/check_dependency_floors.ts +13 -6
- package/src/scripts/check_estate_count.ts +40 -79
- package/src/scripts/check_pack_size.ts +263 -24
- package/src/scripts/check_release_highlights.ts +0 -157
- package/src/scripts/check_roadmap_trackable.ts +19 -16
- package/src/scripts/check_rule_layer_partition.ts +54 -34
- package/src/scripts/check_single_delivery.ts +97 -31
- package/src/scripts/check_update_banner.ts +13 -0
- package/src/scripts/code_graph/build.ts +138 -15
- package/src/scripts/code_graph/cli.ts +76 -8
- package/src/scripts/code_graph/extract.ts +11 -0
- package/src/scripts/code_graph/grammar_probe.ts +43 -0
- package/src/scripts/code_graph/loader.ts +88 -14
- package/src/scripts/code_graph/query.ts +121 -11
- package/src/scripts/code_graph/resolution_tiers.ts +238 -0
- package/src/scripts/code_graph/sqlite_store.ts +247 -6
- package/src/scripts/code_graph/types.ts +66 -1
- package/src/scripts/code_graph/validate.ts +23 -0
- package/src/scripts/condense.ts +34 -22
- package/src/scripts/corpus_manifest.ts +5 -5
- package/src/scripts/council_cli.ts +29 -39
- package/src/scripts/count_intent_disagreement.ts +426 -0
- package/src/scripts/detect_ai_tells.ts +142 -20
- package/src/scripts/handoff_context_hook.ts +79 -0
- package/src/scripts/harvest_reference_tokens.ts +698 -0
- package/src/scripts/hook_effect_doctor.ts +82 -1
- package/src/scripts/hook_manifest.json +1 -1
- package/src/scripts/hook_manifest.yaml +94 -12
- package/src/scripts/hooks/chain_nudge_hook.ts +282 -0
- package/src/scripts/hooks/code_graph_context_hook.ts +212 -0
- package/src/scripts/hooks/concern_registry.ts +6 -2
- package/src/scripts/hooks/delegation_nudge_hook.ts +45 -1
- package/src/scripts/hooks/host_semantics.ts +1 -1
- package/src/scripts/hooks/one_question_per_ask_hook.ts +191 -0
- package/src/scripts/hooks/orchestration_record_hook.ts +2 -2
- package/src/scripts/hooks/rule_inject_hook.ts +1 -1
- package/src/scripts/hooks/skill_route_hook.ts +11 -4
- package/src/scripts/inbox_source_census.ts +322 -0
- package/src/scripts/install-hooks.sh +134 -0
- package/src/scripts/{lint_carrier_integrity.ts → lint_deferral_integrity.ts} +60 -214
- package/src/scripts/lint_plan_risk_register.ts +13 -17
- package/src/scripts/lint_roadmap_blockers.ts +51 -1
- package/src/scripts/mcp_server/suggest_skill.ts +72 -0
- package/src/scripts/mcp_server/tools.ts +2 -56
- package/src/scripts/media/lib/adapter-contract.md +7 -2
- package/src/scripts/media/lib/fixtures/flux/asset-0001.png +0 -0
- package/src/scripts/media/lib/fixtures/gemini-image/asset-0001.png +0 -0
- package/src/scripts/media/lib/fixtures/ideogram/asset-0001.png +0 -0
- package/src/scripts/migrate_command_suggestions.ts +1 -1
- package/src/scripts/probe_unblocked_ask.ts +135 -2
- package/src/scripts/rdp_candidates_delta.ts +276 -0
- package/src/scripts/rdp_quality_eval.ts +217 -19
- package/src/scripts/release.ts +26 -71
- package/src/scripts/release_drill.ts +6 -10
- package/src/scripts/release_env.ts +1 -0
- package/src/scripts/release_mix_taxonomy.json +3 -2
- package/src/scripts/release_publication.ts +34 -29
- package/src/scripts/report_host_injection_effect.ts +189 -0
- package/src/scripts/report_layer_overlap.ts +41 -0
- package/src/scripts/report_skill_menu_census.ts +341 -0
- package/src/scripts/self_review_gate.ts +409 -15
- package/src/scripts/skill_tools/score_skill_relevance.ts +70 -6
- package/src/scripts/skill_tools/suggest_skill_for_task.ts +37 -3
- package/src/vendor/grammars/README.md +61 -0
- package/src/vendor/grammars/tree-sitter-javascript.wasm +0 -0
- package/src/vendor/grammars/tree-sitter-php.wasm +0 -0
- package/src/vendor/grammars/tree-sitter-typescript.wasm +0 -0
- package/src/scripts/_lib/carrier_status.ts +0 -118
- package/src/scripts/_lib/estate_carrier_cases.ts +0 -72
- package/src/scripts/hooks/code_graph_nudge_hook.ts +0 -191
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One binary content ceiling per seat — road-to-admissible-council-seats 2.2.
|
|
3
|
+
*
|
|
4
|
+
* Redaction today is a SECRETS floor and says so
|
|
5
|
+
* (`src/skills/ai-council/references/cost-and-redaction.md`). Nothing above it
|
|
6
|
+
* could express "this project's diff may not go to that seat", which is a
|
|
7
|
+
* different question with a different answer: a diff carrying no secret at all
|
|
8
|
+
* is still the consumer's private source.
|
|
9
|
+
*
|
|
10
|
+
* Deliberately BINARY. A graded scale invites a middle value nobody can define
|
|
11
|
+
* and every caller reads differently; two values force the decision at
|
|
12
|
+
* authoring time, where it belongs.
|
|
13
|
+
*
|
|
14
|
+
* public-artifact — repo-owned artefacts: skill descriptions, the trigger
|
|
15
|
+
* corpus, rule text, bench fixtures. Already public.
|
|
16
|
+
* project-content — everything a consumer supplies. The DEFAULT, so an
|
|
17
|
+
* unclassified payload is treated as private rather than
|
|
18
|
+
* as publishable.
|
|
19
|
+
*
|
|
20
|
+
* A seat's ceiling is the most sensitive class it may receive. `public-artifact`
|
|
21
|
+
* is therefore the NARROWER seat: it admits public artefacts only.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
export const CONTENT_CLASSES = ['public-artifact', 'project-content'] as const;
|
|
25
|
+
export type ContentClass = (typeof CONTENT_CLASSES)[number];
|
|
26
|
+
|
|
27
|
+
/** The default for anything not explicitly classified: private, not public. */
|
|
28
|
+
export const DEFAULT_CONTENT_CLASS: ContentClass = 'project-content';
|
|
29
|
+
|
|
30
|
+
/** The default seat ceiling: unchanged behaviour for every existing seat. */
|
|
31
|
+
export const DEFAULT_SEAT_CEILING: ContentClass = 'project-content';
|
|
32
|
+
|
|
33
|
+
const _SET: ReadonlySet<string> = new Set<string>(CONTENT_CLASSES);
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Any configured value → a content class. Unrecognised input becomes
|
|
37
|
+
* `project-content`, the RESTRICTIVE value: a typo must narrow what a seat may
|
|
38
|
+
* receive, never widen it.
|
|
39
|
+
*/
|
|
40
|
+
export function classifyContentClass(raw: unknown, fallback: ContentClass = DEFAULT_CONTENT_CLASS): ContentClass {
|
|
41
|
+
if (typeof raw === 'string' && _SET.has(raw)) {
|
|
42
|
+
return raw as ContentClass;
|
|
43
|
+
}
|
|
44
|
+
return fallback;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface CeilingVerdict {
|
|
48
|
+
readonly admitted: boolean;
|
|
49
|
+
/** Non-null exactly when `admitted` is false. */
|
|
50
|
+
readonly reason: string | null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Does a seat whose ceiling is `seatCeiling` admit a payload of `contentClass`?
|
|
55
|
+
*
|
|
56
|
+
* The one refusal is `project-content` into a `public-artifact` seat. Every
|
|
57
|
+
* other pairing is admitted, including public artefacts into a
|
|
58
|
+
* `project-content` seat — a seat cleared for private content is trivially
|
|
59
|
+
* cleared for public content, and refusing that would be a nonsense that
|
|
60
|
+
* disables the default seat for its own repo's fixtures.
|
|
61
|
+
*
|
|
62
|
+
* Refusal is a GREEN path (this roadmap's 2.2 verify): the caller drops the
|
|
63
|
+
* seat and the run continues. It is not an error, because a policy working as
|
|
64
|
+
* designed is not a failure.
|
|
65
|
+
*/
|
|
66
|
+
export function admitsContent(
|
|
67
|
+
seatCeiling: ContentClass,
|
|
68
|
+
contentClass: ContentClass,
|
|
69
|
+
): CeilingVerdict {
|
|
70
|
+
if (seatCeiling === 'public-artifact' && contentClass === 'project-content') {
|
|
71
|
+
return {
|
|
72
|
+
admitted: false,
|
|
73
|
+
reason:
|
|
74
|
+
"seat ceiling is 'public-artifact'; the payload is classified " +
|
|
75
|
+
"'project-content' (consumer-supplied)",
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
return { admitted: true, reason: null };
|
|
79
|
+
}
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two per-run deliberation measurements, both computed from outputs the run has
|
|
3
|
+
* already paid for: a per-correction stage attribution, and the
|
|
4
|
+
* `zero_marginal_value_call_rate`.
|
|
5
|
+
*
|
|
6
|
+
* Pure and offline. No model call, no new similarity measure — every judgement
|
|
7
|
+
* below reuses `is_near_duplicate` from `debate_gates.ts`, which is the
|
|
8
|
+
* repository's existing answer to "did this reply add anything", and the
|
|
9
|
+
* objection vocabulary that module already publishes. Forking a second
|
|
10
|
+
* similarity notion here would be two answers to one question, and the second
|
|
11
|
+
* is the one nobody updates.
|
|
12
|
+
*
|
|
13
|
+
* WHAT A CORRECTION IS, AND WHAT IT IS NOT.
|
|
14
|
+
*
|
|
15
|
+
* A correction is an objection-bearing sentence: one that carries a marker from
|
|
16
|
+
* {@link CORRECTION_MARKERS} and therefore names something it is pushing back
|
|
17
|
+
* on. Its FIRST STAGE is the earliest stage whose text contains that sentence
|
|
18
|
+
* or a near-duplicate of it — so a point raised in round 1 and repeated in
|
|
19
|
+
* round 3 is attributed to round 1, which is the whole question 10.2 asks.
|
|
20
|
+
*
|
|
21
|
+
* It decides ATTRIBUTION, never USEFULNESS. Whether a correction improved the
|
|
22
|
+
* verdict is a benchmark question and the benchmark has not run; a mechanism
|
|
23
|
+
* here that claimed to score usefulness would be inventing the number the
|
|
24
|
+
* forbidden-claims list exists to prevent. {@link attributeCorrections} answers
|
|
25
|
+
* "where did this first appear", and the word "useful" in the step it serves is
|
|
26
|
+
* a filter a human applies to the output, not one this code applies for them.
|
|
27
|
+
*
|
|
28
|
+
* WHAT ZERO MARGINAL VALUE IS.
|
|
29
|
+
*
|
|
30
|
+
* A call has zero marginal value when its text near-duplicates something the
|
|
31
|
+
* run had ALREADY received when the call was made. Not "duplicates the member's
|
|
32
|
+
* own prior round" — a member restating a peer's answer verbatim adds exactly
|
|
33
|
+
* as little, and the per-member reading would score it as novel. So the
|
|
34
|
+
* comparison set is every successful call that preceded it in run order.
|
|
35
|
+
*
|
|
36
|
+
* The first successful call is never zero-marginal: nothing preceded it. That
|
|
37
|
+
* makes the denominator at least 1 whenever the run produced any usable call,
|
|
38
|
+
* which is why the rate is a number rather than a `null` on every real run —
|
|
39
|
+
* and a run that produced NO usable call gets `null` with a stated reason,
|
|
40
|
+
* because zero calls is not a rate of zero.
|
|
41
|
+
*
|
|
42
|
+
* HONEST SCOPE. This measures textual novelty against an 0.8 Jaccard bar. It
|
|
43
|
+
* does not measure whether a call changed the verdict, and no consumer may read
|
|
44
|
+
* it as decision-quality evidence. The threshold is the shipped
|
|
45
|
+
* `NOVELTY_DUP_THRESHOLD`, inherited rather than tuned here; nothing has been
|
|
46
|
+
* benchmarked against it.
|
|
47
|
+
*/
|
|
48
|
+
import type { CouncilResponse } from './clients.js';
|
|
49
|
+
import { is_near_duplicate, NOVELTY_DUP_THRESHOLD } from './debate_gates.js';
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Objection markers. Deliberately the same vocabulary the shipped dissent
|
|
53
|
+
* counter uses, kept as its own constant because that one counts members and
|
|
54
|
+
* this one classifies sentences.
|
|
55
|
+
*/
|
|
56
|
+
export const CORRECTION_MARKERS =
|
|
57
|
+
/\b(disagree|disagrees|object|objection|reject|rejects|however|but|flaw|flawed|wrong|incorrect|counter|counter-position|dissent|contradict|contradicts|refute|refutes|mistaken|overstates|understates)\b/i;
|
|
58
|
+
|
|
59
|
+
/** One call the run made, in the order the run made it. */
|
|
60
|
+
export interface DeliberationCall {
|
|
61
|
+
/** Stage the call belongs to — a round label, a phase name, whatever the caller uses. */
|
|
62
|
+
readonly stage: string;
|
|
63
|
+
/** `provider/model`, or any stable member identity. */
|
|
64
|
+
readonly member: string;
|
|
65
|
+
/** The reply text. Empty or error replies are not scored. */
|
|
66
|
+
readonly text: string;
|
|
67
|
+
/** True when the transport reported a failure; such a call is not scored. */
|
|
68
|
+
readonly failed?: boolean;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Where one correction first entered the deliberation. */
|
|
72
|
+
export interface CorrectionAttribution {
|
|
73
|
+
/** The correction sentence, trimmed and whitespace-collapsed. */
|
|
74
|
+
readonly correction: string;
|
|
75
|
+
/** Stage label of the earliest call carrying it. */
|
|
76
|
+
readonly firstStage: string;
|
|
77
|
+
/** Zero-based index of that call in run order. */
|
|
78
|
+
readonly firstCallIndex: number;
|
|
79
|
+
/** Member whose call first carried it. */
|
|
80
|
+
readonly firstMember: string;
|
|
81
|
+
/** How many later calls restated it. 0 means it was said once. */
|
|
82
|
+
readonly restatements: number;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* One deliberation stage and what it produced — the record shape
|
|
87
|
+
* `CouncilRouteRecord` carries, filled from a real run rather than by a test.
|
|
88
|
+
*/
|
|
89
|
+
export interface StageProduction {
|
|
90
|
+
readonly stage: string;
|
|
91
|
+
readonly produced: number;
|
|
92
|
+
readonly calls: number;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface ZeroMarginalValueReading {
|
|
96
|
+
/**
|
|
97
|
+
* Zero-marginal calls over scored calls, or `null` when the run produced no
|
|
98
|
+
* scorable call at all. `null` is "not measurable here", never zero.
|
|
99
|
+
*/
|
|
100
|
+
readonly rate: number | null;
|
|
101
|
+
/** Why the rate is `null`. Empty string when it is not. */
|
|
102
|
+
readonly unavailableReason: string;
|
|
103
|
+
/** Every call the caller passed, failed and empty ones included. */
|
|
104
|
+
readonly callsTotal: number;
|
|
105
|
+
/** Calls that carried usable text — the denominator. */
|
|
106
|
+
readonly callsScored: number;
|
|
107
|
+
/** Calls that near-duplicated something already received — the numerator. */
|
|
108
|
+
readonly callsZeroMarginal: number;
|
|
109
|
+
/** The Jaccard bar used, published so the number can be re-derived. */
|
|
110
|
+
readonly threshold: number;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Decimals the published rate is rounded to, so a JSON round-trip is exact. */
|
|
114
|
+
export const RATE_PRECISION = 6;
|
|
115
|
+
|
|
116
|
+
function _round(value: number): number {
|
|
117
|
+
const f = 10 ** RATE_PRECISION;
|
|
118
|
+
return Math.round(value * f) / f;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function _usable(c: DeliberationCall): boolean {
|
|
122
|
+
return c.failed !== true && c.text.trim().length > 0;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Split a reply into sentences. Crude on purpose: no NLP, no dependency. */
|
|
126
|
+
function _sentences(text: string): string[] {
|
|
127
|
+
return text
|
|
128
|
+
.split(/(?<=[.!?])\s+|\n+/)
|
|
129
|
+
.map((s) => s.replace(/\s+/g, ' ').trim())
|
|
130
|
+
.filter((s) => s.length > 0);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Correction sentences in one reply.
|
|
135
|
+
*
|
|
136
|
+
* A minimum length is applied because a bare "But." carries a marker and no
|
|
137
|
+
* position; the bar is characters rather than a judgement so it stays
|
|
138
|
+
* predictable.
|
|
139
|
+
*/
|
|
140
|
+
export const MIN_CORRECTION_CHARS = 25;
|
|
141
|
+
|
|
142
|
+
export function correctionSentences(text: string): string[] {
|
|
143
|
+
return _sentences(text).filter(
|
|
144
|
+
(s) => s.length >= MIN_CORRECTION_CHARS && CORRECTION_MARKERS.test(s),
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Attribute every correction to the earliest call that carried it.
|
|
150
|
+
*
|
|
151
|
+
* Calls are read in the order given, which must be run order — the attribution
|
|
152
|
+
* is only meaningful if "earlier in the list" means "earlier in the run".
|
|
153
|
+
*/
|
|
154
|
+
export function attributeCorrections(
|
|
155
|
+
calls: readonly DeliberationCall[],
|
|
156
|
+
threshold: number = NOVELTY_DUP_THRESHOLD,
|
|
157
|
+
): CorrectionAttribution[] {
|
|
158
|
+
const out: CorrectionAttribution[] = [];
|
|
159
|
+
const restatements: number[] = [];
|
|
160
|
+
|
|
161
|
+
calls.forEach((call, idx) => {
|
|
162
|
+
if (!_usable(call)) return;
|
|
163
|
+
for (const sentence of correctionSentences(call.text)) {
|
|
164
|
+
const seen = out.findIndex(
|
|
165
|
+
(a) =>
|
|
166
|
+
a.correction === sentence || is_near_duplicate(a.correction, sentence, threshold),
|
|
167
|
+
);
|
|
168
|
+
if (seen !== -1) {
|
|
169
|
+
restatements[seen] = (restatements[seen] ?? 0) + 1;
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
restatements.push(0);
|
|
173
|
+
out.push({
|
|
174
|
+
correction: sentence,
|
|
175
|
+
firstStage: call.stage,
|
|
176
|
+
firstCallIndex: idx,
|
|
177
|
+
firstMember: call.member,
|
|
178
|
+
restatements: 0,
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
return out.map((a, i) => ({ ...a, restatements: restatements[i] ?? 0 }));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Per-stage production, derived from the attribution so the two can never
|
|
188
|
+
* disagree: `produced` counts the corrections FIRST attributed to that stage.
|
|
189
|
+
*/
|
|
190
|
+
export function stageProduction(
|
|
191
|
+
calls: readonly DeliberationCall[],
|
|
192
|
+
attributions: readonly CorrectionAttribution[],
|
|
193
|
+
): StageProduction[] {
|
|
194
|
+
const order: string[] = [];
|
|
195
|
+
const callCount = new Map<string, number>();
|
|
196
|
+
for (const c of calls) {
|
|
197
|
+
if (!callCount.has(c.stage)) {
|
|
198
|
+
order.push(c.stage);
|
|
199
|
+
callCount.set(c.stage, 0);
|
|
200
|
+
}
|
|
201
|
+
callCount.set(c.stage, (callCount.get(c.stage) ?? 0) + 1);
|
|
202
|
+
}
|
|
203
|
+
const produced = new Map<string, number>();
|
|
204
|
+
for (const a of attributions) {
|
|
205
|
+
produced.set(a.firstStage, (produced.get(a.firstStage) ?? 0) + 1);
|
|
206
|
+
}
|
|
207
|
+
return order.map((stage) => ({
|
|
208
|
+
stage,
|
|
209
|
+
produced: produced.get(stage) ?? 0,
|
|
210
|
+
calls: callCount.get(stage) ?? 0,
|
|
211
|
+
}));
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The `zero_marginal_value_call_rate` for one run.
|
|
216
|
+
*
|
|
217
|
+
* Every scored call is compared against every scored call that preceded it. The
|
|
218
|
+
* comparison is O(n squared) in the number of calls, which is fine: a council
|
|
219
|
+
* run is single digits of calls per round.
|
|
220
|
+
*/
|
|
221
|
+
export function zeroMarginalValueRate(
|
|
222
|
+
calls: readonly DeliberationCall[],
|
|
223
|
+
threshold: number = NOVELTY_DUP_THRESHOLD,
|
|
224
|
+
): ZeroMarginalValueReading {
|
|
225
|
+
const received: string[] = [];
|
|
226
|
+
let zero = 0;
|
|
227
|
+
|
|
228
|
+
for (const call of calls) {
|
|
229
|
+
if (!_usable(call)) continue;
|
|
230
|
+
const dup = received.some((prior) => is_near_duplicate(prior, call.text, threshold));
|
|
231
|
+
if (dup) zero += 1;
|
|
232
|
+
received.push(call.text);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
if (received.length === 0) {
|
|
236
|
+
return {
|
|
237
|
+
rate: null,
|
|
238
|
+
unavailableReason:
|
|
239
|
+
'no scorable call — every call failed or returned empty text, and zero calls is not a rate of zero',
|
|
240
|
+
callsTotal: calls.length,
|
|
241
|
+
callsScored: 0,
|
|
242
|
+
callsZeroMarginal: 0,
|
|
243
|
+
threshold,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
return {
|
|
248
|
+
rate: _round(zero / received.length),
|
|
249
|
+
unavailableReason: '',
|
|
250
|
+
callsTotal: calls.length,
|
|
251
|
+
callsZeroMarginal: zero,
|
|
252
|
+
callsScored: received.length,
|
|
253
|
+
threshold,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** Everything this module emits for one run, ready to serialise. */
|
|
258
|
+
export interface DeliberationMetrics {
|
|
259
|
+
readonly zero_marginal_value_call_rate: number | null;
|
|
260
|
+
readonly zero_marginal_value_unavailable_reason: string;
|
|
261
|
+
readonly calls_total: number;
|
|
262
|
+
readonly calls_scored: number;
|
|
263
|
+
readonly calls_zero_marginal: number;
|
|
264
|
+
readonly novelty_threshold: number;
|
|
265
|
+
readonly stage_outputs: readonly StageProduction[];
|
|
266
|
+
readonly correction_attributions: readonly CorrectionAttribution[];
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Compute both measurements in one pass over the run's calls. */
|
|
270
|
+
export function measureDeliberation(
|
|
271
|
+
calls: readonly DeliberationCall[],
|
|
272
|
+
threshold: number = NOVELTY_DUP_THRESHOLD,
|
|
273
|
+
): DeliberationMetrics {
|
|
274
|
+
const attributions = attributeCorrections(calls, threshold);
|
|
275
|
+
const zmv = zeroMarginalValueRate(calls, threshold);
|
|
276
|
+
return {
|
|
277
|
+
zero_marginal_value_call_rate: zmv.rate,
|
|
278
|
+
zero_marginal_value_unavailable_reason: zmv.unavailableReason,
|
|
279
|
+
calls_total: zmv.callsTotal,
|
|
280
|
+
calls_scored: zmv.callsScored,
|
|
281
|
+
calls_zero_marginal: zmv.callsZeroMarginal,
|
|
282
|
+
novelty_threshold: zmv.threshold,
|
|
283
|
+
stage_outputs: stageProduction(calls, attributions),
|
|
284
|
+
correction_attributions: attributions,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* The parts of one council run that made provider calls, in the order the run
|
|
290
|
+
* made them.
|
|
291
|
+
*
|
|
292
|
+
* Assembled here rather than at the call site because "which calls did this run
|
|
293
|
+
* make, and at which stage" is this module's own subject, and the CLI had the
|
|
294
|
+
* list only as a flat array with the stage boundaries implicit in the order.
|
|
295
|
+
* Both readings come out of one walk, so the billing total and the attribution
|
|
296
|
+
* can never be computed over different sets.
|
|
297
|
+
*/
|
|
298
|
+
export interface RunCallParts {
|
|
299
|
+
/** The deliberation responses `consult()` returned. */
|
|
300
|
+
readonly deliberation: readonly CouncilResponse[];
|
|
301
|
+
readonly peerReview?: readonly CouncilResponse[] | null;
|
|
302
|
+
readonly consensusExtraction?: readonly CouncilResponse[] | null;
|
|
303
|
+
readonly consensusScoring?: readonly CouncilResponse[] | null;
|
|
304
|
+
readonly chairman?: CouncilResponse | null;
|
|
305
|
+
readonly stanceRepairs?: readonly CouncilResponse[] | null;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** The six stage labels, in run order. */
|
|
309
|
+
export const RUN_STAGES = Object.freeze([
|
|
310
|
+
'deliberation',
|
|
311
|
+
'peer-review',
|
|
312
|
+
'consensus-extraction',
|
|
313
|
+
'consensus-scoring',
|
|
314
|
+
'chairman',
|
|
315
|
+
'stance-repair',
|
|
316
|
+
] as const);
|
|
317
|
+
|
|
318
|
+
export interface RunCalls {
|
|
319
|
+
/** Stage-labelled calls, for {@link measureDeliberation}. */
|
|
320
|
+
readonly calls: readonly DeliberationCall[];
|
|
321
|
+
/** The same calls as raw responses, in the same order, for cost aggregation. */
|
|
322
|
+
readonly responses: CouncilResponse[];
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function _asCall(stage: string, r: CouncilResponse): DeliberationCall {
|
|
326
|
+
return {
|
|
327
|
+
stage,
|
|
328
|
+
member: `${r.provider}/${r.model}`,
|
|
329
|
+
text: r.text,
|
|
330
|
+
failed: r.error !== null && r.error !== undefined,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Walk one run's parts into stage-labelled calls plus the flat response list. */
|
|
335
|
+
export function runCallsFrom(parts: RunCallParts): RunCalls {
|
|
336
|
+
const groups: ReadonlyArray<readonly [string, readonly CouncilResponse[]]> = [
|
|
337
|
+
['deliberation', parts.deliberation],
|
|
338
|
+
['peer-review', parts.peerReview ?? []],
|
|
339
|
+
['consensus-extraction', parts.consensusExtraction ?? []],
|
|
340
|
+
['consensus-scoring', parts.consensusScoring ?? []],
|
|
341
|
+
['chairman', parts.chairman === null || parts.chairman === undefined ? [] : [parts.chairman]],
|
|
342
|
+
['stance-repair', parts.stanceRepairs ?? []],
|
|
343
|
+
];
|
|
344
|
+
const calls: DeliberationCall[] = [];
|
|
345
|
+
const responses: CouncilResponse[] = [];
|
|
346
|
+
for (const [stage, group] of groups) {
|
|
347
|
+
for (const r of group) {
|
|
348
|
+
calls.push(_asCall(stage, r));
|
|
349
|
+
responses.push(r);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return { calls, responses };
|
|
353
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jury aggregation — road-to-admissible-council-seats 3.2.
|
|
3
|
+
*
|
|
4
|
+
* BESIDE `consensus.ts`, not inside it, and the separation is the design.
|
|
5
|
+
* `consensus.ts` aggregates a COUNCIL, whose members carry verdicts.
|
|
6
|
+
* A jury does not: per ADR-257 a route this package did not pay for may propose
|
|
7
|
+
* and may score, never decide. Folding jury scoring into the consensus path
|
|
8
|
+
* would put a scorer on the code path that produces verdicts, which is the
|
|
9
|
+
* exact transition this roadmap's Risk 5 names — "scoring quietly becomes
|
|
10
|
+
* deciding".
|
|
11
|
+
*
|
|
12
|
+
* Two rules, and both are refusals.
|
|
13
|
+
*
|
|
14
|
+
* 1. **At least two model families, or the panel is ABSENT.** Not a warning, a
|
|
15
|
+
* refusal: `docs/CLAIMS.md` records that a same-posture second vendor's
|
|
16
|
+
* catches were a strict SUBSET of the first's. A same-family panel therefore
|
|
17
|
+
* measures redundancy, and reporting redundancy as agreement is the failure
|
|
18
|
+
* a diversity requirement exists to prevent.
|
|
19
|
+
* 2. **The aggregate is a trimmed statistic, never a vote count.** For the same
|
|
20
|
+
* reason: counting agreeing voices rewards a panel for being correlated.
|
|
21
|
+
*
|
|
22
|
+
* What "trimmed" means here, exactly: with n >= 3 the single lowest and single highest score are dropped and the
|
|
23
|
+
* rest averaged — which at n = 3 IS the median, and is why one corrupted score
|
|
24
|
+
* in a three-judge panel cannot move the result the way it moves an arithmetic
|
|
25
|
+
* mean. With n = 2 there is nothing to trim; the mean is returned with
|
|
26
|
+
* `trimmed: 0` so a reader can see that no robustness was available rather than
|
|
27
|
+
* inferring it from a method name.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import {
|
|
31
|
+
judgeBothOrders,
|
|
32
|
+
positionConsistency,
|
|
33
|
+
type JudgeConsistency,
|
|
34
|
+
type PairResolution,
|
|
35
|
+
type PairwiseJudge,
|
|
36
|
+
type SwapObservation,
|
|
37
|
+
} from './judge_position_bias.js';
|
|
38
|
+
|
|
39
|
+
/** One judge's score for one item. `family` is the MODEL family, not the seat. */
|
|
40
|
+
export interface JuryScore {
|
|
41
|
+
readonly judge: string;
|
|
42
|
+
readonly family: string;
|
|
43
|
+
/** Any finite number. The aggregator does not impose a scale. */
|
|
44
|
+
readonly score: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Why a panel produced no aggregate. Each value is a REFUSAL, not an error. */
|
|
48
|
+
export type JuryAbsentReason = 'no_scores' | 'single_family' | 'non_finite_score';
|
|
49
|
+
|
|
50
|
+
export type JuryResult =
|
|
51
|
+
| {
|
|
52
|
+
readonly present: false;
|
|
53
|
+
readonly absentReason: JuryAbsentReason;
|
|
54
|
+
readonly reason: string;
|
|
55
|
+
}
|
|
56
|
+
| {
|
|
57
|
+
readonly present: true;
|
|
58
|
+
readonly aggregate: number;
|
|
59
|
+
readonly method: 'trimmed-mean';
|
|
60
|
+
/** How many scores the trim discarded. 0 when n < 3. */
|
|
61
|
+
readonly trimmed: number;
|
|
62
|
+
readonly judges: number;
|
|
63
|
+
readonly families: readonly string[];
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Aggregate one item's jury scores.
|
|
68
|
+
*
|
|
69
|
+
* Never throws on panel composition: a refused panel is a VERDICT the caller
|
|
70
|
+
* reads, the same shape the transport resolver uses for a refused seat. An
|
|
71
|
+
* exception would make "this panel does not qualify" indistinguishable from a
|
|
72
|
+
* bug at every call site.
|
|
73
|
+
*/
|
|
74
|
+
export function juryAggregate(scores: readonly JuryScore[]): JuryResult {
|
|
75
|
+
if (scores.length === 0) {
|
|
76
|
+
return { present: false, absentReason: 'no_scores', reason: 'no jury scores supplied' };
|
|
77
|
+
}
|
|
78
|
+
for (const s of scores) {
|
|
79
|
+
if (!Number.isFinite(s.score)) {
|
|
80
|
+
return {
|
|
81
|
+
present: false,
|
|
82
|
+
absentReason: 'non_finite_score',
|
|
83
|
+
reason: `judge ${s.judge} returned a non-finite score; the panel is refused rather than silently dropping it`,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const families = [...new Set(scores.map((s) => s.family))].sort();
|
|
88
|
+
if (families.length < 2) {
|
|
89
|
+
return {
|
|
90
|
+
present: false,
|
|
91
|
+
absentReason: 'single_family',
|
|
92
|
+
reason:
|
|
93
|
+
`panel spans ${families.length} model family (${families.join(', ') || 'none'}); ` +
|
|
94
|
+
'at least two are required — a same-family panel measures redundancy, not agreement',
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
const sorted = [...scores.map((s) => s.score)].sort((a, b) => a - b);
|
|
98
|
+
const trimmed = sorted.length >= 3 ? 2 : 0;
|
|
99
|
+
const kept = trimmed === 0 ? sorted : sorted.slice(1, sorted.length - 1);
|
|
100
|
+
const aggregate = kept.reduce((acc, v) => acc + v, 0) / kept.length;
|
|
101
|
+
return {
|
|
102
|
+
present: true,
|
|
103
|
+
aggregate,
|
|
104
|
+
method: 'trimmed-mean',
|
|
105
|
+
trimmed,
|
|
106
|
+
judges: scores.length,
|
|
107
|
+
families,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** One judge's swapped observation plus the family it belongs to. */
|
|
112
|
+
export interface JuryPairwiseJudge {
|
|
113
|
+
readonly judge: string;
|
|
114
|
+
readonly family: string;
|
|
115
|
+
readonly fn: PairwiseJudge;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export interface JuryPairwiseResult {
|
|
119
|
+
readonly present: boolean;
|
|
120
|
+
readonly absentReason: JuryAbsentReason | null;
|
|
121
|
+
readonly reason: string | null;
|
|
122
|
+
/** Panel resolution, or `null` when the panel is absent. */
|
|
123
|
+
readonly resolution: PairResolution | null;
|
|
124
|
+
readonly observations: readonly SwapObservation[];
|
|
125
|
+
readonly consistency: readonly JudgeConsistency[];
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Run a pairwise jury with the position swap applied PER JUDGE.
|
|
130
|
+
*
|
|
131
|
+
* Reuses `judge_position_bias.ts` rather than re-implementing the swap: a
|
|
132
|
+
* second implementation of "show both orders and reconcile" is a second place
|
|
133
|
+
* for the reconciliation to be subtly wrong, and that module already resolves a
|
|
134
|
+
* verdict that does not survive the swap to `inconsistent` rather than to a
|
|
135
|
+
* winner.
|
|
136
|
+
*
|
|
137
|
+
* The panel resolution is the strict majority of the judges' own resolutions,
|
|
138
|
+
* and `inconsistent` judges are counted in the DENOMINATOR — a judge that
|
|
139
|
+
* flipped has an opinion about nothing, and dropping it from the denominator
|
|
140
|
+
* would let two stable judges out of five look unanimous.
|
|
141
|
+
*/
|
|
142
|
+
export function juryPairwise(
|
|
143
|
+
id: string,
|
|
144
|
+
a: string,
|
|
145
|
+
b: string,
|
|
146
|
+
judges: readonly JuryPairwiseJudge[],
|
|
147
|
+
): JuryPairwiseResult {
|
|
148
|
+
const families = [...new Set(judges.map((j) => j.family))].sort();
|
|
149
|
+
if (judges.length === 0 || families.length < 2) {
|
|
150
|
+
return {
|
|
151
|
+
present: false,
|
|
152
|
+
absentReason: judges.length === 0 ? 'no_scores' : 'single_family',
|
|
153
|
+
reason:
|
|
154
|
+
judges.length === 0
|
|
155
|
+
? 'no judges supplied'
|
|
156
|
+
: `panel spans ${families.length} model family; at least two are required`,
|
|
157
|
+
resolution: null,
|
|
158
|
+
observations: [],
|
|
159
|
+
consistency: [],
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
const observations: SwapObservation[] = [];
|
|
163
|
+
const consistency: JudgeConsistency[] = [];
|
|
164
|
+
const tally = new Map<PairResolution, number>();
|
|
165
|
+
for (const j of judges) {
|
|
166
|
+
const obs = judgeBothOrders(id, a, b, j.fn);
|
|
167
|
+
observations.push(obs);
|
|
168
|
+
consistency.push(positionConsistency(j.judge, [obs]));
|
|
169
|
+
tally.set(obs.resolution, (tally.get(obs.resolution) ?? 0) + 1);
|
|
170
|
+
}
|
|
171
|
+
let resolution: PairResolution = 'inconsistent';
|
|
172
|
+
for (const [k, v] of tally) {
|
|
173
|
+
if (v * 2 > judges.length) {
|
|
174
|
+
resolution = k;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return { present: true, absentReason: null, reason: null, resolution, observations, consistency };
|
|
178
|
+
}
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
import { wrapUntrustedBlocks } from '../_lib/untrusted_content.js';
|
|
26
26
|
|
|
27
27
|
import type { ProjectContext } from './project_context.js';
|
|
28
|
+
import { RESOLVING_EVIDENCE_SECTION } from './synthesis_disagreement.js';
|
|
28
29
|
|
|
29
30
|
// Python: NEUTRALITY_PREAMBLE = """\...""".strip()
|
|
30
31
|
export const NEUTRALITY_PREAMBLE = `You are an independent reviewer. You have NOT seen any prior reasoning,
|
|
@@ -302,6 +303,8 @@ read it directly from a response.
|
|
|
302
303
|
A single sentence: which course the host agent should advise the
|
|
303
304
|
user to take, grounded in the strongest converged point.
|
|
304
305
|
|
|
306
|
+
${RESOLVING_EVIDENCE_SECTION}
|
|
307
|
+
|
|
305
308
|
### Kill criteria
|
|
306
309
|
Observable conditions that would falsify this recommendation without
|
|
307
310
|
re-convening the council. Each entry names a threshold or a specific
|
|
@@ -329,6 +332,8 @@ and the host agent confirms are load-bearing. Maximum five.
|
|
|
329
332
|
APPROVE / REQUEST_CHANGES / REJECT and a single sentence justifying
|
|
330
333
|
the verdict, anchored on the strongest consensus or must-fix line.
|
|
331
334
|
|
|
335
|
+
${RESOLVING_EVIDENCE_SECTION}
|
|
336
|
+
|
|
332
337
|
### Kill criteria
|
|
333
338
|
Observable conditions that would falsify this verdict without re-convening
|
|
334
339
|
the council (a failing check, a missing test, a regression signal). Each
|
|
@@ -363,6 +368,8 @@ separate certainty word for that; \`evidence:\` measures how well a
|
|
|
363
368
|
finding is supported, and being alone in raising it is a different
|
|
364
369
|
property the section already carries.
|
|
365
370
|
|
|
371
|
+
${RESOLVING_EVIDENCE_SECTION}
|
|
372
|
+
|
|
366
373
|
### Kill criteria
|
|
367
374
|
Observable conditions that would falsify the analysis's leading finding
|
|
368
375
|
without re-convening the council. Each entry names a threshold or event.
|
|
@@ -381,6 +388,8 @@ export const CREATIVE_SYNTHESIS = `Write a free-form convergence / divergence
|
|
|
381
388
|
synthesis of the council in prose — no fixed section shape for the body.
|
|
382
389
|
Then close with exactly these two sections:
|
|
383
390
|
|
|
391
|
+
${RESOLVING_EVIDENCE_SECTION}
|
|
392
|
+
|
|
384
393
|
### Kill criteria
|
|
385
394
|
Observable conditions that would falsify the direction you converged on,
|
|
386
395
|
without re-convening the council. Each entry names a threshold or event.
|