2sagaco48 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/diagnostics.mjs +141 -0
- package/dist/compute.worker-j5ahIpF-.js +1 -0
- package/dist/graph.worker-DG6iGCB9.js +1 -0
- package/dist/index.html +2314 -0
- package/flatten-guide.md +248 -0
- package/index.html +12 -0
- package/package.json +38 -0
- package/src/App.orig.tsx +204 -0
- package/src/App.orig1.tsx +27 -0
- package/src/App.orig2.tsx +25 -0
- package/src/App.tsx +3 -0
- package/src/BaseApp.tsx +18 -0
- package/src/INTEGRATION_LEDGER.md +22 -0
- package/src/PERSIST_CANARY.orig.txt +1 -0
- package/src/PERSIST_CANARY.txt +1 -0
- package/src/REGRESSION_LEDGER.md +651 -0
- package/src/TEMPLATE_PIPELINE_AUDIT.md +208 -0
- package/src/WILLIAMS_PERSONA_GUIDE.md +62 -0
- package/src/chat/VeritasChat.tsx +69 -0
- package/src/chat/synthesis.ts +31 -0
- package/src/chat/tier.ts +45 -0
- package/src/components/AdaptersPage.orig.tsx +99 -0
- package/src/components/AdaptersPage.tsx +3 -0
- package/src/components/AdversarialPanel.tsx +136 -0
- package/src/components/AnchorBaselinePanel.tsx +75 -0
- package/src/components/AnswerPanel.tsx +60 -0
- package/src/components/ArtifactPanel.tsx +43 -0
- package/src/components/ChatApp.orig.tsx +1582 -0
- package/src/components/ChatApp.orig1.tsx +722 -0
- package/src/components/ChatApp.tsx +95 -0
- package/src/components/ChatAugmentPanels.tsx +73 -0
- package/src/components/CitationLedgerPanel.tsx +313 -0
- package/src/components/ComputeSandboxPanel.tsx +130 -0
- package/src/components/ConfigPanel.tsx +110 -0
- package/src/components/ControlPlanePage.tsx +195 -0
- package/src/components/CreativeTreeLifePage.tsx +234 -0
- package/src/components/CreativeTreeOfLifePanel.tsx +149 -0
- package/src/components/DebugTracePanel.tsx +137 -0
- package/src/components/DeepReasoningTrace.tsx +48 -0
- package/src/components/EntitySheetPanel.tsx +51 -0
- package/src/components/FailureModesPanel.tsx +104 -0
- package/src/components/GBSDashboard.tsx +239 -0
- package/src/components/GraphView.tsx +116 -0
- package/src/components/HUD.tsx +125 -0
- package/src/components/HypothesisPanel.tsx +67 -0
- package/src/components/InnovationGenomeEngine.tsx +792 -0
- package/src/components/InnovationPersonaGuide.tsx +348 -0
- package/src/components/InnovationPersonaPanel.tsx +289 -0
- package/src/components/LiveResourceHUD.tsx +109 -0
- package/src/components/LongWriterPanel.tsx +112 -0
- package/src/components/MainPipelineV10Bridge.tsx +72 -0
- package/src/components/MarkdownLite.tsx +63 -0
- package/src/components/MemoryInspector.tsx +38 -0
- package/src/components/MemoryMonitor.tsx +52 -0
- package/src/components/MemoryStressPanel.tsx +176 -0
- package/src/components/ModulesPage.tsx +171 -0
- package/src/components/PipelineDebugConsole.orig.tsx +1856 -0
- package/src/components/PipelineDebugConsole.tsx +3 -0
- package/src/components/PreFlightHUD.tsx +67 -0
- package/src/components/PrismaFetchTracePanel.tsx +87 -0
- package/src/components/ReportOSPanel.tsx +165 -0
- package/src/components/ResourceEstimatorPage.tsx +327 -0
- package/src/components/RichText.tsx +181 -0
- package/src/components/SagaOsPanel.tsx +605 -0
- package/src/components/SagaOsReference.tsx +421 -0
- package/src/components/SagaOsV2Panel.tsx +594 -0
- package/src/components/SagaStyleTab.tsx +382 -0
- package/src/components/SageBlueprintPanel.tsx +416 -0
- package/src/components/SharedChatInput.tsx +77 -0
- package/src/components/StatFinancePanel.tsx +190 -0
- package/src/components/StylePersonaPanel.tsx +128 -0
- package/src/components/TemplatesPage.tsx +300 -0
- package/src/components/TestPanel.tsx +63 -0
- package/src/components/TraceLog.tsx +89 -0
- package/src/components/V15CalibrationAugment.tsx +1134 -0
- package/src/components/V15CalibrationDialog.tsx +1871 -0
- package/src/components/V15Overlay.orig.tsx +120 -0
- package/src/components/V15Overlay.tsx +4 -0
- package/src/components/V15OverlayWrapper.tsx +362 -0
- package/src/components/V15Toggle.tsx +34 -0
- package/src/console/DiagnosticRecoveryPanel.tsx +358 -0
- package/src/console/EngineeringConsole.tsx +103 -0
- package/src/console/FictionSystemPanels.tsx +882 -0
- package/src/console/InspectorPanels.tsx +254 -0
- package/src/console/ManuscriptGraderPanel.tsx +182 -0
- package/src/console/RecursiveReviserPanel.tsx +245 -0
- package/src/console/SeedControlPanel.tsx +135 -0
- package/src/console/StoryControllerPanel.tsx +443 -0
- package/src/console/engineBridge.ts +258 -0
- package/src/console/gemini.ts +106 -0
- package/src/console/grading.ts +250 -0
- package/src/console/util.ts +45 -0
- package/src/fiction/continuity.ts +372 -0
- package/src/fiction/crash-protection.ts +111 -0
- package/src/fiction/diagnostics.ts +208 -0
- package/src/fiction/diff.ts +74 -0
- package/src/fiction/entity-state-machine.ts +214 -0
- package/src/fiction/nive-engine.ts +404 -0
- package/src/fiction/pipeline-selftest.ts +132 -0
- package/src/fiction/seed-control.ts +193 -0
- package/src/fiction/story-engine.ts +326 -0
- package/src/fiction/types.ts +68 -0
- package/src/flatten-guide.md +379 -0
- package/src/index.css +14 -0
- package/src/index.orig.css +13 -0
- package/src/lib/academic-sources.base.ts +188 -0
- package/src/lib/academic-sources.orig.ts +28 -0
- package/src/lib/academic-sources.ts +10 -0
- package/src/lib/advanced-math.ts +119 -0
- package/src/lib/adversarial-engine.base.ts +409 -0
- package/src/lib/adversarial-engine.ts +82 -0
- package/src/lib/ai-revision.ts +623 -0
- package/src/lib/app-state.tsx +426 -0
- package/src/lib/artifact-registry.ts +97 -0
- package/src/lib/artifacts.ts +180 -0
- package/src/lib/atlas-dr.ts +167 -0
- package/src/lib/browser-mtls.ts +127 -0
- package/src/lib/browser-queue.ts +221 -0
- package/src/lib/browser-search-scraper.base.ts +382 -0
- package/src/lib/browser-search-scraper.orig.ts +90 -0
- package/src/lib/browser-search-scraper.ts +10 -0
- package/src/lib/calc-interceptor.ts +82 -0
- package/src/lib/citation-lane-tap.ts +78 -0
- package/src/lib/citation-ledger-store.ts +369 -0
- package/src/lib/citation-ledger.ts +203 -0
- package/src/lib/compute-sandbox.ts +215 -0
- package/src/lib/compute.worker.ts +153 -0
- package/src/lib/connectors/gemini.ts +120 -0
- package/src/lib/connectors/jina.ts +233 -0
- package/src/lib/connectors/marketdata.ts +51 -0
- package/src/lib/connectors/prismafetch.base.ts +175 -0
- package/src/lib/connectors/prismafetch.ts +1 -0
- package/src/lib/connectors/serpapi.ts +65 -0
- package/src/lib/connectors/wikidata.ts +44 -0
- package/src/lib/constraints.ts +331 -0
- package/src/lib/continuation-detector.base.ts +153 -0
- package/src/lib/continuation-detector.ts +48 -0
- package/src/lib/contradraft.ts +168 -0
- package/src/lib/cors-proxy.ts +120 -0
- package/src/lib/coverage.ts +81 -0
- package/src/lib/debug/architecture-prescription.ts +556 -0
- package/src/lib/debug/covea-repair.ts +543 -0
- package/src/lib/debug/helios-ground.ts +319 -0
- package/src/lib/debug/intent-decomposer.ts +281 -0
- package/src/lib/debug/intent-lattice.ts +349 -0
- package/src/lib/debug/pipeline-diagnosis.ts +1149 -0
- package/src/lib/debug/pipeline-trace-bus.ts +424 -0
- package/src/lib/debug/prompt-forge.ts +678 -0
- package/src/lib/debug/repair-sites.ts +513 -0
- package/src/lib/debug/research-phase.ts +266 -0
- package/src/lib/debug/retrieval-context.ts +76 -0
- package/src/lib/debug/retrieval-hardener.ts +142 -0
- package/src/lib/debug/scraper-debug-runner.ts +576 -0
- package/src/lib/debug/scraper-forensics.ts +418 -0
- package/src/lib/debug/scraper-lane-roadmap.ts +136 -0
- package/src/lib/debug/self-test.ts +415 -0
- package/src/lib/debug/step-attribution.ts +211 -0
- package/src/lib/debug/template-rubric.ts +301 -0
- package/src/lib/debug/unified-innovation.ts +126 -0
- package/src/lib/debug/veritas-hybrid-scraper.ts +335 -0
- package/src/lib/defense-registry.ts +104 -0
- package/src/lib/defenses.ts +162 -0
- package/src/lib/deterministic-citation-ledger.ts +174 -0
- package/src/lib/elo-registry.base.ts +117 -0
- package/src/lib/elo-registry.ts +1 -0
- package/src/lib/entity-resolver.ts +291 -0
- package/src/lib/failure-modes.ts +267 -0
- package/src/lib/feature-registry.ts +69 -0
- package/src/lib/flaw-registry.ts +143 -0
- package/src/lib/flaws/_template.ts +52 -0
- package/src/lib/flaws/builtins.ts +63 -0
- package/src/lib/flaws/finance.ts +169 -0
- package/src/lib/flaws/fixers.ts +10 -0
- package/src/lib/flaws/index.ts +32 -0
- package/src/lib/flaws/legal.ts +157 -0
- package/src/lib/flaws/medical.ts +188 -0
- package/src/lib/flaws/original-defenses-pack.ts +81 -0
- package/src/lib/flaws/sample-declarative-pack.json +38 -0
- package/src/lib/flaws/selftest.ts +201 -0
- package/src/lib/flaws/software-extended.ts +125 -0
- package/src/lib/flaws/software-rn-webgl.ts +182 -0
- package/src/lib/flaws/statistics-advanced.ts +46 -0
- package/src/lib/flaws/statistics.ts +61 -0
- package/src/lib/gbse/config.ts +20 -0
- package/src/lib/gbse/engine.ts +226 -0
- package/src/lib/gbse/graph.ts +118 -0
- package/src/lib/gbse/graph.worker.ts +21 -0
- package/src/lib/gbse/tests.ts +117 -0
- package/src/lib/gbse/tiers.ts +65 -0
- package/src/lib/gbse/types.ts +94 -0
- package/src/lib/innovation-genome-engine-v2.orig.ts +1001 -0
- package/src/lib/innovation-genome-engine-v2.ts +3 -0
- package/src/lib/innovation-genome-engine.orig.ts +330 -0
- package/src/lib/innovation-genome-engine.ts +3 -0
- package/src/lib/innovation-genome-v10.orig.ts +314 -0
- package/src/lib/innovation-genome-v10.ts +2 -0
- package/src/lib/innovation-genome-v3.orig.ts +2015 -0
- package/src/lib/innovation-genome-v3.ts +2 -0
- package/src/lib/innovation-genome-v4.orig.ts +1959 -0
- package/src/lib/innovation-genome-v4.ts +2 -0
- package/src/lib/innovation-genome-v5.orig.ts +1210 -0
- package/src/lib/innovation-genome-v5.ts +2 -0
- package/src/lib/innovation-genome-v7.orig.ts +2549 -0
- package/src/lib/innovation-genome-v7.ts +2 -0
- package/src/lib/innovation-genome-v8.orig.ts +1485 -0
- package/src/lib/innovation-genome-v8.ts +2 -0
- package/src/lib/innovation-genome-v9.orig.ts +643 -0
- package/src/lib/innovation-genome-v9.ts +2 -0
- package/src/lib/jina.base.ts +293 -0
- package/src/lib/jina.ts +1 -0
- package/src/lib/live-telemetry.ts +61 -0
- package/src/lib/longwriter.ts +84 -0
- package/src/lib/manuscript-grade-v2.ts +298 -0
- package/src/lib/memory-governor.ts +173 -0
- package/src/lib/memory-stress-tests.ts +248 -0
- package/src/lib/model-intelligence.ts +48 -0
- package/src/lib/model-rotator.ts +75 -0
- package/src/lib/models.orig.ts +513 -0
- package/src/lib/models.ts +151 -0
- package/src/lib/n-deep.base.ts +592 -0
- package/src/lib/n-deep.ts +51 -0
- package/src/lib/nih-simulator.ts +56 -0
- package/src/lib/nih-vulnerability-fixes.ts +460 -0
- package/src/lib/omega-templates.base.ts +294 -0
- package/src/lib/omega-templates.ts +225 -0
- package/src/lib/omni-nexus.ts +64 -0
- package/src/lib/oracle-adapters.ts +67 -0
- package/src/lib/oracle-registry.ts +163 -0
- package/src/lib/orchestrator.ts +357 -0
- package/src/lib/output-boundary.ts +235 -0
- package/src/lib/overrides/vite-native-chaos-harness.ts +154 -0
- package/src/lib/overrides/vite-native-contract-plane.ts +245 -0
- package/src/lib/overrides/vite-native-doctor-plane.ts +208 -0
- package/src/lib/overrides/vite-native-knowledge-store.ts +267 -0
- package/src/lib/overrides/vite-native-policy-plane.ts +190 -0
- package/src/lib/overrides/vite-native-replay-plane.ts +197 -0
- package/src/lib/overrides/vite-native-runtime-plane.ts +106 -0
- package/src/lib/overrides/vite-native-scraper.ts +848 -0
- package/src/lib/overrides/vite-native-selftest-functional.ts +217 -0
- package/src/lib/overrides/vite-native-selftest.ts +183 -0
- package/src/lib/overrides/vite-native-snapshot-plane.ts +123 -0
- package/src/lib/philosophy-toolkit.ts +75 -0
- package/src/lib/pipeline.orig.ts +659 -0
- package/src/lib/pipeline.ts +42 -0
- package/src/lib/precache.ts +128 -0
- package/src/lib/py-sandbox.ts +205 -0
- package/src/lib/quality-score.ts +96 -0
- package/src/lib/quant-engine.ts +231 -0
- package/src/lib/quant-lib.ts +80 -0
- package/src/lib/reportos.ts +175 -0
- package/src/lib/research-os.ts +306 -0
- package/src/lib/resource-estimator.ts +114 -0
- package/src/lib/rpm-governor.ts +93 -0
- package/src/lib/runtime-estimator.ts +135 -0
- package/src/lib/sage/empty-space-v6.ts +560 -0
- package/src/lib/sage/empty-space-v7.ts +222 -0
- package/src/lib/sage/empty-space.ts +733 -0
- package/src/lib/sage/engine.ts +786 -0
- package/src/lib/sage/index.ts +17 -0
- package/src/lib/sage/judge.ts +246 -0
- package/src/lib/sage/prng.ts +53 -0
- package/src/lib/sage/reality-grounding.ts +380 -0
- package/src/lib/sage/repetition.ts +151 -0
- package/src/lib/sage/saga-os-v2-engines.ts +1870 -0
- package/src/lib/sage/saga-os-v2.ts +274 -0
- package/src/lib/sage/saga-os-v3-core.ts +1491 -0
- package/src/lib/sage/saga-os-v3-release.ts +739 -0
- package/src/lib/sage/saga-os-v4-aureate.ts +1161 -0
- package/src/lib/sage/saga-os-v4.ts +212 -0
- package/src/lib/sage/saga-os-v5.ts +205 -0
- package/src/lib/sage/saga-os-v6.ts +346 -0
- package/src/lib/sage/saga-os-v7.ts +314 -0
- package/src/lib/sage/saga-os-v8.ts +505 -0
- package/src/lib/sage/saga-os.ts +329 -0
- package/src/lib/sage/taxonomy.ts +403 -0
- package/src/lib/sage/unified-roll.ts +524 -0
- package/src/lib/sage/worked-sketches.ts +44 -0
- package/src/lib/sage.ts +166 -0
- package/src/lib/scraper-debug-bus.ts +53 -0
- package/src/lib/scraper-enhanced.orig.ts +153 -0
- package/src/lib/scraper-enhanced.ts +10 -0
- package/src/lib/scraper-hardener.base.orig.ts +155 -0
- package/src/lib/scraper-hardener.base.ts +317 -0
- package/src/lib/scraper-hardener.ts +1 -0
- package/src/lib/scraper-palisade/palisade-adjudicator.orig.ts +419 -0
- package/src/lib/scraper-palisade/palisade-adjudicator.ts +13 -0
- package/src/lib/scraper-vnext/arbiter-omega.orig.ts +688 -0
- package/src/lib/scraper-vnext/arbiter-omega.ts +13 -0
- package/src/lib/scraper-vnext/canonical-portfolio-augments.ts +479 -0
- package/src/lib/scraper-vnext/canonical-portfolio-orchestrator.orig.ts +1529 -0
- package/src/lib/scraper-vnext/canonical-portfolio-orchestrator.ts +13 -0
- package/src/lib/scraper-vnext/conclave-omega.ts +1830 -0
- package/src/lib/scraper-vnext/content-extractor-v2.ts +226 -0
- package/src/lib/scraper-vnext/diagnostics-suite.ts +36 -0
- package/src/lib/scraper-vnext/epistemic-packer.ts +97 -0
- package/src/lib/scraper-vnext/fusion-v2.ts +65 -0
- package/src/lib/scraper-vnext/hydra-reader.orig.ts +1216 -0
- package/src/lib/scraper-vnext/hydra-reader.ts +13 -0
- package/src/lib/scraper-vnext/native-scraper-browser-vnext.orig.ts +91 -0
- package/src/lib/scraper-vnext/native-scraper-browser-vnext.ts +13 -0
- package/src/lib/scraper-vnext/nexus-consensus.orig.ts +905 -0
- package/src/lib/scraper-vnext/nexus-consensus.ts +13 -0
- package/src/lib/scraper-vnext/portfolio-consensus-adjudicator.ts +804 -0
- package/src/lib/scraper-vnext/portfolio-consensus-memory.ts +377 -0
- package/src/lib/scraper-vnext/portfolio-terminal-governor.ts +548 -0
- package/src/lib/scraper-vnext/query-strategist.orig.ts +224 -0
- package/src/lib/scraper-vnext/query-strategist.ts +423 -0
- package/src/lib/scraper-vnext/retrieval-accelerator.ts +708 -0
- package/src/lib/scraper-vnext/retrieval-audit-augments.ts +477 -0
- package/src/lib/scraper-vnext/retrieval-control-plane.ts +904 -0
- package/src/lib/scraper-vnext/retrieval-policy-augments.ts +358 -0
- package/src/lib/scraper-vnext/safe-fetch-v2.ts +157 -0
- package/src/lib/scraper-vnext/sentinel-omega.ts +56 -0
- package/src/lib/scraper-vnext/sentinel-orchestrator.ts +69 -0
- package/src/lib/scraper-vnext/sibyl-oracle.orig.ts +413 -0
- package/src/lib/scraper-vnext/sibyl-oracle.ts +13 -0
- package/src/lib/scraper-vnext/smart-read-v2.ts +55 -0
- package/src/lib/scraper-vnext/spa-rescue-bridge.ts +608 -0
- package/src/lib/scraper-vnext/strata-engine.orig.ts +2666 -0
- package/src/lib/scraper-vnext/strata-engine.ts +13 -0
- package/src/lib/scraper-vnext/structured-source-adapter.orig.ts +369 -0
- package/src/lib/scraper-vnext/structured-source-adapter.ts +34 -0
- package/src/lib/scraper-vnext/terminal-complete.ts +224 -0
- package/src/lib/scraper-vnext/terminal-final.ts +277 -0
- package/src/lib/scraper-vnext/terminal-saturation.ts +345 -0
- package/src/lib/scraper-vnext/terminal-wire.ts +425 -0
- package/src/lib/scraper-vnext/vanguard-titanium.orig.ts +467 -0
- package/src/lib/scraper-vnext/vanguard-titanium.ts +17 -0
- package/src/lib/search-cache.ts +138 -0
- package/src/lib/sloop-runner.base.ts +223 -0
- package/src/lib/sloop-runner.ts +104 -0
- package/src/lib/sloop.ts +114 -0
- package/src/lib/small-model-adapters.ts +82 -0
- package/src/lib/sscp.ts +86 -0
- package/src/lib/system-map.ts +54 -0
- package/src/lib/uni-node.ts +86 -0
- package/src/lib/universal-rigor-guard.ts +433 -0
- package/src/lib/v15-gate-testbed.ts +274 -0
- package/src/lib/v15-gemini.base.ts +89 -0
- package/src/lib/v15-gemini.ts +1 -0
- package/src/lib/v15-grounding.orig.ts +241 -0
- package/src/lib/v15-grounding.ts +79 -0
- package/src/lib/v15-pipeline.base.ts +769 -0
- package/src/lib/v15-pipeline.orig.ts +1154 -0
- package/src/lib/v15-pipeline.ts +674 -0
- package/src/lib/v15-questions.ts +64 -0
- package/src/lib/v15-rate-limiter.orig.ts +91 -0
- package/src/lib/v15-rate-limiter.ts +48 -0
- package/src/lib/v15-state.base.ts +87 -0
- package/src/lib/v15-state.ts +28 -0
- package/src/lib/visual-table-generator.tsx +207 -0
- package/src/lib/wasm-runtime.ts +133 -0
- package/src/lib/williams-persona-guide.ts +265 -0
- package/src/lib/williams-style.base.ts +464 -0
- package/src/lib/williams-style.ts +59 -0
- package/src/lib/worker-pool.ts +96 -0
- package/src/lib/writing-tiers.ts +125 -0
- package/src/main.orig.tsx +10 -0
- package/src/main.tsx +10 -0
- package/src/utils/cn.orig.ts +6 -0
- package/src/utils/cn.ts +6 -0
- package/src/vite-env.d.ts +1 -0
- package/tsconfig.json +31 -0
- package/unify.mjs +537 -0
- package/vite.config.ts +18 -0
|
@@ -0,0 +1,1154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v15-engine.ts — additive enhanced orchestration engine.
|
|
3
|
+
*
|
|
4
|
+
* This module is intentionally separate from `v15-pipeline.ts`: legacy exports
|
|
5
|
+
* stay available, while the calibration UI prefers these enhanced functions.
|
|
6
|
+
*
|
|
7
|
+
* Core upgrades (all additive, none remove capabilities):
|
|
8
|
+
* - Real, honored token budgets (Gemini otherwise silently truncates)
|
|
9
|
+
* - No fake 7.5 judge fallback — parse failures are honestly excluded
|
|
10
|
+
* - Reduced/capped testbed scoring (experimental gates can't collapse score)
|
|
11
|
+
* - Monotonic best-pass selection (never returns a worse pass than we had)
|
|
12
|
+
* - Per-pass deterministic PassDraftStats (chars/words/gates/etc, no LLM)
|
|
13
|
+
*
|
|
14
|
+
* TURN 11 ADDITIONS (research-grounded, dramatic quality lift):
|
|
15
|
+
* A. Best-of-N Physical Cluster Drafting — when profile.cluster is true, we
|
|
16
|
+
* PHYSICALLY generate min(clusterSize, 5) candidate drafts in parallel
|
|
17
|
+
* on different rotated models, then deterministically pick the highest-
|
|
18
|
+
* scoring one. Previously "cluster" was just a prompt hint the LLM
|
|
19
|
+
* mostly ignored; now it's a real multi-hypothesis synthesis stage.
|
|
20
|
+
* B. Chain-of-Verification (CoVe) — Meta AI's ACL-2024 pattern that
|
|
21
|
+
* empirically reduces hallucination up to 46% on Wikipedia QA. After the
|
|
22
|
+
* draft, the model plans verification questions about its own claims,
|
|
23
|
+
* answers each INDEPENDENTLY (no cross-contamination), then any
|
|
24
|
+
* mismatches are injected as high-priority items for the editor.
|
|
25
|
+
* C. Reasoning preamble — the draft LLM is asked to (silently) outline
|
|
26
|
+
* the structure of a 9.9 answer before writing prose. This dramatically
|
|
27
|
+
* improves completeness on complex questions.
|
|
28
|
+
* D. Enhanced editor prompt — explicitly names what to preserve verbatim,
|
|
29
|
+
* what to expand with specifics (dates/quantities/entities), and what
|
|
30
|
+
* to cross-reference to the evidence block. Enforces monotonic growth.
|
|
31
|
+
* E. Polish pass — a final zero-content-change pass runs after N-Deep to
|
|
32
|
+
* enforce terminal punctuation, add a References section if [S#] tags
|
|
33
|
+
* are used, strip any leaked scaffolding, and complete truncated
|
|
34
|
+
* sections. Guaranteed additive — deterministic checks first, only
|
|
35
|
+
* invokes an LLM call if the guard score is below 9.0.
|
|
36
|
+
* F. Enhanced judge prompt — richer rubric with concrete anchors so
|
|
37
|
+
* judges score consistently at the top end instead of clustering low.
|
|
38
|
+
*
|
|
39
|
+
* All new features are ON BY DEFAULT (via profile flags that already exist
|
|
40
|
+
* in the calibration UI). Nothing existing is removed or degraded.
|
|
41
|
+
*/
|
|
42
|
+
export * from "./v15-pipeline.base";
|
|
43
|
+
|
|
44
|
+
import { ensureFlawsLoaded } from "@/lib/flaws";
|
|
45
|
+
import { runFlawScan, runAutoFix, type ScanContext, type FlawIssue } from "@/lib/flaw-registry";
|
|
46
|
+
import { ensureOriginalDefensePackLoaded } from "@/lib/flaws/original-defenses-pack";
|
|
47
|
+
import { geminiGenerate } from "./v15-gemini";
|
|
48
|
+
import { calculateEloConsensus, getModelEloInfo, type EloJudgment } from "./elo-registry";
|
|
49
|
+
import { runAdversarialRedTeam } from "./adversarial-engine";
|
|
50
|
+
import { generateWithRotation, getActiveRotationPool } from "./model-rotator";
|
|
51
|
+
import { tryAcquire, recordResult, pickLeastLoaded } from "./v15-rate-limiter";
|
|
52
|
+
import { runTestbedGates, proposeGateWithLLM, type TestbedGate, type TestbedGateIssue } from "./v15-gate-testbed";
|
|
53
|
+
import { groundQuestion } from "./v15-grounding";
|
|
54
|
+
import { getPersonaDirective } from "./williams-style";
|
|
55
|
+
import { buildAdaptiveTemplateContract, buildTemplatePrompt, OMEGA_TEMPLATES, buildTemplateSearchQueries, buildHandTraceInstruction } from "./omega-templates";
|
|
56
|
+
import { CitationLedger, type CitationLedgerSnapshot } from "./citation-ledger";
|
|
57
|
+
import { detectTruncation, type V15RunOutcome, type V15Profile } from "./v15-pipeline.base";
|
|
58
|
+
|
|
59
|
+
// ─── Types ─────────────────────────────────────────────────────────────────
|
|
60
|
+
export interface PassDraftStats {
|
|
61
|
+
pass: number;
|
|
62
|
+
modelUsed: string;
|
|
63
|
+
charCount: number;
|
|
64
|
+
wordCount: number;
|
|
65
|
+
sentenceCount: number;
|
|
66
|
+
avgSentenceLen: number;
|
|
67
|
+
citationCount: number;
|
|
68
|
+
codeBlockCount: number;
|
|
69
|
+
headingCount: number;
|
|
70
|
+
tableRowCount: number;
|
|
71
|
+
criticalCount: number;
|
|
72
|
+
majorCount: number;
|
|
73
|
+
warningCount: number;
|
|
74
|
+
canonicalGateHits: string[];
|
|
75
|
+
testbedGateHits: string[];
|
|
76
|
+
guardScore: number;
|
|
77
|
+
isBest: boolean;
|
|
78
|
+
}
|
|
79
|
+
export interface AdversarialPreview {
|
|
80
|
+
rawCritique: string;
|
|
81
|
+
defectCount: number;
|
|
82
|
+
verdict: "pass" | "revise";
|
|
83
|
+
categories: string[];
|
|
84
|
+
}
|
|
85
|
+
/** NEW turn-11: CoVe verification result — surfaced to the UI so the user
|
|
86
|
+
* can see what factual claims were checked and which failed. */
|
|
87
|
+
export interface CoVeReport {
|
|
88
|
+
questions: { question: string; expectedAnswer: string; verifiedAnswer: string; consistent: boolean }[];
|
|
89
|
+
inconsistencies: number;
|
|
90
|
+
ok: boolean;
|
|
91
|
+
}
|
|
92
|
+
/** NEW turn-11: Best-of-N draft candidates — one row per parallel draft with
|
|
93
|
+
* its score, so the UI can visualize why one draft won. */
|
|
94
|
+
export interface BestOfNCandidate {
|
|
95
|
+
index: number;
|
|
96
|
+
model: string;
|
|
97
|
+
charCount: number;
|
|
98
|
+
guardScore: number;
|
|
99
|
+
chosen: boolean;
|
|
100
|
+
/** NEW: outline-first mode — "outline" = dense skeleton only (not expanded),
|
|
101
|
+
* "expanded" = the winning outline after full-token expansion. */
|
|
102
|
+
stage?: "outline" | "expanded";
|
|
103
|
+
/** NEW: short preview of the candidate's content for UI transparency. */
|
|
104
|
+
snippet?: string;
|
|
105
|
+
}
|
|
106
|
+
export interface V15EnhancedOutcome extends V15RunOutcome {
|
|
107
|
+
passHistory: PassDraftStats[];
|
|
108
|
+
bestPassIndex: number;
|
|
109
|
+
adversarialPreview?: AdversarialPreview;
|
|
110
|
+
judgeExcluded?: { model: string; reason: string }[];
|
|
111
|
+
/** NEW turn-11: only populated when profile.cluster generates real parallel drafts. */
|
|
112
|
+
bestOfNCandidates?: BestOfNCandidate[];
|
|
113
|
+
/** NEW turn-11: only populated when profile.webSearch enables CoVe verification. */
|
|
114
|
+
coveReport?: CoVeReport;
|
|
115
|
+
/** NEW turn-11: true if the deterministic polish pass produced a real change. */
|
|
116
|
+
polishApplied?: boolean;
|
|
117
|
+
/** Citation provenance audit — each [S#] tag mapped to its source with trust score. */
|
|
118
|
+
citationAudit?: CitationLedgerSnapshot;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Local extension to the profile shape — additive only.
|
|
122
|
+
type EnhancedV15Profile = V15Profile & {
|
|
123
|
+
/** How many DISTINCT LLMs/models to use in the real Best-of-N stage. */
|
|
124
|
+
bestOfNModels?: number;
|
|
125
|
+
/** How many hypotheses/candidates to generate in total. */
|
|
126
|
+
bestOfNHypotheses?: number;
|
|
127
|
+
/** When true, allow one model to generate multiple hypotheses in a single call to save RPM/RPD. */
|
|
128
|
+
bestOfNPackHypotheses?: boolean;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
// ─── Deterministic Draft Stats (no LLM) ────────────────────────────────────
|
|
132
|
+
function computeDraftStats(
|
|
133
|
+
text: string,
|
|
134
|
+
canonical: FlawIssue[],
|
|
135
|
+
testbed: TestbedGateIssue[],
|
|
136
|
+
pass: number,
|
|
137
|
+
modelUsed: string,
|
|
138
|
+
guardScore: number,
|
|
139
|
+
): PassDraftStats {
|
|
140
|
+
const words = text.trim().split(/\s+/).filter(Boolean);
|
|
141
|
+
const sentences = text.split(/(?<=[.!?])\s+/).filter(s => s.trim().length > 0);
|
|
142
|
+
const sentenceCount = sentences.length || 1;
|
|
143
|
+
const avgSentenceLen = Math.round((words.length / sentenceCount) * 10) / 10;
|
|
144
|
+
const citationCount = (text.match(/\[S\d+\]/g) || []).length;
|
|
145
|
+
const codeBlockCount = Math.floor((text.match(/```/g) || []).length / 2);
|
|
146
|
+
const headingCount = (text.match(/^#{1,6}\s+\S/gm) || []).length;
|
|
147
|
+
const tableRowCount = (text.match(/^\|.*\|$/gm) || []).length;
|
|
148
|
+
let criticalCount = 0, majorCount = 0, warningCount = 0;
|
|
149
|
+
const canonicalGateHits: string[] = [];
|
|
150
|
+
for (const i of canonical) {
|
|
151
|
+
canonicalGateHits.push(i.code);
|
|
152
|
+
if (i.severity === "critical") criticalCount++;
|
|
153
|
+
else if (i.severity === "major") majorCount++;
|
|
154
|
+
else if (i.severity === "warning") warningCount++;
|
|
155
|
+
}
|
|
156
|
+
const testbedGateHits: string[] = [];
|
|
157
|
+
for (const i of testbed) {
|
|
158
|
+
testbedGateHits.push(i.code);
|
|
159
|
+
if (i.severity === "critical") criticalCount++;
|
|
160
|
+
else if (i.severity === "major") majorCount++;
|
|
161
|
+
else if (i.severity === "warning") warningCount++;
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
pass, modelUsed, charCount: text.length, wordCount: words.length,
|
|
165
|
+
sentenceCount, avgSentenceLen,
|
|
166
|
+
citationCount, codeBlockCount, headingCount, tableRowCount,
|
|
167
|
+
criticalCount, majorCount, warningCount,
|
|
168
|
+
canonicalGateHits, testbedGateHits, guardScore, isBest: false,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ─── Causal-family deduplication for gate scoring ──────────────────────────
|
|
173
|
+
// Groups related gate codes into causal families to prevent correlated detectors
|
|
174
|
+
// (e.g., HALLUCINATED_CITATION + MISSING_CITATION_REF + HAL_CITE_REF_VOID) from
|
|
175
|
+
// acting as three independent failures when they share the same root cause.
|
|
176
|
+
// Each causal family contributes at most 2 representative issues to the score.
|
|
177
|
+
const GATE_FAMILY_PREFIXES = [
|
|
178
|
+
["HALLUCINAT", "HAL_", "CITES_WITH_NO_SOURCES", "MISSING_CITATION_REF", "HAL_CITE_REF_VOID"],
|
|
179
|
+
["TRUNC", "INCOMPLETE_"],
|
|
180
|
+
["LLM_PROMPT_LEAKAGE", "LLM_GEN_TEMPLATE", "INTERNAL_PROMPT"],
|
|
181
|
+
["REPEAT_", "DUPLICATE_"],
|
|
182
|
+
["UNSAFE_", "SAFETY_"],
|
|
183
|
+
["LANG_", "STYLE_"],
|
|
184
|
+
["CODE_", "FMT_"],
|
|
185
|
+
];
|
|
186
|
+
function getFamilyKey(code: string): string {
|
|
187
|
+
for (let i = 0; i < GATE_FAMILY_PREFIXES.length; i++) {
|
|
188
|
+
if (GATE_FAMILY_PREFIXES[i].some(pfx => code.startsWith(pfx) || code === pfx)) return `F${i}`;
|
|
189
|
+
}
|
|
190
|
+
return code; // unique family per unknown code
|
|
191
|
+
}
|
|
192
|
+
function dedupeByFamily(issues: Array<{ code: string; severity: string; message: string; remediation?: string }>) {
|
|
193
|
+
const familyCounts = new Map<string, number>();
|
|
194
|
+
const deduped: typeof issues = [];
|
|
195
|
+
for (const i of issues) {
|
|
196
|
+
const fk = getFamilyKey(i.code);
|
|
197
|
+
const count = familyCounts.get(fk) ?? 0;
|
|
198
|
+
if (count < 2) { deduped.push(i); familyCounts.set(fk, count + 1); }
|
|
199
|
+
}
|
|
200
|
+
return deduped;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// ─── Reduced-weight, capped guard scoring ──────────────────────────────────
|
|
204
|
+
function scoreFromIssuesV2(canonical: FlawIssue[], testbed: TestbedGateIssue[], textLen: number): number {
|
|
205
|
+
// Dedupe by causal family before scoring: 2 issues max per family
|
|
206
|
+
const deduped = dedupeByFamily([
|
|
207
|
+
...canonical.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
208
|
+
]);
|
|
209
|
+
let err = 10, math = 10, style = 9.5, hall = 0;
|
|
210
|
+
for (const i of deduped) {
|
|
211
|
+
if (i.severity === "critical") { err -= 2.5; math -= 2.0; hall += 2.5; }
|
|
212
|
+
else if (i.severity === "major") { err -= 1.2; math -= 1.5; hall += 1.2; }
|
|
213
|
+
else if (i.severity === "warning") { err -= 0.4; math -= 0.4; hall += 0.4; }
|
|
214
|
+
}
|
|
215
|
+
// Testbed gets reduced weight + family cap + hard ceiling
|
|
216
|
+
const testbedDeduped = dedupeByFamily(testbed.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })));
|
|
217
|
+
let tErr = 0, tMath = 0, tHall = 0;
|
|
218
|
+
for (const i of testbedDeduped) {
|
|
219
|
+
if (i.severity === "critical") { tErr += 1.0; tMath += 0.8; tHall += 1.0; }
|
|
220
|
+
else if (i.severity === "major") { tErr += 0.48; tMath += 0.6; tHall += 0.48; }
|
|
221
|
+
else if (i.severity === "warning") { tErr += 0.16; tMath += 0.16; tHall += 0.16; }
|
|
222
|
+
}
|
|
223
|
+
err -= Math.min(tErr, 2.0); math -= Math.min(tMath, 1.6); hall += Math.min(tHall, 2.0);
|
|
224
|
+
if (textLen < 150) style -= 1.0;
|
|
225
|
+
const clamp = (n: number) => Math.max(0, Math.min(10, n));
|
|
226
|
+
err = clamp(err); math = clamp(math); style = clamp(style); hall = clamp(hall);
|
|
227
|
+
return Math.round((0.40 * err + 0.30 * math + 0.20 * style + 0.10 * (10 - hall)) * 100) / 100;
|
|
228
|
+
}
|
|
229
|
+
function sevRank(s: string): number { return s === "critical" ? 4 : s === "major" ? 3 : s === "warning" ? 2 : 1; }
|
|
230
|
+
|
|
231
|
+
function issueVector(issues: Array<{ severity: string }>) {
|
|
232
|
+
return {
|
|
233
|
+
critical: issues.filter(i => i.severity === "critical").length,
|
|
234
|
+
major: issues.filter(i => i.severity === "major").length,
|
|
235
|
+
warning: issues.filter(i => i.severity === "warning").length,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Apply only explicit section patches. Unmentioned sections remain byte-for-byte. */
|
|
240
|
+
function applySectionPatches(draft: string, raw: string): { text: string; applied: number } {
|
|
241
|
+
const blocks = [...raw.matchAll(/<<<REVISE_SECTION>>>\s*ANCHOR:\s*([^\n]+)\nREVISED:\s*([\s\S]*?)<<<END_SECTION>>>/g)];
|
|
242
|
+
let text = draft;
|
|
243
|
+
let applied = 0;
|
|
244
|
+
for (const block of blocks.slice(0, 5)) {
|
|
245
|
+
const anchor = block[1].trim();
|
|
246
|
+
const replacement = block[2].trim();
|
|
247
|
+
if (!anchor || replacement.length < 40) continue;
|
|
248
|
+
const at = text.indexOf(anchor);
|
|
249
|
+
if (at < 0) continue;
|
|
250
|
+
const lineStart = text.lastIndexOf("\n", at) + 1;
|
|
251
|
+
const anchorLine = text.slice(lineStart, text.indexOf("\n", at) < 0 ? text.length : text.indexOf("\n", at));
|
|
252
|
+
let end = text.length;
|
|
253
|
+
if (/^#{1,6}\s/.test(anchorLine.trim())) {
|
|
254
|
+
const after = text.slice(lineStart + anchorLine.length + 1);
|
|
255
|
+
const next = after.search(/^#{1,6}\s/m);
|
|
256
|
+
if (next >= 0) end = lineStart + anchorLine.length + 1 + next;
|
|
257
|
+
} else {
|
|
258
|
+
const para = text.slice(at).search(/\n\s*\n/);
|
|
259
|
+
if (para >= 0) end = at + para;
|
|
260
|
+
}
|
|
261
|
+
text = text.slice(0, lineStart) + replacement + text.slice(end);
|
|
262
|
+
applied++;
|
|
263
|
+
}
|
|
264
|
+
return { text, applied };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ─── Model selection: round-robin fair + rate-aware ────────────────────────
|
|
268
|
+
const usedInRound = new Set<string>();
|
|
269
|
+
function pickModel(pool: string[]): string {
|
|
270
|
+
const leastLoaded = pickLeastLoaded(pool.filter(m => !usedInRound.has(m)));
|
|
271
|
+
const winner = leastLoaded ?? pickLeastLoaded(pool) ?? pool[Math.floor(Math.random() * pool.length)];
|
|
272
|
+
usedInRound.add(winner);
|
|
273
|
+
if (usedInRound.size >= pool.length) usedInRound.clear();
|
|
274
|
+
return winner;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Deterministically score a candidate text through the full gate stack. */
|
|
278
|
+
function scoreCandidate(question: string, text: string, longForm = false, targetPages = 4) {
|
|
279
|
+
const ctx: ScanContext = { prompt: question, answer: text, lowerAnswer: text.toLowerCase(), computeRecords: [], constraints: { explicitComparisonTargets: [], exclusions: [], formatHints: [], namedEntities: [] } as any };
|
|
280
|
+
const canonical = runFlawScan(ctx);
|
|
281
|
+
const testbed = runTestbedGates(text);
|
|
282
|
+
const trunc = detectTruncation(text, { longForm, targetPages });
|
|
283
|
+
const canonicalWithTrunc: FlawIssue[] = trunc.truncated
|
|
284
|
+
? [{ code: "TRUNCATED_OPENING", severity: "critical" as const, message: `Structural truncation: ${trunc.reason}.`, remediation: "Regenerate a COMPLETE answer.", autofixable: false } as FlawIssue, ...canonical]
|
|
285
|
+
: canonical;
|
|
286
|
+
const guardScore = scoreFromIssuesV2(canonicalWithTrunc, testbed, text.length);
|
|
287
|
+
return { guardScore, canonical: canonicalWithTrunc, testbed, truncated: trunc.truncated };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ─── Best-of-N Outline-First Cluster Drafting (Skeleton-of-Thought / STORM) ──
|
|
291
|
+
/**
|
|
292
|
+
* TURN 12 REWRITE — Outline-first Best-of-N, replacing full-draft-per-hypothesis.
|
|
293
|
+
*
|
|
294
|
+
* Root cause fixed: generating N COMPLETE full-length drafts (previous turn-11
|
|
295
|
+
* design) burns N× the draft token budget even though only ONE candidate is
|
|
296
|
+
* ever kept — pure waste, exactly as flagged. This is also NOT how frontier
|
|
297
|
+
* "deep research" agents (Claude/Gemini/GPT/Grok) or the published literature
|
|
298
|
+
* approach multi-hypothesis synthesis:
|
|
299
|
+
* - Skeleton-of-Thought (Ning et al., ICLR 2024): draft a short SKELETON
|
|
300
|
+
* first, then expand only the chosen structure — reduces generation cost
|
|
301
|
+
* substantially while maintaining/improving quality vs. single-shot.
|
|
302
|
+
* - STORM (Shao et al., NAACL 2024): outline-driven long-form synthesis —
|
|
303
|
+
* generate compact multi-perspective OUTLINES, select/merge the strongest,
|
|
304
|
+
* THEN write full prose from the winning outline.
|
|
305
|
+
* Applying this pattern here: each "hypothesis" is now a DENSE, information-
|
|
306
|
+
* rich OUTLINE (bullet skeleton of thesis + section plan + key facts/numbers/
|
|
307
|
+
* caveats to cover) generated at a SMALL token budget (~350-500 tokens each),
|
|
308
|
+
* scored with a fast structural-density heuristic (no LLM call), and ONLY the
|
|
309
|
+
* winning outline is expanded into a full draft using the full token budget.
|
|
310
|
+
* Net effect: N outlines + 1 expansion, instead of N full drafts — a ~(N-1)/N
|
|
311
|
+
* reduction in draft-stage token spend for the SAME or better final quality,
|
|
312
|
+
* because the winning structure is chosen BEFORE committing prose tokens.
|
|
313
|
+
*/
|
|
314
|
+
|
|
315
|
+
/** Fast, deterministic density/coverage heuristic for scoring a compact outline
|
|
316
|
+
* (no LLM call — mirrors what a competent editor would look for structurally).
|
|
317
|
+
* Now also rewards template section coverage for OMEGA template tasks. */
|
|
318
|
+
function scoreOutlineDensity(outline: string, templateId?: string): number {
|
|
319
|
+
const t = outline.trim();
|
|
320
|
+
if (t.length < 20) return 0;
|
|
321
|
+
let score = 0;
|
|
322
|
+
const bulletCount = (t.match(/^\s*[-*•\d]/gm) || []).length;
|
|
323
|
+
score += Math.min(bulletCount, 10) * 0.6; // structural breadth, capped
|
|
324
|
+
if (/\bassumption/i.test(t)) score += 1;
|
|
325
|
+
if (/\b(unit|units|%|percent|\$|USD|hour|day|month|year)\b/i.test(t)) score += 1; // quantitative intent
|
|
326
|
+
if (/\bjurisdiction|scope|caveat|limitation/i.test(t)) score += 1;
|
|
327
|
+
if (/\breference|citation|\[S\d+\]|source/i.test(t)) score += 1;
|
|
328
|
+
if (/\bworked example|calculation|derivation/i.test(t)) score += 1;
|
|
329
|
+
if (/\bfabricat|hallucinat|invent(ed)?\b/i.test(t)) score -= 2; // self-flagged risk
|
|
330
|
+
const wordCount = t.split(/\s+/).filter(Boolean).length;
|
|
331
|
+
if (wordCount < 25) score -= 1; // too thin to be a real plan
|
|
332
|
+
if (wordCount > 400) score -= 0.5; // outline should stay dense, not become a draft
|
|
333
|
+
// Bonus: OMEGA template section coverage — reward outlines that mention required sections
|
|
334
|
+
if (templateId) {
|
|
335
|
+
const tmpl = OMEGA_TEMPLATES.find(x => x.id === templateId);
|
|
336
|
+
if (tmpl) {
|
|
337
|
+
const lower = t.toLowerCase();
|
|
338
|
+
const covered = tmpl.sections.filter(s => lower.includes(s.title.toLowerCase().slice(0, 8)));
|
|
339
|
+
score += Math.min(covered.length * 0.4, 2.0);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return Math.max(0, score);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
async function runBestOfNDraft(opts: {
|
|
346
|
+
apiKey: string;
|
|
347
|
+
question: string;
|
|
348
|
+
evidenceBlock: string;
|
|
349
|
+
systemInstruction: string;
|
|
350
|
+
hypothesisCount: number;
|
|
351
|
+
modelCount: number;
|
|
352
|
+
packedPerModel?: boolean;
|
|
353
|
+
maxOutputTokens: number;
|
|
354
|
+
longForm: boolean;
|
|
355
|
+
targetPages: number;
|
|
356
|
+
templateId?: string;
|
|
357
|
+
onProgress?: (s: string) => void;
|
|
358
|
+
}): Promise<{ text: string; modelUsed: string; candidates: BestOfNCandidate[] }> {
|
|
359
|
+
const pool = getActiveRotationPool();
|
|
360
|
+
const totalHypotheses = Math.max(2, Math.min(8, opts.hypothesisCount));
|
|
361
|
+
const modelCount = Math.max(1, Math.min(5, opts.modelCount));
|
|
362
|
+
const evidenceForOutline = opts.evidenceBlock ? `\n\n${opts.evidenceBlock.slice(0, 3000)}` : "";
|
|
363
|
+
const OUTLINE_TOKENS = 500;
|
|
364
|
+
|
|
365
|
+
// Build the actual OMEGA template skeleton for the outline prompt
|
|
366
|
+
const tmpl = opts.templateId ? OMEGA_TEMPLATES.find(x => x.id === opts.templateId) : undefined;
|
|
367
|
+
const templateSectionHint = tmpl
|
|
368
|
+
? `\nREQUIRED SECTIONS (must be represented in outline): ${tmpl.sections.map(s => s.title).join(", ")}.`
|
|
369
|
+
: "";
|
|
370
|
+
|
|
371
|
+
const outlineInstruction = (userQuestion: string) =>
|
|
372
|
+
`Produce a DENSE, information-rich OUTLINE (not a full answer) for a 9.9-quality response to the question below. Use short bullet points. Cover: (1) the core thesis/verdict, (2) ALL required template sections with a one-line summary of what each will contain, (3) any specific numbers/quantities/dates/entities you already know that must appear, (4) assumptions to state, (5) jurisdiction/scope caveats if relevant, (6) whether citations/references will be used. Keep it under 280 words. Do NOT write the full prose answer — only the dense skeleton.${templateSectionHint}${evidenceForOutline}\n\nQUESTION:\n${userQuestion}`;
|
|
373
|
+
|
|
374
|
+
let outlineResults: { model: string; outline: string }[] = [];
|
|
375
|
+
|
|
376
|
+
// === GROUPED-REQUEST MODE (dhuliawala et al., ACL 2024; korikov et al., 2025) ===
|
|
377
|
+
// When modelCount=1 and hypotheses>1, make ONE call with all N outlines grouped —
|
|
378
|
+
// most RPM/RPD efficient. All other combinations split across distinct models.
|
|
379
|
+
const useGroupedSingleModel = modelCount === 1 && totalHypotheses > 1;
|
|
380
|
+
|
|
381
|
+
if (useGroupedSingleModel) {
|
|
382
|
+
// Single model, all hypotheses in ONE call — maximally RPM-efficient
|
|
383
|
+
const singleModel = pickModel(pool);
|
|
384
|
+
opts.onProgress?.(`best-of-N: grouped request — generating ${totalHypotheses} outlines in ONE call to ${singleModel} (RPM-efficient)`);
|
|
385
|
+
const prompt = `${outlineInstruction(opts.question)}\n\nGenerate ${totalHypotheses} DISTINCT outlines (different structural approaches/reasoning angles for the SAME question). Each outline must address all required sections but with a different analytical lens.\n\nOutput EXACTLY in this format:\n<<<OUTLINE 1>>>\n<dense bullet outline — unique angle>\n<<<END>>>\n<<<OUTLINE 2>>>\n<dense bullet outline — different angle>\n<<<END>>>\n[continue for all ${totalHypotheses} outlines]\n\nNever output prose paragraphs, commentary, or text outside the outline blocks.`;
|
|
386
|
+
const r = await generateWithRotation({ apiKey: opts.apiKey, prompt, preferredModel: singleModel, maxOutputTokens: Math.max(OUTLINE_TOKENS * totalHypotheses, totalHypotheses * 250) });
|
|
387
|
+
if (r.ok) {
|
|
388
|
+
const blocks = [...r.text.matchAll(/<<<OUTLINE\s+\d+>>>([\s\S]*?)<<<END>>>/g)].map(x => x[1].trim()).filter(Boolean).slice(0, totalHypotheses);
|
|
389
|
+
outlineResults = blocks.map(o => ({ model: r.modelUsed, outline: o }));
|
|
390
|
+
}
|
|
391
|
+
} else if (opts.packedPerModel && modelCount < totalHypotheses) {
|
|
392
|
+
// Packed mode: each model emits multiple SHORT outlines in one call
|
|
393
|
+
const perModel = Math.ceil(totalHypotheses / modelCount);
|
|
394
|
+
opts.onProgress?.(`best-of-N: generating ${totalHypotheses} dense outlines via ${modelCount} model(s), packed ${perModel}/call (outline-first, RPM-saving)`);
|
|
395
|
+
const models = Array.from({ length: modelCount }, () => pickModel(pool));
|
|
396
|
+
const results = await Promise.all(models.map(async (m) => {
|
|
397
|
+
const prompt = `${outlineInstruction(opts.question)}\n\nGenerate ${perModel} DISTINCT outlines (different structural approaches/angles). Output EXACTLY:\n<<<OUTLINE 1>>>\n<dense bullet outline>\n<<<END>>>\n<<<OUTLINE 2>>>\n<dense bullet outline>\n<<<END>>>\n...\nNever output full prose paragraphs or commentary outside the outline blocks.`;
|
|
398
|
+
const r = await generateWithRotation({ apiKey: opts.apiKey, prompt, preferredModel: m, maxOutputTokens: Math.max(OUTLINE_TOKENS, perModel * 300) });
|
|
399
|
+
if (!r.ok) return { model: m, outlines: [] as string[] };
|
|
400
|
+
const blocks = [...r.text.matchAll(/<<<OUTLINE\s+\d+>>>([\s\S]*?)<<<END>>>/g)].map(x => x[1].trim()).filter(Boolean).slice(0, perModel);
|
|
401
|
+
return { model: r.modelUsed, outlines: blocks };
|
|
402
|
+
}));
|
|
403
|
+
outlineResults = results.flatMap(r => r.outlines.map(o => ({ model: r.model, outline: o }))).slice(0, totalHypotheses);
|
|
404
|
+
} else {
|
|
405
|
+
// Physical parallel mode: one short outline per distinct model.
|
|
406
|
+
const n = Math.max(2, Math.min(5, totalHypotheses));
|
|
407
|
+
opts.onProgress?.(`best-of-N: generating ${n} dense outlines on distinct rotated models (outline-first, expansion deferred to winner only)`);
|
|
408
|
+
const models = Array.from({ length: n }, () => pickModel(pool));
|
|
409
|
+
const results = await Promise.all(models.map(m => generateWithRotation({ apiKey: opts.apiKey, prompt: outlineInstruction(opts.question), preferredModel: m, maxOutputTokens: OUTLINE_TOKENS })));
|
|
410
|
+
outlineResults = results.map((r, i) => ({ model: r.ok ? r.modelUsed : models[i], outline: r.ok ? r.text.trim() : "" })).filter(r => r.outline);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
if (!outlineResults.length) {
|
|
414
|
+
// Fall back to a single direct full draft if outline generation totally failed.
|
|
415
|
+
opts.onProgress?.("best-of-N: outline generation failed for all candidates — falling back to single direct draft");
|
|
416
|
+
const fallback = await generateWithRotation({ apiKey: opts.apiKey, prompt: opts.evidenceBlock ? `${opts.evidenceBlock}\n\nUSER QUESTION:\n${opts.question}` : opts.question, preferredModel: pickModel(pool), systemInstruction: opts.systemInstruction, maxOutputTokens: opts.maxOutputTokens });
|
|
417
|
+
return { text: fallback.ok ? fallback.text : "", modelUsed: fallback.modelUsed, candidates: [] };
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const scoredOutlines = outlineResults.map((o, i) => ({ index: i, model: o.model, outline: o.outline, density: scoreOutlineDensity(o.outline, opts.templateId) }));
|
|
421
|
+
scoredOutlines.sort((a, b) => b.density - a.density);
|
|
422
|
+
const winnerOutline = scoredOutlines[0];
|
|
423
|
+
opts.onProgress?.(`best-of-N: selected outline #${winnerOutline.index + 1} from ${winnerOutline.model} (density ${winnerOutline.density.toFixed(1)}) — expanding to full draft`);
|
|
424
|
+
|
|
425
|
+
// Expand ONLY the winning outline into the full draft, using the full token budget.
|
|
426
|
+
// Include actual OMEGA template skeleton if available — gives the model the real sections
|
|
427
|
+
const tmplForExpansion = opts.templateId ? OMEGA_TEMPLATES.find(x => x.id === opts.templateId) : undefined;
|
|
428
|
+
const templateSkeletonBlock = tmplForExpansion
|
|
429
|
+
? `\n\nOUTPUT TEMPLATE STRUCTURE — follow EXACTLY this section order:\n${tmplForExpansion.sections.map(s => `${s.id} ${s.title}${s.pages ? ` (${s.pages})` : ""} — ${s.hint}`).join("\n")}\n`
|
|
430
|
+
: "";
|
|
431
|
+
const expansionPrompt = `You planned the following DENSE OUTLINE for your answer. Now write the COMPLETE, full-prose, publication-quality answer that fully realizes this outline — covering every planned section with substantive detail, worked numbers with units, assumptions, and caveats. Do not just repeat the outline; write real, complete prose.${templateSkeletonBlock}\n\nYOUR OUTLINE:\n${winnerOutline.outline}\n${opts.evidenceBlock ? `\n${opts.evidenceBlock}\n` : ""}\nUSER QUESTION:\n${opts.question}`;
|
|
432
|
+
const expandRes = await generateWithRotation({ apiKey: opts.apiKey, prompt: expansionPrompt, preferredModel: winnerOutline.model, systemInstruction: opts.systemInstruction, maxOutputTokens: opts.maxOutputTokens });
|
|
433
|
+
|
|
434
|
+
if (!expandRes.ok || !expandRes.text.trim()) {
|
|
435
|
+
// Expansion failed on the winner's own model — retry once on a different rotated model.
|
|
436
|
+
opts.onProgress?.(`best-of-N: expansion failed on ${winnerOutline.model} — retrying on a different model`);
|
|
437
|
+
const retryModel = pickModel(pool.filter(m => m !== winnerOutline.model));
|
|
438
|
+
const retryRes = await generateWithRotation({ apiKey: opts.apiKey, prompt: expansionPrompt, preferredModel: retryModel, systemInstruction: opts.systemInstruction, maxOutputTokens: opts.maxOutputTokens });
|
|
439
|
+
const candidates: BestOfNCandidate[] = scoredOutlines.map(s => ({ index: s.index, model: s.model, charCount: s.outline.length, guardScore: s.density, chosen: s === winnerOutline, stage: "outline", snippet: s.outline.slice(0, 160) }));
|
|
440
|
+
return { text: retryRes.ok ? retryRes.text : "", modelUsed: retryRes.modelUsed, candidates };
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
const candidates: BestOfNCandidate[] = scoredOutlines.map(s => {
|
|
444
|
+
const isWinner = s === winnerOutline;
|
|
445
|
+
return {
|
|
446
|
+
index: s.index,
|
|
447
|
+
model: s.model,
|
|
448
|
+
charCount: isWinner ? expandRes.text.length : s.outline.length,
|
|
449
|
+
guardScore: isWinner ? scoreCandidate(opts.question, expandRes.text, opts.longForm, opts.targetPages).guardScore : s.density,
|
|
450
|
+
chosen: isWinner,
|
|
451
|
+
stage: isWinner ? "expanded" : "outline",
|
|
452
|
+
snippet: isWinner ? expandRes.text.slice(0, 160) : s.outline.slice(0, 160),
|
|
453
|
+
};
|
|
454
|
+
});
|
|
455
|
+
opts.onProgress?.(`best-of-N: expansion complete (${expandRes.text.length} chars from ${expandRes.modelUsed}) — ${scoredOutlines.length - 1} outline(s) discarded before full-length generation, saving their draft-stage tokens`);
|
|
456
|
+
return { text: expandRes.text, modelUsed: expandRes.modelUsed, candidates };
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// ─── NEW turn-11: Chain-of-Verification (CoVe) ─────────────────────────────
|
|
460
|
+
async function runCoVeVerification(opts: {
|
|
461
|
+
apiKey: string;
|
|
462
|
+
question: string;
|
|
463
|
+
draft: string;
|
|
464
|
+
evidenceBlock: string;
|
|
465
|
+
onProgress?: (s: string) => void;
|
|
466
|
+
}): Promise<CoVeReport> {
|
|
467
|
+
try {
|
|
468
|
+
opts.onProgress?.("CoVe: planning verification questions");
|
|
469
|
+
const planPrompt = `You are a rigorous fact-checker. The following draft was produced in response to a USER QUESTION. Identify up to 4 SPECIFIC factual claims (dates, quantities, entities, causal relationships, definitions) whose incorrectness would materially damage the answer. For each claim, write a short verification question AND the answer the draft implies.\n\nUSER QUESTION:\n${opts.question}\n\nDRAFT:\n${opts.draft.slice(0, 6000)}\n\nReturn ONLY JSON: {"claims":[{"question":"<verification question>","expectedAnswer":"<what the draft implies>"}]}`;
|
|
470
|
+
const planRes = await generateWithRotation({ apiKey: opts.apiKey, prompt: planPrompt, maxOutputTokens: 700 });
|
|
471
|
+
if (!planRes.ok) return { questions: [], inconsistencies: 0, ok: false };
|
|
472
|
+
const cleaned = planRes.text.replace(/```json\s*/gi, "").replace(/```/g, "");
|
|
473
|
+
const match = cleaned.match(/\{[\s\S]*\}/);
|
|
474
|
+
if (!match) return { questions: [], inconsistencies: 0, ok: false };
|
|
475
|
+
let plan: { claims: { question: string; expectedAnswer: string }[] } = { claims: [] };
|
|
476
|
+
try { plan = JSON.parse(match[0]); } catch { return { questions: [], inconsistencies: 0, ok: false }; }
|
|
477
|
+
const claims = Array.isArray(plan.claims) ? plan.claims.slice(0, 4) : [];
|
|
478
|
+
if (claims.length === 0) return { questions: [], inconsistencies: 0, ok: true };
|
|
479
|
+
opts.onProgress?.(`CoVe: verifying ${claims.length} claim(s) independently`);
|
|
480
|
+
const verifyResults = await Promise.all(claims.map(async (c) => {
|
|
481
|
+
const nonce = Math.random().toString(36).slice(2);
|
|
482
|
+
const vp = `You are answering ONE factual question in isolation (nonce ${nonce}). Give the shortest correct answer possible.\n${opts.evidenceBlock ? `EVIDENCE:\n${opts.evidenceBlock.slice(0, 2000)}\n\n` : ""}QUESTION: ${c.question}\n\nAnswer (concise, factual, no hedging):`;
|
|
483
|
+
const r = await generateWithRotation({ apiKey: opts.apiKey, prompt: vp, maxOutputTokens: 200 });
|
|
484
|
+
const verified = r.ok ? r.text.trim().slice(0, 300) : "(verification failed)";
|
|
485
|
+
const norm = (s: string) => s.toLowerCase().replace(/[^\w\s]/g, "").trim();
|
|
486
|
+
const consistent = norm(verified).includes(norm(c.expectedAnswer).slice(0, 50)) || norm(c.expectedAnswer).includes(norm(verified).slice(0, 50));
|
|
487
|
+
return { question: c.question, expectedAnswer: c.expectedAnswer, verifiedAnswer: verified, consistent };
|
|
488
|
+
}));
|
|
489
|
+
const inconsistencies = verifyResults.filter(v => !v.consistent).length;
|
|
490
|
+
opts.onProgress?.(`CoVe: ${inconsistencies}/${verifyResults.length} claim(s) failed verification`);
|
|
491
|
+
return { questions: verifyResults, inconsistencies, ok: true };
|
|
492
|
+
} catch { return { questions: [], inconsistencies: 0, ok: false }; }
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// ─── NEW turn-11: Polish Pass ──────────────────────────────────────────────
|
|
496
|
+
async function runPolishPass(opts: {
|
|
497
|
+
apiKey: string;
|
|
498
|
+
question: string;
|
|
499
|
+
text: string;
|
|
500
|
+
guardScore: number;
|
|
501
|
+
onProgress?: (s: string) => void;
|
|
502
|
+
}): Promise<{ text: string; applied: boolean }> {
|
|
503
|
+
const t = opts.text;
|
|
504
|
+
const hasCitations = /\[S\d+\]/.test(t);
|
|
505
|
+
const hasReferencesSection = /(?:^|\n)(References|Sources|Bibliography)[:\s]/i.test(t);
|
|
506
|
+
// Exclude "Hand-Trace" from scaffolding detection — it's a legitimate appendix section
|
|
507
|
+
const hasScaffoldingLeak = /(JUDGE_NOTE|Meticulous Technical Editor|Recursive Refinement Pass|DRAFT TO REVISE|TESTBED_)/i.test(t) && !/Analytical Hand-Trace/i.test(t);
|
|
508
|
+
const lastChar = t.trim().slice(-1);
|
|
509
|
+
const needsTerminal = !!lastChar && !/[.!?)"'»”\]\}`]/.test(lastChar);
|
|
510
|
+
const unclosedFence = ((t.match(/```/g) ?? []).length % 2) !== 0;
|
|
511
|
+
const unclosedMath = ((t.match(/\$\$/g) ?? []).length % 2) !== 0;
|
|
512
|
+
const needsPolish = (hasCitations && !hasReferencesSection) || hasScaffoldingLeak || needsTerminal || unclosedFence || unclosedMath || opts.guardScore < 9.0;
|
|
513
|
+
if (!needsPolish) return { text: opts.text, applied: false };
|
|
514
|
+
opts.onProgress?.("polish pass: fixing structure/scaffolding without changing content");
|
|
515
|
+
const issues: string[] = [];
|
|
516
|
+
if (hasCitations && !hasReferencesSection) issues.push("Add a 'References' section at the end listing each [S#] cited (title + URL if known). Never invent URLs.");
|
|
517
|
+
if (hasScaffoldingLeak) issues.push("Remove any leaked scaffolding words: JUDGE_NOTE, 'Meticulous Technical Editor', 'Recursive Refinement Pass', 'DRAFT TO REVISE', 'TESTBED_'.");
|
|
518
|
+
if (needsTerminal) issues.push("Ensure the final sentence ends with proper terminal punctuation.");
|
|
519
|
+
if (unclosedFence) issues.push("Close every unclosed ``` code fence.");
|
|
520
|
+
if (unclosedMath) issues.push("Close every unclosed $$ math block.");
|
|
521
|
+
if (issues.length === 0) issues.push("Polish for prose clarity, section transitions, and completeness without adding new claims.");
|
|
522
|
+
const prompt = `You are a copy-editor. Return the ENTIRE answer, rewritten ONLY to satisfy the fixes below. Preserve every factual claim, quantity, citation tag, section, and section order. Do NOT shorten. Do NOT paraphrase content — only fix the specific defects.\n\nFIXES:\n${issues.map((i, k) => `${k + 1}. ${i}`).join("\n")}\n\nUSER QUESTION:\n${opts.question}\n\nANSWER TO POLISH:\n${opts.text}`;
|
|
523
|
+
const res = await generateWithRotation({ apiKey: opts.apiKey, prompt, maxOutputTokens: Math.max(3000, opts.text.length / 3 + 500) });
|
|
524
|
+
if (res.ok && res.text.trim().length >= opts.text.length * 0.85) {
|
|
525
|
+
opts.onProgress?.(`polish: applied (${res.text.length} chars vs ${opts.text.length})`);
|
|
526
|
+
return { text: res.text.trim(), applied: true };
|
|
527
|
+
}
|
|
528
|
+
return { text: opts.text, applied: false };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// ─── Template-aware judge builder ──────────────────────────────────────────
|
|
532
|
+
function buildJudgePrompt(templateId?: string, styleOverride?: string): string {
|
|
533
|
+
const tmpl = templateId ? OMEGA_TEMPLATES.find(t => t.id === templateId) : undefined;
|
|
534
|
+
const lines: string[] = [];
|
|
535
|
+
lines.push("You are an independent expert judge evaluating an AI answer" + (tmpl ? " structured as a " + tmpl.name + " professional report" : "") + ".");
|
|
536
|
+
lines.push("Grade the ANSWER to the QUESTION on a strict 0-10 scale.");
|
|
537
|
+
if (tmpl) {
|
|
538
|
+
lines.push("IMPORTANT TEMPLATE CONTEXT: This answer is a " + tmpl.name + " report (" + tmpl.tagline + "). Evaluate it as a professional " + tmpl.name + " deliverable, NOT as a simple Q&A answer. Corporate vocabulary, formal tone, section headings, numbered sections, and industry-specific terminology are CORRECT and REQUIRED — do NOT penalize them. Required sections: " + tmpl.sections.map(s => s.title).join(", ") + ". An answer covering all required sections with substantive prose should score higher than a plain Q&A answer.");
|
|
539
|
+
}
|
|
540
|
+
if (styleOverride && styleOverride !== "default") {
|
|
541
|
+
lines.push("Style override active: " + styleOverride + ". Evaluate style appropriateness for this specific corporate reporting format.");
|
|
542
|
+
}
|
|
543
|
+
lines.push("MANDATORY CAPS (apply the LOWEST that fits BEFORE the fine-grained rubric):");
|
|
544
|
+
lines.push("- Truncated / mid-sentence / fragment → HARD CAP 1");
|
|
545
|
+
lines.push("- Formula-only / partial calc without explanation → HARD CAP 3");
|
|
546
|
+
lines.push("- Restates question / requires reader to already know the answer → HARD CAP 4");
|
|
547
|
+
lines.push("- Missing required units, jurisdiction, or scope → HARD CAP 6");
|
|
548
|
+
lines.push("- Factually wrong on a load-bearing claim → HARD CAP 4");
|
|
549
|
+
lines.push("- Fabricated citations OR [S#] tags without a References section → HARD CAP 3");
|
|
550
|
+
lines.push("- Empty, off-topic, or leaks scratchpad/JSON → HARD CAP 2");
|
|
551
|
+
if (tmpl) { lines.push("- Missing >2 required template sections with no substantive content → HARD CAP 5"); }
|
|
552
|
+
lines.push("RUBRIC (within the applicable cap; sum 0-10):");
|
|
553
|
+
lines.push("- (0-3) Correctness & factual grounding, zero hallucination.");
|
|
554
|
+
lines.push("- (0-3) Direct, complete, standalone answer — reader can act on it immediately without more research." + (tmpl ? " Covers all required sections with substantive prose." : ""));
|
|
555
|
+
lines.push("- (0-2) Explicit reasoning: assumptions stated, variables defined, worked numbers WITH units.");
|
|
556
|
+
lines.push("- (0-2) Calibrated hedging: uncertainty surfaced, jurisdiction/scope declared, caveats named.");
|
|
557
|
+
lines.push("SCORE ANCHORS:");
|
|
558
|
+
lines.push("- 9.5-10: A domain expert would sign off without changes. Every claim verified, worked examples present." + (tmpl ? " All required sections substantively populated." : ""));
|
|
559
|
+
lines.push("- 8.0-9.4: Strong, complete, actionable — missing one or two sign-off items.");
|
|
560
|
+
return lines.join("\n");
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
// ─── Enhanced judge prompt (richer rubric, concrete anchors) ───────────────
|
|
564
|
+
// Default (no template) judge prompt — template-aware version built per-call below
|
|
565
|
+
const JUDGE_PROMPT_V2 = buildJudgePrompt() + "\n- 6.0-7.9: Correct but partial: broadly right, missing detail or hedging.\n- 4.0-5.9: Restates the question, hedges without answering, or requires the reader to already know the answer.\n- 1.0-3.9: Structurally broken (truncated / fragmented / off-topic) OR factually wrong on load-bearing claims.\nReturn ONLY strict JSON: {\"combinedScore\": <0-10 number>, \"shortNote\": \"<one sentence citing the specific cap and/or rubric anchor>\"}";
|
|
566
|
+
void JUDGE_PROMPT_V2; // used by judgeOneEnhanced below
|
|
567
|
+
|
|
568
|
+
async function judgeOneEnhanced(apiKey: string, question: string, answer: string, model: string, templateId?: string, styleOverride?: string) {
|
|
569
|
+
const acquired = await tryAcquire(model, true);
|
|
570
|
+
if (!acquired) return { model, score: 0, note: "", ok: false, excludedReason: "rate-limited (RPM/RPD exhausted)" };
|
|
571
|
+
const judgePrompt = (templateId ? buildJudgePrompt(templateId, styleOverride) : JUDGE_PROMPT_V2) +
|
|
572
|
+
"\n- 6.0-7.9: Correct but partial.\n- 4.0-5.9: Restates or requires prior knowledge.\n- 1.0-3.9: Structurally broken or factually wrong.\nReturn ONLY strict JSON: {\"combinedScore\": <0-10 number>, \"shortNote\": \"<one sentence citing the specific cap and/or rubric anchor>\"}";
|
|
573
|
+
const res = await geminiGenerate({ apiKey, model, prompt: judgePrompt + "\n\nQUESTION:\n" + question + "\n\nANSWER:\n" + answer.slice(0, 8000), maxOutputTokens: 500 });
|
|
574
|
+
recordResult(model, res.ok);
|
|
575
|
+
if (!res.ok) return { model, score: 0, note: "", ok: false, excludedReason: res.error ?? "generation failed" };
|
|
576
|
+
try {
|
|
577
|
+
const match = res.text.replace(/```json\s*/gi, "").replace(/```/g, "").match(/\{[\s\S]*?\}/);
|
|
578
|
+
const json = JSON.parse(match ? match[0] : res.text);
|
|
579
|
+
if (typeof json.combinedScore !== "number") return { model, score: 0, note: "", ok: false, excludedReason: "JSON missing combinedScore" };
|
|
580
|
+
return { model, score: Math.max(0, Math.min(10, json.combinedScore)), note: String(json.shortNote ?? "Judged").slice(0, 150), ok: true };
|
|
581
|
+
} catch { return { model, score: 0, note: "", ok: false, excludedReason: "JSON parse failure (not fabricated as 7.5)" }; }
|
|
582
|
+
}
|
|
583
|
+
export async function judgePanelEnhanced(opts: { apiKey: string; question: string; answer: string; judgeModels?: string[]; templateId?: string; styleOverride?: string }) {
|
|
584
|
+
const pool = getActiveRotationPool();
|
|
585
|
+
const models = opts.judgeModels ?? pool;
|
|
586
|
+
const t0 = Date.now();
|
|
587
|
+
const results = await Promise.all(models.map(m => judgeOneEnhanced(opts.apiKey, opts.question, opts.answer, m, opts.templateId, opts.styleOverride)));
|
|
588
|
+
const excluded: { model: string; reason: string }[] = [];
|
|
589
|
+
const finalResults = [...results];
|
|
590
|
+
for (let i = 0; i < finalResults.length; i++) {
|
|
591
|
+
if (finalResults[i].ok) continue;
|
|
592
|
+
const usedModels = new Set(finalResults.map(r => r.model));
|
|
593
|
+
const substitute = pickLeastLoaded(pool.filter(m => !usedModels.has(m)));
|
|
594
|
+
if (substitute) {
|
|
595
|
+
const retry = await judgeOneEnhanced(opts.apiKey, opts.question, opts.answer, substitute, opts.templateId, opts.styleOverride);
|
|
596
|
+
if (retry.ok) { finalResults[i] = retry; continue; }
|
|
597
|
+
excluded.push({ model: substitute, reason: retry.excludedReason ?? "unknown" });
|
|
598
|
+
}
|
|
599
|
+
excluded.push({ model: finalResults[i].model, reason: finalResults[i].excludedReason ?? "unknown" });
|
|
600
|
+
}
|
|
601
|
+
const valid = finalResults.filter(r => r.ok);
|
|
602
|
+
const judgments: EloJudgment[] = valid.map(r => ({ model: r.model, score: r.score, note: r.note }));
|
|
603
|
+
const latencyEach = Math.round((Date.now() - t0) / Math.max(1, models.length));
|
|
604
|
+
const roster = finalResults.map(r => {
|
|
605
|
+
const info = getModelEloInfo(r.model);
|
|
606
|
+
return { model: r.model, elo: info.elo, tier: info.tier, ok: r.ok, latencyMs: latencyEach, score: r.ok ? r.score : undefined };
|
|
607
|
+
});
|
|
608
|
+
return { judgments, roster, excluded };
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// ─── Independent Comparative Judge ─────────────────────────────────────────
|
|
612
|
+
export interface ComparativeJudgeResultV2 {
|
|
613
|
+
baselineScore: number; v15Score: number; gap: number;
|
|
614
|
+
winner: "baseline" | "v15" | "tie"; baselineImprovements: string[]; v15Improvements: string[];
|
|
615
|
+
rationale: string; judgeModel: string; ok: boolean; error?: string; rotationAttempts?: number;
|
|
616
|
+
}
|
|
617
|
+
export async function runComparativeJudge(opts: { apiKey: string; question: string; baselineAnswer: string; v15Answer: string; judgeModel?: string }): Promise<ComparativeJudgeResultV2> {
|
|
618
|
+
const pool = getActiveRotationPool();
|
|
619
|
+
let lastErr = "";
|
|
620
|
+
const tried = new Set<string>();
|
|
621
|
+
for (let attempt = 0; attempt < 4; attempt++) {
|
|
622
|
+
const remaining = pool.filter(m => !tried.has(m));
|
|
623
|
+
const model = (attempt === 0 && opts.judgeModel) ? opts.judgeModel : (pickLeastLoaded(remaining.length ? remaining : pool) ?? pool[0]);
|
|
624
|
+
tried.add(model);
|
|
625
|
+
const acquired = await tryAcquire(model, true);
|
|
626
|
+
if (!acquired) { lastErr = `${model}: rate-limited`; continue; }
|
|
627
|
+
const prompt = `You are an INDEPENDENT comparative judge. Score A (baseline) and B (V15) 0-10.\n\nMANDATORY CAPS (apply the LOWEST that fits per answer):\n- Truncated/fragment → cap 1\n- Formula-only/partial calc → cap 3\n- Restates question / requires reader to already know → cap 5\n- Missing required units/jurisdiction/scope → cap 6\n\nAnchors: reserve 9+ ONLY for expert-sign-off answers.\n\nReturn STRICT JSON only: {"baselineScore": <0-10>, "v15Score": <0-10>, "winner": "baseline"|"v15"|"tie", "rationale": "one paragraph explaining WHY the winner won", "baselineImprovements": ["<specific missing item>","..."], "v15Improvements": ["<specific missing item>","..."]}\n\nQUESTION:\n${opts.question}\n\nA (baseline):\n${opts.baselineAnswer.slice(0, 7000)}\n\nB (V15):\n${opts.v15Answer.slice(0, 7000)}`;
|
|
628
|
+
const res = await geminiGenerate({ apiKey: opts.apiKey, model, prompt, maxOutputTokens: 1600 });
|
|
629
|
+
recordResult(model, res.ok);
|
|
630
|
+
if (!res.ok) { lastErr = `${model}: ${res.error ?? "generation failed"}`; continue; }
|
|
631
|
+
try {
|
|
632
|
+
const m = res.text.replace(/```json\s*/gi, "").replace(/```/g, "").match(/\{[\s\S]*\}/);
|
|
633
|
+
const j = JSON.parse(m ? m[0] : res.text);
|
|
634
|
+
const b = Math.max(0, Math.min(10, Number(j.baselineScore) || 0));
|
|
635
|
+
const v = Math.max(0, Math.min(10, Number(j.v15Score) || 0));
|
|
636
|
+
return {
|
|
637
|
+
baselineScore: b, v15Score: v, gap: Math.round((v - b) * 100) / 100,
|
|
638
|
+
winner: (j.winner === "baseline" || j.winner === "v15" || j.winner === "tie") ? j.winner : (v > b ? "v15" : v < b ? "baseline" : "tie"),
|
|
639
|
+
baselineImprovements: Array.isArray(j.baselineImprovements) ? j.baselineImprovements.map(String).slice(0, 6) : [],
|
|
640
|
+
v15Improvements: Array.isArray(j.v15Improvements) ? j.v15Improvements.map(String).slice(0, 6) : [],
|
|
641
|
+
rationale: String(j.rationale ?? "").slice(0, 700),
|
|
642
|
+
judgeModel: model, ok: true, rotationAttempts: attempt + 1,
|
|
643
|
+
};
|
|
644
|
+
} catch { lastErr = `${model}: JSON parse failed (response length ${res.text.length} chars)`; }
|
|
645
|
+
}
|
|
646
|
+
return { baselineScore: 0, v15Score: 0, gap: 0, winner: "tie", baselineImprovements: [], v15Improvements: [], rationale: `Independent judge unavailable after ${tried.size} rotation attempt(s). Last error: ${lastErr}`, judgeModel: "rotation-exhausted", ok: false, error: lastErr, rotationAttempts: tried.size };
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// ─── Enhanced Baseline ─────────────────────────────────────────────────────
|
|
650
|
+
export async function runBaselineOnQuestion(opts: { apiKey: string; question: string; draftModel?: string; singleJudge?: boolean; judgeSampleSize?: number; onProgress?: (s: string) => void }): Promise<V15EnhancedOutcome> {
|
|
651
|
+
const t0 = Date.now();
|
|
652
|
+
const pool = getActiveRotationPool();
|
|
653
|
+
const draftModel = opts.draftModel ?? pickModel(pool);
|
|
654
|
+
opts.onProgress?.("drafting baseline (enhanced, 2400-token budget)");
|
|
655
|
+
const draftRes = await generateWithRotation({ apiKey: opts.apiKey, prompt: opts.question, preferredModel: draftModel, maxOutputTokens: 2400 });
|
|
656
|
+
if (!draftRes.ok) return { question: opts.question, draft: "", fixed: "", issues: [], autoFixesApplied: [], guardScore: 0, judgeScore: null, judgeNote: "", modelUsed: draftModel, passes: 1, stable: false, totalLatencyMs: Date.now() - t0, error: draftRes.error, passHistory: [], bestPassIndex: 0 };
|
|
657
|
+
opts.onProgress?.("judging baseline (enhanced panel, no 7.5 fallback)");
|
|
658
|
+
let judgeModels: string[] | undefined;
|
|
659
|
+
if (opts.singleJudge) judgeModels = [pool[0]];
|
|
660
|
+
else if (opts.judgeSampleSize && opts.judgeSampleSize > 0) judgeModels = pool.slice(0, Math.min(opts.judgeSampleSize, pool.length));
|
|
661
|
+
const { judgments, roster, excluded } = await judgePanelEnhanced({ apiKey: opts.apiKey, question: opts.question, answer: draftRes.text, judgeModels });
|
|
662
|
+
let judgeScore: number | null = null, judgeNote = "";
|
|
663
|
+
if (judgments.length > 0) { const elo = calculateEloConsensus(judgments); judgeScore = elo.weightedScore; judgeNote = elo.rationale; }
|
|
664
|
+
else judgeNote = `All judges excluded (${excluded.map(e => `${e.model}: ${e.reason}`).join("; ")})`;
|
|
665
|
+
opts.onProgress?.("done");
|
|
666
|
+
return { question: opts.question, draft: draftRes.text, fixed: draftRes.text, issues: [], autoFixesApplied: [], guardScore: 0, judgeScore, judgeNote, judgeRoster: roster, modelUsed: draftRes.modelUsed, passes: 1, stable: judgeScore !== null && judgeScore >= 9.0, totalLatencyMs: Date.now() - t0, passHistory: [], bestPassIndex: 0, judgeExcluded: excluded };
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// ─── The Enhanced Engine ───────────────────────────────────────────────────
|
|
670
|
+
export async function runV15OnQuestion(opts: {
|
|
671
|
+
apiKey: string;
|
|
672
|
+
question: string;
|
|
673
|
+
draftModel?: string;
|
|
674
|
+
maxDepth?: number;
|
|
675
|
+
singleJudge?: boolean;
|
|
676
|
+
judgeSampleSize?: number;
|
|
677
|
+
runJudge?: boolean;
|
|
678
|
+
advancedGates?: boolean;
|
|
679
|
+
profile?: EnhancedV15Profile;
|
|
680
|
+
onProgress?: (s: string) => void;
|
|
681
|
+
}): Promise<V15EnhancedOutcome> {
|
|
682
|
+
ensureFlawsLoaded();
|
|
683
|
+
const profile = opts.profile ?? {};
|
|
684
|
+
if (profile.useOriginalDefensePack) ensureOriginalDefensePackLoaded();
|
|
685
|
+
const t0 = Date.now();
|
|
686
|
+
const { apiKey, question, runJudge = true, onProgress } = opts;
|
|
687
|
+
const maxDepth = Math.max(1, Math.min(8, opts.maxDepth ?? 3));
|
|
688
|
+
const pool = getActiveRotationPool();
|
|
689
|
+
const draftModel = opts.draftModel ?? pickModel(pool);
|
|
690
|
+
|
|
691
|
+
// ── Reasoning preamble + directives ─────────────────────────────────────
|
|
692
|
+
const directives = [
|
|
693
|
+
"You are an elite, highly calibrated domain expert.",
|
|
694
|
+
"Silently, before writing, sketch the 3-5 structural components a 9.9-quality answer must have (definitions, worked examples, quantitative evidence, jurisdiction/scope, caveats, references). Then write the answer covering ALL of them.",
|
|
695
|
+
"Answer directly and comprehensively so the reader can act on it without further research.",
|
|
696
|
+
"Always state assumptions, define variables, give worked numbers with units, and add jurisdiction/scope caveats when relevant.",
|
|
697
|
+
"If you use [S#] citations, ALWAYS include a References section at the end listing each citation. Never fabricate a URL — write 'source not directly available' if unknown.",
|
|
698
|
+
"Never emit fragments. Always produce a COMPLETE answer with proper terminal punctuation on the final sentence.",
|
|
699
|
+
"Never leak internal scaffolding: JUDGE_NOTE, DRAFT TO REVISE, TESTBED_, Recursive Refinement Pass, or Meticulous Technical Editor.",
|
|
700
|
+
];
|
|
701
|
+
const personaDirective = getPersonaDirective(profile.williamsPersona);
|
|
702
|
+
if (personaDirective) directives.push(personaDirective);
|
|
703
|
+
if (profile.fourStage) directives.push("Use the 4-Stage micro-agent framing internally (Plan → Draft → Critique → Polish), emit only the polished final answer.");
|
|
704
|
+
if (profile.nDeep) directives.push(`Use N-Deep adversarial refinement across ${maxDepth} internal passes to catch missed edge cases.`);
|
|
705
|
+
if (profile.cluster) directives.push(`Cluster-synthesize across ${Math.max(1, Math.min(16, profile.clusterSize ?? 8))} parallel hypotheses.`);
|
|
706
|
+
if (profile.sloop) directives.push(`SLOOP long-form report mode: target ~${Math.max(1, Math.min(32, profile.sloopPages ?? 4))} pages with substantive prose in every section.`);
|
|
707
|
+
// Use the real OMEGA template skeleton — not a vague sentence hint
|
|
708
|
+
if (profile.templateId) {
|
|
709
|
+
const resolvedTemplate = OMEGA_TEMPLATES.find(t => t.id === profile.templateId);
|
|
710
|
+
if (resolvedTemplate) {
|
|
711
|
+
directives.push(buildTemplatePrompt(resolvedTemplate, profile.styleOverride ?? "default"));
|
|
712
|
+
} else {
|
|
713
|
+
directives.push(`Follow the "${profile.templateId}" template's section structure exactly.`);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
if (profile.styleOverride) directives.push(`Apply the "${profile.styleOverride}" style-override modulation silently.`);
|
|
717
|
+
if (profile.webSearch) directives.push("If your underlying model has native web-search / browsing tool capability, use it to cross-verify any claim you are uncertain about before finalizing your answer.");
|
|
718
|
+
|
|
719
|
+
// Hand-trace appendix: require step-by-step derivation for all quantitative/logical claims
|
|
720
|
+
const handTraceBlock = buildHandTraceInstruction(profile.templateId);
|
|
721
|
+
if (handTraceBlock) directives.push(handTraceBlock);
|
|
722
|
+
|
|
723
|
+
// ── Template-directed grounding (with CitationLedger for provenance tracking) ──
|
|
724
|
+
const citationLedger = new CitationLedger();
|
|
725
|
+
let evidenceBlock = "";
|
|
726
|
+
let groundingProvider: string | undefined;
|
|
727
|
+
let groundingCount = 0;
|
|
728
|
+
if (profile.webSearch) {
|
|
729
|
+
const backends = profile.webBackends ?? { ogScraper: true };
|
|
730
|
+
|
|
731
|
+
// Use template-directed search queries instead of a single heuristic query
|
|
732
|
+
if (profile.templateId) {
|
|
733
|
+
const sectionQueries = buildTemplateSearchQueries(profile.templateId, question);
|
|
734
|
+
onProgress?.(`template-directed grounding: ${sectionQueries.length} section(s) × ${sectionQueries.reduce((a, s) => a + s.queries.length, 0)} targeted queries`);
|
|
735
|
+
|
|
736
|
+
// Execute the most important section queries IN PARALLEL (max 4 concurrent to avoid timeout)
|
|
737
|
+
// Previously ran sequentially — caused 40+ second timeouts on 8 queries
|
|
738
|
+
const allQueries = sectionQueries.flatMap(s => s.queries.map(q => ({ section: s.section, query: q })));
|
|
739
|
+
const cappedQueries = allQueries.slice(0, 6); // max 6 targeted queries (reduced from 8)
|
|
740
|
+
|
|
741
|
+
// Run in parallel batches of 3 — balances speed vs RPM pressure
|
|
742
|
+
for (let batch = 0; batch < cappedQueries.length; batch += 3) {
|
|
743
|
+
const batchQueries = cappedQueries.slice(batch, batch + 3);
|
|
744
|
+
await Promise.all(batchQueries.map(async ({ section, query }) => {
|
|
745
|
+
try {
|
|
746
|
+
const grounded = await groundQuestion({ question: query, backends, depth: 3, onDebug: m => onProgress?.(`grounding [${section}] · ${m}`) });
|
|
747
|
+
if (grounded.ok && grounded.sources.length > 0) {
|
|
748
|
+
citationLedger.addSources(grounded.sources.slice(0, 2), "initial");
|
|
749
|
+
onProgress?.(`grounded [${section}]: +${grounded.sources.slice(0, 2).length} source(s) for "${query.slice(0, 50)}…"`);
|
|
750
|
+
if (!groundingProvider) groundingProvider = grounded.provider;
|
|
751
|
+
return true;
|
|
752
|
+
}
|
|
753
|
+
} catch {}
|
|
754
|
+
return false;
|
|
755
|
+
}));
|
|
756
|
+
// If we already have enough sources, stop early
|
|
757
|
+
if (citationLedger.count >= 8) break;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
if (citationLedger.count > 0) {
|
|
761
|
+
evidenceBlock = citationLedger.buildEvidenceBlock(groundingProvider ?? "template-directed");
|
|
762
|
+
groundingCount = citationLedger.count;
|
|
763
|
+
onProgress?.(`template-directed grounding complete: ${groundingCount} total sources across ${cappedQueries.length} queries`);
|
|
764
|
+
} else {
|
|
765
|
+
groundingProvider = "template-directed: no results";
|
|
766
|
+
onProgress?.("template-directed grounding returned 0 sources — proceeding ungrounded");
|
|
767
|
+
}
|
|
768
|
+
} else {
|
|
769
|
+
// Fallback: single query (non-template mode)
|
|
770
|
+
onProgress?.("web grounding (single query)");
|
|
771
|
+
const grounded = await groundQuestion({ question, backends, onDebug: m => onProgress?.(`grounding · ${m}`) });
|
|
772
|
+
if (grounded.ok) {
|
|
773
|
+
citationLedger.addSources(grounded.sources, "initial");
|
|
774
|
+
evidenceBlock = citationLedger.buildEvidenceBlock(grounded.provider);
|
|
775
|
+
groundingProvider = grounded.provider;
|
|
776
|
+
groundingCount = citationLedger.count;
|
|
777
|
+
onProgress?.(`grounded via ${grounded.provider} · ${groundingCount} sources (ledger-tracked)`);
|
|
778
|
+
} else {
|
|
779
|
+
groundingProvider = `unavailable: ${grounded.error}`;
|
|
780
|
+
onProgress?.(`grounding unavailable (${grounded.error}) — proceeding ungrounded`);
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
const templateContract = buildAdaptiveTemplateContract({
|
|
786
|
+
templateId: profile.templateId,
|
|
787
|
+
styleOverride: profile.styleOverride,
|
|
788
|
+
targetPages: profile.sloopPages ?? 4,
|
|
789
|
+
evidenceAvailable: groundingCount > 0,
|
|
790
|
+
});
|
|
791
|
+
if (templateContract) directives.push(templateContract);
|
|
792
|
+
|
|
793
|
+
// ── Draft phase (Best-of-N when cluster is enabled) ──────────────────────
|
|
794
|
+
const draftMaxToks = profile.sloop ? Math.max(4500, (profile.sloopPages ?? 4) * 900) : 4000;
|
|
795
|
+
const longForm = !!profile.sloop, targetPages = profile.sloopPages ?? 4;
|
|
796
|
+
let currentText = "", modelUsedForDraft = draftModel, bestOfNCandidates: BestOfNCandidate[] | undefined;
|
|
797
|
+
if (profile.cluster && (profile.clusterSize ?? 0) >= 2) {
|
|
798
|
+
const best = await runBestOfNDraft({
|
|
799
|
+
apiKey, question, evidenceBlock, systemInstruction: directives.join("\n"),
|
|
800
|
+
hypothesisCount: profile.bestOfNHypotheses ?? Math.max(2, Math.min(5, profile.clusterSize ?? 3)),
|
|
801
|
+
modelCount: profile.bestOfNModels ?? Math.max(1, Math.min(3, profile.clusterSize ?? 3)),
|
|
802
|
+
packedPerModel: profile.bestOfNPackHypotheses ?? false,
|
|
803
|
+
maxOutputTokens: draftMaxToks, longForm, targetPages,
|
|
804
|
+
templateId: profile.templateId, onProgress,
|
|
805
|
+
});
|
|
806
|
+
if (!best.text) return { question, draft: "", fixed: "", issues: [], autoFixesApplied: [], guardScore: 0, judgeScore: null, judgeNote: "", modelUsed: draftModel, passes: 0, stable: false, totalLatencyMs: Date.now() - t0, error: "best-of-N failed to produce any usable candidate", passHistory: [], bestPassIndex: 0 };
|
|
807
|
+
currentText = best.text;
|
|
808
|
+
modelUsedForDraft = best.modelUsed;
|
|
809
|
+
bestOfNCandidates = best.candidates;
|
|
810
|
+
} else {
|
|
811
|
+
onProgress?.(`drafting (enhanced, ${draftMaxToks}-token budget — real, honored)`);
|
|
812
|
+
const draftRes = await generateWithRotation({ apiKey, prompt: evidenceBlock ? `${evidenceBlock}\n\nUSER QUESTION:\n${question}` : question, preferredModel: draftModel, systemInstruction: directives.join("\n"), maxOutputTokens: draftMaxToks });
|
|
813
|
+
if (!draftRes.ok) return { question, draft: "", fixed: "", issues: [], autoFixesApplied: [], guardScore: 0, judgeScore: null, judgeNote: "", modelUsed: draftRes.modelUsed, passes: 0, stable: false, totalLatencyMs: Date.now() - t0, error: draftRes.error, passHistory: [], bestPassIndex: 0 };
|
|
814
|
+
currentText = draftRes.text;
|
|
815
|
+
modelUsedForDraft = draftRes.modelUsed;
|
|
816
|
+
}
|
|
817
|
+
const originalDraft = currentText;
|
|
818
|
+
|
|
819
|
+
// ── HDIG grounding-driven verification ───────────────────────────────────
|
|
820
|
+
if (profile.webSearch && groundingCount > 0) {
|
|
821
|
+
onProgress?.("hypothesis-driven iterative grounding (HDIG)");
|
|
822
|
+
try {
|
|
823
|
+
const gapRes = await generateWithRotation({ apiKey, prompt: `Draft:\n${currentText.slice(0,3000)}\n\nIdentify up to 3 factual claims needing web verification. Return JSON: {"gaps":[{"claim":"...","searchQuery":"..."}]}`, maxOutputTokens: 400 });
|
|
824
|
+
const m = gapRes.text.match(/\{[\s\S]*\}/);
|
|
825
|
+
const j = JSON.parse(m ? m[0] : gapRes.text);
|
|
826
|
+
const gaps = Array.isArray(j.gaps) ? j.gaps.slice(0, 3) : [];
|
|
827
|
+
for (const gap of gaps) {
|
|
828
|
+
const hit = await groundQuestion({ question: gap.searchQuery, backends: profile.webBackends ?? { ogScraper: true }, depth: 3, onDebug: m2 => onProgress?.(`HDIG · ${m2}`) });
|
|
829
|
+
if (hit.ok && hit.sources.length) {
|
|
830
|
+
citationLedger.addSources(hit.sources.slice(0, 2), "hdig");
|
|
831
|
+
evidenceBlock = citationLedger.buildEvidenceBlock(groundingProvider ?? "hdig");
|
|
832
|
+
groundingCount = citationLedger.count;
|
|
833
|
+
onProgress?.(`HDIG: +${hit.sources.length} source(s) for "${String(gap.claim).slice(0, 40)}…" (ledger now ${groundingCount})`);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
} catch { onProgress?.("HDIG: gap analysis unavailable — continuing with existing evidence"); }
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
// Helper for mid-pipeline re-grounding
|
|
840
|
+
async function performReGrounding(textToAnalyze: string, stageName: string) {
|
|
841
|
+
if (!profile.webSearch) return;
|
|
842
|
+
try {
|
|
843
|
+
const gapRes = await generateWithRotation({ apiKey, prompt: `Draft:\n${textToAnalyze.slice(0,4000)}\n\nIdentify 1-2 newly added factual claims that are NOT backed by the current citations and need web verification. Return JSON: {"gaps":[{"claim":"...","searchQuery":"..."}]}`, maxOutputTokens: 300 });
|
|
844
|
+
const m = gapRes.text.match(/\{[\s\S]*\}/);
|
|
845
|
+
const j = JSON.parse(m ? m[0] : gapRes.text);
|
|
846
|
+
const gaps = Array.isArray(j.gaps) ? j.gaps.slice(0, 2) : [];
|
|
847
|
+
for (const gap of gaps) {
|
|
848
|
+
const hit = await groundQuestion({ question: gap.searchQuery, backends: profile.webBackends ?? { ogScraper: true }, depth: 2, onDebug: m2 => onProgress?.(`Re-ground (${stageName}) · ${m2}`) });
|
|
849
|
+
if (hit.ok && hit.sources.length) {
|
|
850
|
+
citationLedger.addSources(hit.sources.slice(0, 2), "n-deep");
|
|
851
|
+
evidenceBlock = citationLedger.buildEvidenceBlock(groundingProvider ?? "n-deep");
|
|
852
|
+
groundingCount = citationLedger.count;
|
|
853
|
+
onProgress?.(`Re-ground (${stageName}): +${hit.sources.length} source(s) for "${String(gap.claim).slice(0, 40)}…" (ledger now ${groundingCount})`);
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
} catch { /* skip */ }
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// ── CoVe ────────────────────────────────────────────────────────────────
|
|
860
|
+
let coveReport: CoVeReport | undefined;
|
|
861
|
+
let coveInjection = "";
|
|
862
|
+
if (profile.webSearch) {
|
|
863
|
+
coveReport = await runCoVeVerification({ apiKey, question, draft: currentText, evidenceBlock, onProgress });
|
|
864
|
+
if (coveReport.inconsistencies > 0) {
|
|
865
|
+
coveInjection = "\n\nMANDATORY CONSTRAINTS — COVE MISMATCHES [COVE_MISMATCH] (these factual errors MUST be corrected before other edits):\n" + coveReport.questions
|
|
866
|
+
.filter(v => !v.consistent)
|
|
867
|
+
.map(v => `[COVE_MISMATCH] Claim "${v.expectedAnswer.slice(0, 100)}" — verified answer: "${v.verifiedAnswer.slice(0, 100)}". Fix or remove. DO NOT preserve this claim verbatim.`)
|
|
868
|
+
.join("\n");
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
// ── Adversarial PREFLIGHT (before N-Deep, so defects are constraints in editor) ──
|
|
873
|
+
// Run a lightweight adversarial pre-scan to generate [ADV_DEFECT] constraints
|
|
874
|
+
// that become MANDATORY fixes in every subsequent N-Deep editor pass.
|
|
875
|
+
let advPreflightConstraints = "";
|
|
876
|
+
if (profile.adversarial) {
|
|
877
|
+
try {
|
|
878
|
+
onProgress?.("adversarial preflight (pre-N-Deep constraint generation)");
|
|
879
|
+
const preAdv = await runAdversarialRedTeam(currentText, question, { provider: "gemini", model: modelUsedForDraft, apiKey, userMessage: question, conversationHistory: [] } as any, { onDebug: m => onProgress?.(`adv-preflight · ${m}`) });
|
|
880
|
+
const preBlocking = (preAdv.defects ?? []).filter((d: any) => d.severity === "critical" || d.severity === "major");
|
|
881
|
+
if (preBlocking.length > 0) {
|
|
882
|
+
const _categories = [...new Set(preBlocking.map((d: any) => String(d.category || "")))];
|
|
883
|
+
void _categories; // surfaced for future use in UI preview
|
|
884
|
+
advPreflightConstraints = "\n\nMANDATORY CONSTRAINTS — ADVERSARIAL DEFECTS [ADV_DEFECT] (fix BEFORE other edits):\n" +
|
|
885
|
+
preBlocking.slice(0, 6).map((d: any, k: number) => `[ADV_DEFECT ${k + 1}] [${String(d.severity).toUpperCase()}·${d.category}] ${d.detail}`).join("\n");
|
|
886
|
+
onProgress?.(`adversarial preflight: ${preBlocking.length} blocking defect(s) → injected as N-Deep mandatory constraints`);
|
|
887
|
+
} else {
|
|
888
|
+
onProgress?.("adversarial preflight: no critical/major defects — proceeding to N-Deep");
|
|
889
|
+
}
|
|
890
|
+
} catch (e: any) { onProgress?.(`adversarial preflight unavailable: ${e?.message ?? "error"}`); }
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// ── N-Deep loop with monotonic best-pass tracking ────────────────────────
|
|
894
|
+
const allFixes: string[] = [];
|
|
895
|
+
const passHistory: PassDraftStats[] = [];
|
|
896
|
+
const testbedGatesProposed: TestbedGate[] = [];
|
|
897
|
+
let bestScore = -1;
|
|
898
|
+
let bestText = currentText;
|
|
899
|
+
let bestIssuesFlat: { code: string; severity: string; message: string; remediation?: string }[] = [];
|
|
900
|
+
let bestIdx = -1;
|
|
901
|
+
for (let d = 1; d <= maxDepth; d++) {
|
|
902
|
+
const ctx: ScanContext = { prompt: question, answer: currentText, lowerAnswer: currentText.toLowerCase(), computeRecords: [], constraints: { explicitComparisonTargets: [], exclusions: [], formatHints: [], namedEntities: [] } as any };
|
|
903
|
+
const af = runAutoFix(currentText, ctx);
|
|
904
|
+
currentText = af.text;
|
|
905
|
+
if (af.applied.length) allFixes.push(...af.applied);
|
|
906
|
+
const scanResult = scoreCandidate(question, currentText, longForm, targetPages);
|
|
907
|
+
const stats = computeDraftStats(currentText, scanResult.canonical, scanResult.testbed, d, modelUsedForDraft, scanResult.guardScore);
|
|
908
|
+
passHistory.push(stats);
|
|
909
|
+
if (scanResult.guardScore > bestScore) {
|
|
910
|
+
bestScore = scanResult.guardScore;
|
|
911
|
+
bestText = currentText;
|
|
912
|
+
bestIdx = passHistory.length - 1;
|
|
913
|
+
bestIssuesFlat = [
|
|
914
|
+
...scanResult.canonical.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
915
|
+
...scanResult.testbed.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
916
|
+
];
|
|
917
|
+
}
|
|
918
|
+
onProgress?.(`depth ${d}: guard ${scanResult.guardScore.toFixed(2)} (best-so-far ${bestScore.toFixed(2)} @ pass ${bestIdx + 1}) · ${scanResult.canonical.length} canonical + ${scanResult.testbed.length} testbed`);
|
|
919
|
+
if (opts.advancedGates && d < maxDepth) {
|
|
920
|
+
const proposed = await proposeGateWithLLM({ apiKey, question, answer: currentText, judgeNote: scanResult.canonical.slice(0, 5).map(i => i.code).join(", "), model: "gemini-3.1-flash-lite" });
|
|
921
|
+
if (proposed && !testbedGatesProposed.some(g => g.code === proposed.code)) testbedGatesProposed.push(proposed);
|
|
922
|
+
}
|
|
923
|
+
if (maxDepth <= 2 && !scanResult.truncated && scanResult.guardScore >= 9.2 && !scanResult.canonical.some(i => i.severity === "critical" || i.severity === "major")) break;
|
|
924
|
+
if (d < maxDepth) {
|
|
925
|
+
const ordered = [...scanResult.canonical, ...scanResult.testbed].sort((a, b) => sevRank(b.severity) - sevRank(a.severity)).slice(0, 40);
|
|
926
|
+
const issuesBlock = ordered.map(i => `- [${i.code}] (${i.severity}): ${i.remediation || i.message}`).join("\n");
|
|
927
|
+
const testbedBlock = testbedGatesProposed.length > 0 ? `\n\nCUTTING-EDGE TESTBED GATES (also satisfy these newly-discovered patterns):\n${testbedGatesProposed.map(g => `- [${g.code}] ${g.message} → ${g.remediation}`).join("\n")}` : "";
|
|
928
|
+
// TURN 12 FIX: rule 1 previously told the editor to preserve EVERY citation
|
|
929
|
+
// tag verbatim, which directly contradicted fixing citation-integrity
|
|
930
|
+
// defects (HALLUCINATED_CITATION, CITES_WITH_NO_SOURCES, HAL_CITE_REF_VOID,
|
|
931
|
+
// MISSING_CITATION_REF) — a confirmed cause of the SAME critical/major
|
|
932
|
+
// count persisting unchanged across multiple N-Deep passes. Rule 1 now
|
|
933
|
+
// explicitly carves out the exception, and a concrete valid-citation-ID
|
|
934
|
+
// range is enforced when grounding evidence is available.
|
|
935
|
+
const validCiteRange = groundingCount > 0 ? `\n 0. VALID CITATION IDS: only [S1] through [S${groundingCount}] are backed by real evidence. Remove or replace ANY citation tag outside this range — do not preserve fabricated or out-of-range citation tags.` : "";
|
|
936
|
+
// Determine if this is a near-complete answer (few flaws, late pass) → use STRICT LOCALIZED mode
|
|
937
|
+
const isDeepRefinement = d >= 2 && !scanResult.canonical.some(i => i.severity === "critical");
|
|
938
|
+
const strictModeNote = isDeepRefinement
|
|
939
|
+
? "\n\nLOCALIZED REVISION STRICT MODE (active: answer is near-complete): COPY ALL UNAFFECTED SECTIONS EXACTLY VERBATIM. DO NOT REWRITE OR ALTER CORRECT SECTIONS. Change ONLY the specific sentences or paragraphs where a listed defect is found. This is mandatory — unaffected sections must not be paraphrased, shortened, or reorganized."
|
|
940
|
+
: "";
|
|
941
|
+
// Build template context for the editor so it knows the required sections and hand-trace rules
|
|
942
|
+
const editorTemplateContext = templateContract ? `\nTEMPLATE CONTRACT (editor must respect this structure):\n${templateContract}\n` : "";
|
|
943
|
+
const editorHandTrace = handTraceBlock ? `\n${handTraceBlock}\nWhen patching a section that contains quantitative claims, ensure the hand-trace entry for each claim is updated or added.\n` : "";
|
|
944
|
+
|
|
945
|
+
const editorPrompt = `You are a LOCALIZED technical editor performing refinement pass ${d + 1}/${maxDepth}. You are DENIED permission to rewrite the whole report. Patch ONLY sections or paragraphs that directly contain a listed defect. Every unmentioned character of the draft will be preserved byte-for-byte by the controller.${validCiteRange}${strictModeNote}
|
|
946
|
+
1. Preserve all correct sections, facts, and headings. Correct/remove only citations explicitly unsupported or out-of-range.
|
|
947
|
+
2. Expand only under-specified problem sections with evidence-backed specifics; never invent dates, quantities, entities, interviews, or results.
|
|
948
|
+
3. Fix every critical issue before any major issue; fix major before warning.
|
|
949
|
+
4. A replacement must include its original Markdown heading (when present) plus the complete replacement body, and must end cleanly.
|
|
950
|
+
5. Emit at most 5 patch blocks and NOTHING outside them.
|
|
951
|
+
6. Every quantitative claim in a patched section MUST have a corresponding hand-trace entry in the Appendix.
|
|
952
|
+
EXACT PATCH FORMAT:
|
|
953
|
+
<<<REVISE_SECTION>>>
|
|
954
|
+
ANCHOR: <exact existing Markdown heading line, or exact first sentence of the problem paragraph>
|
|
955
|
+
REVISED:
|
|
956
|
+
<complete replacement section/paragraph; include heading if the anchor is a heading>
|
|
957
|
+
<<<END_SECTION>>>${editorTemplateContext}${editorHandTrace}
|
|
958
|
+
${advPreflightConstraints}${coveInjection ? coveInjection : ""}
|
|
959
|
+
DETECTED FLAWS (ordered by severity, fix all):
|
|
960
|
+
${issuesBlock || "Enhance clarity, completeness, and rigor."}${testbedBlock}
|
|
961
|
+
${evidenceBlock ? `\nEVIDENCE:\n${evidenceBlock}\n` : ""}
|
|
962
|
+
USER PROMPT:
|
|
963
|
+
${question}
|
|
964
|
+
DRAFT TO REVISE:
|
|
965
|
+
${currentText}`;
|
|
966
|
+
const editMaxToks = profile.sloop ? Math.max(4500, (profile.sloopPages ?? 4) * 900) : 4000;
|
|
967
|
+
const editRes = await generateWithRotation({ apiKey, prompt: editorPrompt, preferredModel: pickModel(pool), maxOutputTokens: editMaxToks });
|
|
968
|
+
if (editRes.ok && editRes.text.trim().length > 50) {
|
|
969
|
+
const patched = applySectionPatches(currentText, editRes.text);
|
|
970
|
+
if (!patched.applied) {
|
|
971
|
+
onProgress?.(`depth ${d}: editor returned no anchorable section patches — stopping to avoid verbatim duplicate passes`);
|
|
972
|
+
break;
|
|
973
|
+
}
|
|
974
|
+
const candScan = scoreCandidate(question, patched.text, longForm, targetPages);
|
|
975
|
+
const before = issueVector([...scanResult.canonical, ...scanResult.testbed]);
|
|
976
|
+
const after = issueVector([...candScan.canonical, ...candScan.testbed]);
|
|
977
|
+
const severityImproved = after.critical < before.critical ||
|
|
978
|
+
(after.critical === before.critical && after.major < before.major) ||
|
|
979
|
+
(after.critical === before.critical && after.major === before.major && after.warning < before.warning);
|
|
980
|
+
const scoreImproved = candScan.guardScore > scanResult.guardScore + 0.01;
|
|
981
|
+
if (severityImproved || scoreImproved) {
|
|
982
|
+
currentText = patched.text;
|
|
983
|
+
onProgress?.(`depth ${d}: accepted ${patched.applied} localized patch(es) · guard ${scanResult.guardScore.toFixed(2)}→${candScan.guardScore.toFixed(2)} · crit/major ${before.critical}/${before.major}→${after.critical}/${after.major}`);
|
|
984
|
+
|
|
985
|
+
// Mid-pipeline re-grounding: check if the new patches added claims that need sources
|
|
986
|
+
await performReGrounding(patched.text, `pass ${d}`);
|
|
987
|
+
} else {
|
|
988
|
+
onProgress?.(`depth ${d}: rejected localized patches (no score or severity improvement) — stopping instead of rescanning an unchanged draft`);
|
|
989
|
+
break;
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
currentText = bestText;
|
|
995
|
+
if (passHistory[bestIdx]) passHistory[bestIdx].isBest = true;
|
|
996
|
+
|
|
997
|
+
// ── Adversarial red-team (moved BEFORE polish/judge, WITH repair-on-blocking) ──
|
|
998
|
+
// TURN 12 FIX (confirmed regression): the Turn-11 engine captured adversarial
|
|
999
|
+
// defects into `adversarialPreview` for display ONLY — they were never merged
|
|
1000
|
+
// into `finalIssues`/`guardScore`, and adversarial ran AFTER the judge panel,
|
|
1001
|
+
// meaning the judge scored PRE-adversarial text while the UI displayed
|
|
1002
|
+
// POST-adversarial critique. This made red-team findings cosmetic and the
|
|
1003
|
+
// judge score inconsistent with the actual final answer. Fixed: adversarial
|
|
1004
|
+
// now runs BEFORE polish/judge; any critical/major defects trigger ONE
|
|
1005
|
+
// monotonic repair pass (never regresses — rejected unless it scores >= the
|
|
1006
|
+
// pre-repair guard score minus a small tolerance), and the repaired text
|
|
1007
|
+
// becomes the input to polish and judging so all downstream scores are
|
|
1008
|
+
// consistent with what actually ships.
|
|
1009
|
+
let adversarialPreview: AdversarialPreview | undefined;
|
|
1010
|
+
if (profile.adversarial) {
|
|
1011
|
+
try {
|
|
1012
|
+
onProgress?.("adversarial red-team");
|
|
1013
|
+
const adv = await runAdversarialRedTeam(currentText, question, { provider: "gemini", model: modelUsedForDraft, apiKey, userMessage: question, conversationHistory: [] } as any, { onDebug: m => onProgress?.(`adversarial · ${m}`) });
|
|
1014
|
+
adversarialPreview = { rawCritique: adv.rawCritique || "(no critique text returned — structural gates only)", defectCount: adv.defects.length, verdict: adv.verdict, categories: [...new Set(adv.defects.map(d => d.category))] };
|
|
1015
|
+
onProgress?.(`adversarial · captured ${adv.defects.length} defect(s), verdict: ${adv.verdict}`);
|
|
1016
|
+
const blocking = (adv.defects ?? []).filter((d: any) => d.severity === "critical" || d.severity === "major");
|
|
1017
|
+
if (blocking.length > 0) {
|
|
1018
|
+
onProgress?.(`adversarial: ${blocking.length} blocking defect(s) found — issuing monotonic repair pass`);
|
|
1019
|
+
const advIssuesForFlat = blocking.slice(0, 8).map((d: any) => ({ code: `ADV_${String(d.category ?? d.id ?? "DEFECT").toUpperCase().replace(/\s+/g, "_")}`, severity: d.severity as string, message: String(d.detail ?? "adversarial defect") }));
|
|
1020
|
+
const preScore = bestScore;
|
|
1021
|
+
const repairPrompt = `A hostile expert reviewer found the following defects in your draft. Rewrite the draft to fix EVERY one while preserving all correct content, citations backed by real evidence, and structure. Do NOT acknowledge this review in the output.${templateContract ? `\n\nTEMPLATE CONTRACT (must be respected in repair):\n${templateContract}` : ""}${handTraceBlock ? `\n${handTraceBlock}` : ""}\n\nDEFECTS TO FIX:\n${blocking.slice(0, 8).map((d: any, i: number) => `${i + 1}. [${String(d.severity).toUpperCase()} · ${d.category}] ${d.detail}`).join("\n")}\n${evidenceBlock ? `\nEVIDENCE:\n${evidenceBlock}\n` : ""}\nUSER PROMPT:\n${question}\n\nDRAFT TO REPAIR:\n${currentText}`;
|
|
1022
|
+
const repairMaxToks = profile.sloop ? Math.max(4500, (profile.sloopPages ?? 4) * 900) : 4000;
|
|
1023
|
+
const repairRes = await generateWithRotation({ apiKey, prompt: repairPrompt, preferredModel: pickModel(pool), maxOutputTokens: repairMaxToks });
|
|
1024
|
+
if (repairRes.ok && repairRes.text.trim().length >= currentText.length * 0.6) {
|
|
1025
|
+
const repairedScan = scoreCandidate(question, repairRes.text.trim(), longForm, targetPages);
|
|
1026
|
+
if (repairedScan.guardScore >= preScore - 0.5) {
|
|
1027
|
+
currentText = repairRes.text.trim();
|
|
1028
|
+
bestText = currentText;
|
|
1029
|
+
bestScore = repairedScan.guardScore;
|
|
1030
|
+
bestIssuesFlat = [
|
|
1031
|
+
...repairedScan.canonical.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
1032
|
+
...repairedScan.testbed.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
1033
|
+
...advIssuesForFlat,
|
|
1034
|
+
];
|
|
1035
|
+
passHistory.forEach(p => p.isBest = false);
|
|
1036
|
+
const advStats = computeDraftStats(currentText, repairedScan.canonical, repairedScan.testbed, passHistory.length + 1, repairRes.modelUsed, repairedScan.guardScore);
|
|
1037
|
+
advStats.isBest = true;
|
|
1038
|
+
passHistory.push(advStats);
|
|
1039
|
+
bestIdx = passHistory.length - 1;
|
|
1040
|
+
onProgress?.(`adversarial repair: accepted (${preScore.toFixed(2)}→${repairedScan.guardScore.toFixed(2)})`);
|
|
1041
|
+
|
|
1042
|
+
// Re-ground if adversarial repair added significant new claims
|
|
1043
|
+
await performReGrounding(currentText, "adv-repair");
|
|
1044
|
+
} else {
|
|
1045
|
+
bestIssuesFlat = [...bestIssuesFlat, ...advIssuesForFlat];
|
|
1046
|
+
onProgress?.(`adversarial repair: rejected (would drop score ${preScore.toFixed(2)}→${repairedScan.guardScore.toFixed(2)}) — keeping pre-repair text, defects logged`);
|
|
1047
|
+
}
|
|
1048
|
+
} else {
|
|
1049
|
+
bestIssuesFlat = [...bestIssuesFlat, ...advIssuesForFlat];
|
|
1050
|
+
onProgress?.("adversarial repair: generation failed or shrank too much — keeping pre-repair text, defects logged");
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
} catch (e: any) { onProgress?.(`adversarial engine unavailable: ${e?.message ?? "error"}`); }
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
// ── Polish pass (monotonic — only accepted if it improves or maintains score) ──
|
|
1057
|
+
let polishApplied = false;
|
|
1058
|
+
let guardScore = bestScore;
|
|
1059
|
+
const polishResult = await runPolishPass({ apiKey, question, text: currentText, guardScore: bestScore, onProgress });
|
|
1060
|
+
if (polishResult.applied) {
|
|
1061
|
+
const polishedScan = scoreCandidate(question, polishResult.text, longForm, targetPages);
|
|
1062
|
+
if (polishedScan.guardScore >= bestScore - 0.1) {
|
|
1063
|
+
// Accept: polish improved or maintained score (within noise margin)
|
|
1064
|
+
currentText = polishResult.text;
|
|
1065
|
+
polishApplied = true;
|
|
1066
|
+
if (polishedScan.guardScore > bestScore) {
|
|
1067
|
+
bestScore = polishedScan.guardScore;
|
|
1068
|
+
guardScore = bestScore;
|
|
1069
|
+
bestIssuesFlat = [
|
|
1070
|
+
...polishedScan.canonical.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
1071
|
+
...polishedScan.testbed.map(i => ({ code: i.code, severity: i.severity, message: i.message, remediation: i.remediation })),
|
|
1072
|
+
];
|
|
1073
|
+
}
|
|
1074
|
+
passHistory.forEach(p => p.isBest = false);
|
|
1075
|
+
const polishStats = computeDraftStats(currentText, polishedScan.canonical, polishedScan.testbed, passHistory.length + 1, "polish-pass", polishedScan.guardScore);
|
|
1076
|
+
polishStats.isBest = true;
|
|
1077
|
+
passHistory.push(polishStats);
|
|
1078
|
+
bestIdx = passHistory.length - 1;
|
|
1079
|
+
} else {
|
|
1080
|
+
onProgress?.(`polish: rejected (would drop score ${bestScore.toFixed(2)}→${polishedScan.guardScore.toFixed(2)})`);
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// ── Judge panel (scores the TRUE final text — after adversarial repair + polish) ──
|
|
1085
|
+
let judgeScore: number | null = null;
|
|
1086
|
+
let judgeNote = "";
|
|
1087
|
+
let judgeRoster: V15RunOutcome["judgeRoster"] = [];
|
|
1088
|
+
let judgeExcluded: { model: string; reason: string }[] = [];
|
|
1089
|
+
let eloConsensus: ReturnType<typeof calculateEloConsensus> | undefined;
|
|
1090
|
+
if (runJudge) {
|
|
1091
|
+
let judgeModels: string[] | undefined;
|
|
1092
|
+
if (opts.singleJudge) judgeModels = [pool[0]];
|
|
1093
|
+
else if (opts.judgeSampleSize && opts.judgeSampleSize > 0) judgeModels = pool.slice(0, Math.min(opts.judgeSampleSize, pool.length));
|
|
1094
|
+
const panel = await judgePanelEnhanced({ apiKey, question, answer: currentText, judgeModels, templateId: profile.templateId, styleOverride: profile.styleOverride });
|
|
1095
|
+
judgeRoster = panel.roster; judgeExcluded = panel.excluded;
|
|
1096
|
+
if (panel.judgments.length) {
|
|
1097
|
+
eloConsensus = calculateEloConsensus(panel.judgments);
|
|
1098
|
+
judgeScore = eloConsensus.weightedScore;
|
|
1099
|
+
judgeNote = eloConsensus.rationale;
|
|
1100
|
+
} else {
|
|
1101
|
+
judgeNote = `All judges excluded (${panel.excluded.map(e => `${e.model}: ${e.reason}`).join("; ")})`;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
// ── Citation provenance audit ──────────────────────────────────────────
|
|
1106
|
+
let citationAudit = citationLedger.count > 0 ? citationLedger.auditCitations(currentText) : undefined;
|
|
1107
|
+
if (citationAudit) {
|
|
1108
|
+
onProgress?.(`citation audit: ${citationAudit.totalCitations} tag(s) — ${citationAudit.trustedCount} trusted, ${citationAudit.untrustedCount} untrusted, ${citationAudit.missingCount} missing`);
|
|
1109
|
+
if (citationAudit.untrustedCount > 0) {
|
|
1110
|
+
citationAudit = await citationLedger.verifyEntailment(citationAudit, apiKey, modelUsedForDraft, onProgress);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
onProgress?.("done");
|
|
1115
|
+
// Computed HERE (not earlier) so it reflects any adversarial-repair and/or
|
|
1116
|
+
// polish-pass mutations to bestIssuesFlat that happened after the N-Deep loop.
|
|
1117
|
+
const finalIssues = bestIssuesFlat;
|
|
1118
|
+
const combined = judgeScore !== null ? Math.min(guardScore, judgeScore) : guardScore;
|
|
1119
|
+
return {
|
|
1120
|
+
question,
|
|
1121
|
+
draft: originalDraft,
|
|
1122
|
+
fixed: currentText,
|
|
1123
|
+
issues: finalIssues,
|
|
1124
|
+
autoFixesApplied: [...new Set(allFixes)],
|
|
1125
|
+
guardScore,
|
|
1126
|
+
judgeScore,
|
|
1127
|
+
judgeNote,
|
|
1128
|
+
eloConsensus,
|
|
1129
|
+
testbedGatesProposed,
|
|
1130
|
+
judgeRoster,
|
|
1131
|
+
modelUsed: modelUsedForDraft,
|
|
1132
|
+
passes: passHistory.length,
|
|
1133
|
+
stable: combined >= 9.0 && !finalIssues.some(i => i.severity === "critical"),
|
|
1134
|
+
totalLatencyMs: Date.now() - t0,
|
|
1135
|
+
groundingProvider,
|
|
1136
|
+
groundingCount,
|
|
1137
|
+
runSettings: {
|
|
1138
|
+
depth: maxDepth, fourStage: !!profile.fourStage, cluster: !!profile.cluster,
|
|
1139
|
+
clusterSize: profile.clusterSize ?? 8, sloop: !!profile.sloop, sloopPages: profile.sloopPages ?? 4,
|
|
1140
|
+
templateId: profile.templateId, styleOverride: profile.styleOverride,
|
|
1141
|
+
williamsPersona: profile.williamsPersona, adversarial: !!profile.adversarial,
|
|
1142
|
+
webSearch: !!profile.webSearch, defensePack: !!profile.useOriginalDefensePack,
|
|
1143
|
+
advancedGates: !!opts.advancedGates, singleJudge: !!opts.singleJudge,
|
|
1144
|
+
},
|
|
1145
|
+
passHistory,
|
|
1146
|
+
bestPassIndex: bestIdx,
|
|
1147
|
+
adversarialPreview,
|
|
1148
|
+
judgeExcluded,
|
|
1149
|
+
bestOfNCandidates,
|
|
1150
|
+
coveReport,
|
|
1151
|
+
polishApplied,
|
|
1152
|
+
citationAudit,
|
|
1153
|
+
};
|
|
1154
|
+
}
|