@mutagent/helix 0.1.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/mutagent-helix/SKILL.md +113 -0
- package/.claude/skills/mutagent-helix/scripts/cli/codex-transcode.ts +143 -0
- package/.claude/skills/mutagent-helix/scripts/cli/doctor.ts +111 -0
- package/.claude/skills/mutagent-helix/scripts/cli/init.ts +382 -0
- package/.claude/skills/mutagent-helix/scripts/cli/run.sh +61 -0
- package/CLAUDE.md +193 -0
- package/LICENSE +201 -0
- package/NOTICE +22 -0
- package/README.md +209 -0
- package/assets/adl-loop.svg +162 -0
- package/assets/system-status.svg +422 -0
- package/bin/mutagent-cli.mjs +9950 -0
- package/bin/mutagent-helix.mjs +78 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/CLAUDE.md +28 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/SKILL.md +138 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/assets/templates/agentspec.yaml.tpl +247 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/frameworks/doc-pins.md +53 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/principles.md +91 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/requirements.yaml +295 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/workflows/orchestrator-protocol.md +302 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/doctor.ts +116 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/init.ts +230 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/run.sh +60 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/config/resolve-spec-dir.ts +199 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/contract/agentspec.schema.ts +455 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/setup/detect.ts +67 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/validate/validate-spec.ts +93 -0
- package/mutagent-agentspec/LICENSE +201 -0
- package/mutagent-agentspec/NOTICE +19 -0
- package/mutagent-agentspec/README.md +44 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/SKILL.md +100 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-architect.md +312 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-engineer.md +357 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/templates/build-report.md.tpl +39 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/principles.md +31 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/build-protocol.md +61 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/improve-handoff.md +23 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/cli/run.sh +60 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/handoff/validate-build-input.ts +72 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/sync-spec/check-sync-spec.ts +499 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/verify/spec-impl-coverage.ts +182 -0
- package/mutagent-builder/CLAUDE.md +25 -0
- package/mutagent-builder/README.md +26 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/README.md +192 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/SKILL.md +352 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-architect.md +256 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-engineer.md +344 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/diagnostics-analyzer.md +576 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/discovery.md +425 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.json.tpl +24 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.md.tpl +49 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/config.yaml.tpl +100 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/pr-body.md.tpl +56 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/report.html.tpl +327 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/README.md +27 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/iter-N-handover.md.tpl +125 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/self-diagnosis-contract.v0.1.0.yaml.tpl +234 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/spec.yaml.tpl +152 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/team.yaml.tpl +127 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/wave-N-dashboard.html.tpl +1712 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.1-nl-query.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.2-tier0-progress.md +58 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.3-analysis-progress.md +44 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.4-html-report.md +61 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.5-rca-detail.md +69 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.6-stale-warning.md +53 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.1-welcome.md +60 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.2-source-platform-picker.md +41 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.3-source-auth.md +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.4-target-picker.md +48 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.5-ask-tool.md +43 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.6-config-review.md +57 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.7-filter-defaults.md +52 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.8-first-run.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-findings.json +92 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-meta.json +47 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/adapter-strategy.md +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config-migration.md +84 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config.md +159 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/filter-search-matrix.md +66 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/harness-knowledge.md +22 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/memory-format.md +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/onboarding-decisions.yaml +187 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/operation-inventory.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/overview.md +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/reference.md +119 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/script-index.md +101 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-adapter-contract.md +93 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/claude-code-transcripts.md +66 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex-transcripts.md +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex.md +145 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/install-paths.md +57 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/langfuse.md +178 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/local-jsonl.md +55 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/openobserve.md +102 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/otel.md +63 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/cloud-rest.md +84 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-claude.md +65 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cloud-agent-sdk.md +92 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-codex.md +72 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cursor.md +108 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-mastra.md +77 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-opencode.md +70 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/apply-pr-comment-format.md +82 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/diagnostics.md +35 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/handover-contract.md +247 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/onboarding.md +300 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/orchestrator-protocol.md +1861 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rca.md +331 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rendering-anatomy.md +314 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/schedule-prep.md +100 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/scope-model.md +128 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/verification-methodology.md +174 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/aggregate/sort-findings.ts +0 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/blind-spots.ts +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/llm-sample.ts +681 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/doctor.ts +215 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/init.ts +1013 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/install-agents.ts +350 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/run.sh +61 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/shared-templates.ts +79 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/load.ts +392 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/resolve-credential.ts +120 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/schema.ts +641 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/validate.ts +134 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/build-diagnosis-context.ts +417 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/load-context.ts +105 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/contract/types.ts +166 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/build-render-input.ts +1964 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/corpus-severity.ts +172 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/dismissal-match.ts +270 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/rank-remedies.ts +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/install-hooks.sh +46 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/pre-push.sh +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/invocation/parse-brief.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/match.ts +114 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/paths.ts +95 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/store.ts +535 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/types.ts +181 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/lint/template-inline-js.ts +267 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/append.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/read.ts +218 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/platforms/entity-context.ts +686 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/read-unitf.ts +267 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/trace.ts +1078 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-adapter.ts +0 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-types.ts +213 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/persist-selections.ts +174 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/render.ts +3254 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/diagnose.ts +64 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/session.ts +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/caps.ts +345 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/deep-read-gate.ts +424 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/representative.ts +584 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/objection.ts +224 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/trajectory.ts +489 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/dispatch.ts +135 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/probe.ts +170 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/detect.ts +307 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/ensure-cli.ts +315 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/reconfigure.ts +77 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/verify-agents.ts +207 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/slicer.ts +166 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/claude-code.ts +17 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/codex.ts +11 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/langfuse.ts +17 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/local-jsonl.ts +12 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/otel.ts +12 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/index.ts +54 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/registry.ts +45 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/types.ts +86 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/stale-detector.ts +74 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/test/render-playwright.cli.ts +477 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/claude-code.ts +133 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/langfuse.ts +143 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0-scan.ts +423 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/completeness-check.ts +436 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/doc-refs.ts +190 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/finalize-gate.ts +417 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/findings-contract.ts +628 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/render-js-syntax.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/report-checklist.yaml +217 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/trace-shape.ts +262 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/wave6-checklist.ts +326 -0
- package/mutagent-diagnostics/CLAUDE.md +41 -0
- package/mutagent-diagnostics/LICENSE +201 -0
- package/mutagent-diagnostics/NOTICE +18 -0
- package/mutagent-diagnostics/README.md +343 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/SKILL.md +548 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/audit-executor.md +169 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/discovery.md +425 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +1108 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/theme.css +213 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/wordmark.html +9 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/context-flow-lens.md +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/data-lens.md +47 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/decision-lens.md +48 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +53 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/trajectory-lens.md +44 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/build-review-interface.md +83 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/edd-loop.md +134 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/error-analysis.md +113 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-audit.md +154 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-stage.md +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/generate-synthetic-data.md +81 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/grounded-adjudication.md +221 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/memory-format.md +65 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/methodology.md +201 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/operation-inventory.md +197 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/validate-evaluator.md +125 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/workflows/orchestrator-protocol.md +300 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/write-judge-prompt.md +123 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/behavior-tree.schema.yaml +73 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/dataset.schema.yaml +66 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/edd-change-request.schema.yaml +114 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/eval-matrix.schema.yaml +74 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-graph.schema.yaml +69 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-profile.schema.yaml +49 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +40 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/scorecard.schema.yaml +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +543 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/artifact-paths.ts +115 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/assemble-scorecard.ts +172 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-dataset.ts +186 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-evals.ts +93 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +393 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/check-method-router.ts +170 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/aggregate.ts +112 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts +182 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/doctor.ts +211 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/dogfood.ts +133 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/init.ts +601 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/methodology-review.ts +122 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +295 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +165 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/run.sh +56 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/variance-check.ts +105 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-eval.ts +248 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +285 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/codegen-evals.ts +173 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-project.ts +151 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-sampler.ts +267 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/load.ts +343 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +380 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/dataset.ts +149 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-engine.ts +118 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +577 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +1074 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/flow-graph.ts +194 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/types.ts +303 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/validation.ts +193 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/derive-dataset.ts +152 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +219 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/diff-discriminate.ts +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/discover-criteria.ts +348 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +232 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +210 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/variance-gate.ts +186 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/emit-completeness.ts +111 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/eval-engine.ts +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/evaluate.ts +333 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/flow-graph.ts +230 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +253 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-provider.ts +135 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-grounding.ts +229 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-uniformity.ts +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/living-suite.ts +109 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-bundle.ts +203 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile-vocab.ts +64 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile.ts +106 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/mask.ts +138 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +113 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +750 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +215 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/read.ts +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts +232 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +125 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/profile-subject.ts +310 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/publish-report.ts +131 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +81 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +195 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-discover-report.ts +1640 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +3838 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-report.ts +212 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-credential.ts +110 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-ref.ts +98 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/result-verify.ts +129 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +330 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts +367 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +719 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-judge.ts +155 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +175 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sample-traces.ts +210 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/self-audit.ts +387 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/source-map.ts +106 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +134 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/substrate.ts +162 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +244 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/ui-slots.ts +119 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +284 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/validate-judge.ts +358 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/variance-compare.ts +177 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/behavior-tree.yaml +140 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/eval-matrix.yaml +1270 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/methodology-review.yaml +105 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/audit.workflow.js +82 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/data-leak.workflow.js +236 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/variance.workflow.js +163 -0
- package/mutagent-evaluator/CLAUDE.md +45 -0
- package/mutagent-evaluator/LICENSE +201 -0
- package/mutagent-evaluator/NOTICE +23 -0
- package/mutagent-evaluator/README.md +88 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/SKILL.md +287 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/contracts/amend-request.ts +503 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/improve-loop-run.ts +304 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state-cli.ts +300 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state.ts +440 -0
- package/mutagent-optimize/CLAUDE.md +49 -0
- package/mutagent-optimize/README.md +26 -0
- package/mutagent-orchestrator/CLAUDE.md +102 -0
- package/mutagent-orchestrator/README.md +20 -0
- package/mutagent-orchestrator/assets/agents/discovery.md +425 -0
- package/mutagent-orchestrator/assets/agents/dogfood-monitor.md +266 -0
- package/mutagent-orchestrator/assets/agents/monitor.md +258 -0
- package/mutagent-orchestrator/orchestrator.md +782 -0
- package/mutagent-orchestrator/references/MONITORING.md +209 -0
- package/mutagent-orchestrator/references/config-fields.md +132 -0
- package/mutagent-orchestrator/references/config-migration.md +132 -0
- package/mutagent-orchestrator/references/index-registry.md +92 -0
- package/mutagent-orchestrator/routing.yaml +391 -0
- package/mutagent-orchestrator/scripts/config-schema.ts +1010 -0
- package/mutagent-orchestrator/scripts/dispatch.ts +439 -0
- package/mutagent-orchestrator/scripts/dogfood/extract-feedback.ts +203 -0
- package/mutagent-orchestrator/scripts/dogfood/reconstruct-trajectory.ts +327 -0
- package/mutagent-orchestrator/scripts/dogfood/render-dogfood-report.ts +347 -0
- package/mutagent-orchestrator/scripts/dogfood/resolve-subagents.ts +160 -0
- package/mutagent-orchestrator/scripts/dogfood/run-report.ts +264 -0
- package/mutagent-orchestrator/scripts/dogfood/slack-thread.ts +106 -0
- package/mutagent-orchestrator/scripts/dogfood/types.ts +257 -0
- package/mutagent-orchestrator/scripts/eval-routing.ts +753 -0
- package/mutagent-orchestrator/scripts/gate.ts +219 -0
- package/mutagent-orchestrator/scripts/handover-contract.ts +362 -0
- package/mutagent-orchestrator/scripts/index/build-index.ts +351 -0
- package/mutagent-orchestrator/scripts/index/render-index.ts +110 -0
- package/mutagent-orchestrator/scripts/monitor/slack-notify.ts +111 -0
- package/mutagent-orchestrator/scripts/monitor/triggers.ts +162 -0
- package/mutagent-orchestrator/scripts/onboarding-check.ts +369 -0
- package/mutagent-orchestrator/scripts/render-roster.ts +308 -0
- package/mutagent-orchestrator/scripts/resolve-credential.ts +185 -0
- package/mutagent-orchestrator/scripts/resolve-paths.ts +114 -0
- package/mutagent-orchestrator/scripts/slack/format.ts +63 -0
- package/mutagent-orchestrator/scripts/slack/post.ts +115 -0
- package/mutagent-orchestrator/scripts/sync-index.ts +311 -0
- package/package.json +64 -0
|
@@ -0,0 +1,3838 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scripts/render-eval-report.ts — P4 (D-1/D-2/D-3): the v2 `*evaluate` reporting.
|
|
3
|
+
* ---------------------------------------------------------------------------
|
|
4
|
+
* The `*evaluate` path (P5) produces a Scorecard (GATE + variance + per-criterion
|
|
5
|
+
* verdicts); this module is the v2 HTML reporting. It is a faithful TS PORT of the
|
|
6
|
+
* operator-APPROVED 5-tab component spec (the sample-eval-1 reference renderer —
|
|
7
|
+
* `.mutagent/evaluator/sample-eval-1/scripts/render-report.py`), NOT a flat table:
|
|
8
|
+
*
|
|
9
|
+
* - D-1 renderEvalCards : terminal per-criterion cards from the Scorecard.
|
|
10
|
+
* - D-2/D-3 renderEvalReport: the 5-tab HTML eval-report —
|
|
11
|
+
* §1 Overview — KPIs + coverage-contract note + gating-criteria
|
|
12
|
+
* table (pass-rate over applicable) + top-findings teaser.
|
|
13
|
+
* §2 Trajectory·Judge — per-trace ledger (filter/search) + click-row drill:
|
|
14
|
+
* Target-Agent‖Judge SIDE-BY-SIDE (gather-context band,
|
|
15
|
+
* two lanes, judge micro-steps anchored to agent steps,
|
|
16
|
+
* localize band, judge-health) when judge_steps exist,
|
|
17
|
+
* else the per-trajectory scorecard.
|
|
18
|
+
* §3 Eval Scorecard — cohort heatmap (criteria × route cohort) + nested
|
|
19
|
+
* subcards (grounding / verdict / why-abstained) +
|
|
20
|
+
* inline calibration buttons.
|
|
21
|
+
* §4 Findings — verbatim evidence bullets + judge reasoning chain +
|
|
22
|
+
* agree/revise/refute alignment review.
|
|
23
|
+
* §5 Self-Eval [INTERNAL] — judge-health + methodology + EV-051 routed
|
|
24
|
+
* decisions; carries a `strip-for-client` marker.
|
|
25
|
+
* - autoOpenCommand : cross-platform open helper (built; the orchestrator
|
|
26
|
+
* fires it post-render — never fired here).
|
|
27
|
+
*
|
|
28
|
+
* GRACEFUL DEGRADE: the rich tabs CONSUME the §9.4 judge contract (a dense
|
|
29
|
+
* na-explicit per-criterion map + `judge_steps[]` anchored to agent steps) WHEN
|
|
30
|
+
* the judge emits it — and fall back to the scorecard-derived rendering when it is
|
|
31
|
+
* absent. The judge-AGENT emission of dense-map + judge_steps is a SEPARATE change
|
|
32
|
+
* (see TODO[§9.4-judge-emit] in buildEvalReportInput + the additive optional
|
|
33
|
+
* fields the renderer reads off `MatrixVerdictFile`).
|
|
34
|
+
*
|
|
35
|
+
* Brand: rendered from the evaluator's OWN bundled `assets/brand/theme.css` (the
|
|
36
|
+
* unified design-system tokens, aligned to @mutagent/templates/tokens.css) +
|
|
37
|
+
* report-component CSS referencing those same tokens — no diagnostics-package
|
|
38
|
+
* runtime ref (sealed-sibling). SHARP corners · SUBTLE non-black cards · TONED
|
|
39
|
+
* status · RESTRAINED glow. Tabs are <button> (NOT <s>) — no strikethrough.
|
|
40
|
+
*
|
|
41
|
+
* DETERMINISTIC + null-guarded: the ONLY non-deterministic input is the injected
|
|
42
|
+
* `generatedAt`, so two renders of the same report input are BYTE-IDENTICAL after
|
|
43
|
+
* the generatedAt mask (`mask.ts`). v1 render-report.ts / evaluate.ts / mask.ts are
|
|
44
|
+
* UNTOUCHED — this is a separate v2 renderer.
|
|
45
|
+
*/
|
|
46
|
+
import { readFileSync } from "node:fs";
|
|
47
|
+
import { dirname, join } from "node:path";
|
|
48
|
+
import { fileURLToPath } from "node:url";
|
|
49
|
+
import {
|
|
50
|
+
AssumptionKind,
|
|
51
|
+
CriterionFlag,
|
|
52
|
+
Grounding,
|
|
53
|
+
OutcomeVerdict,
|
|
54
|
+
type CriterionVerdict,
|
|
55
|
+
type DiscoveryRationale,
|
|
56
|
+
type EvalTrace,
|
|
57
|
+
type MinedCriterion,
|
|
58
|
+
} from "./contracts/eval-types.ts";
|
|
59
|
+
import { localizeText, PROFILE_UNKNOWN } from "./contracts/eval-matrix.ts";
|
|
60
|
+
import type {
|
|
61
|
+
ExpectedStep,
|
|
62
|
+
MatrixCriterion,
|
|
63
|
+
MatrixVerdictFile,
|
|
64
|
+
SubjectProfile,
|
|
65
|
+
Understanding,
|
|
66
|
+
} from "./contracts/eval-matrix.ts";
|
|
67
|
+
import { deriveWalkHealth } from "./matrix-judge.ts";
|
|
68
|
+
import { assessEmitCompleteness, type EmitCompleteness } from "./emit-completeness.ts";
|
|
69
|
+
import type { Scorecard } from "./evaluate.ts";
|
|
70
|
+
import type { SourceMap } from "./source-map.ts";
|
|
71
|
+
import { routeFailures, type FailureRef, type HandoverBundle } from "./route-failures.ts";
|
|
72
|
+
|
|
73
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
74
|
+
const BRAND_DIR = join(HERE, "..", "assets", "brand");
|
|
75
|
+
|
|
76
|
+
/** HTML-escape (null-guarded — no throw on undefined). Mirrors render-report.ts. */
|
|
77
|
+
export function esc(s: unknown): string {
|
|
78
|
+
return String(s ?? "")
|
|
79
|
+
.replace(/&/g, "&")
|
|
80
|
+
.replace(/</g, "<")
|
|
81
|
+
.replace(/>/g, ">")
|
|
82
|
+
.replace(/"/g, """);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── Render input (rich; every rich field OPTIONAL → graceful degrade) ─────────
|
|
86
|
+
|
|
87
|
+
/** §9.4.4 R4 — where a criterion CAME FROM (defined-in-suite vs source-mined). */
|
|
88
|
+
export interface CriterionProvenance {
|
|
89
|
+
/** `defined` = an authored matrix row · `source` = mined from the trace batch. */
|
|
90
|
+
kind: "defined" | "source";
|
|
91
|
+
/** a human-readable origin label (e.g. "eval-matrix.yaml" / "mined: ✓/✗ split"). */
|
|
92
|
+
label: string;
|
|
93
|
+
/** the source detail (e.g. how many traces it was seen in), when known. */
|
|
94
|
+
detail?: string;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** One criterion as the report needs it (mined OR matrix-derived). */
|
|
98
|
+
export interface ReportCriterion {
|
|
99
|
+
id: string;
|
|
100
|
+
statement: string;
|
|
101
|
+
severity: string; // CRIT | HIGH | MED | LOW
|
|
102
|
+
gating?: boolean;
|
|
103
|
+
dimension?: string;
|
|
104
|
+
level?: string; // context | output | cross-stage
|
|
105
|
+
appliesTo?: string[];
|
|
106
|
+
/** §9.4.4 R4 — the concrete binary pass condition (the full definition for the hover). */
|
|
107
|
+
passCondition?: string;
|
|
108
|
+
/** UI-7 — what the judge READS to evaluate this criterion (the "how it's judged"
|
|
109
|
+
* inputs, eval-matrix `judgeInputs`). Render-layer surfacing of an existing field. */
|
|
110
|
+
judgeInputs?: string[];
|
|
111
|
+
/** §9.4.4 R4 — where the criterion came from (defined vs source). */
|
|
112
|
+
provenance?: CriterionProvenance;
|
|
113
|
+
/** UI-10 — the code-vs-judge ROUTER for this criterion (eval-matrix `checkMethod`
|
|
114
|
+
* / mined `metadata.check_method`): `deterministic` (CODE — a code-eval, no judge
|
|
115
|
+
* tokens) · `hybrid` (code pre-filter + judge) · `llm-judge` (an LLM-judge
|
|
116
|
+
* criterion). Defaults to `llm-judge` when the source row omits it (the legacy
|
|
117
|
+
* all-judge behaviour). Render-layer surfacing — drives the CODE vs JUDGE chip;
|
|
118
|
+
* NO judge-protocol change. */
|
|
119
|
+
checkMethod?: string;
|
|
120
|
+
/** the §5c DR-2 discovery-rationale block (present iff mined). */
|
|
121
|
+
discovery?: DiscoveryRationale;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** One ordered judge micro-step (§9.4 `judge_steps[]`), anchored to an agent step. */
|
|
125
|
+
export interface JudgeStep {
|
|
126
|
+
/** context | examine | detect | bind | ground | critique | decide | verify */
|
|
127
|
+
kind: string;
|
|
128
|
+
text?: string;
|
|
129
|
+
/** a grounding citation — a string OR a structured {obs,path,value} ref (coerced in the renderer). */
|
|
130
|
+
ref?: string | { obs: string; path: string; value: string };
|
|
131
|
+
/** the agent-step index this judge step anchors to (0 / "context" = gather band).
|
|
132
|
+
* TOLERANT: real judges may emit a stringified index / label / range (coerced below). */
|
|
133
|
+
anchor?: number | string;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** One target-agent step (the left lane of the §2 side-by-side). */
|
|
137
|
+
export interface AgentStep {
|
|
138
|
+
n: number;
|
|
139
|
+
tool?: string;
|
|
140
|
+
/** ok | error | warn | false-success */
|
|
141
|
+
status?: string;
|
|
142
|
+
detail?: string;
|
|
143
|
+
/** The trace OBSERVATION id this step corresponds to. Carried best-effort by
|
|
144
|
+
* `enrichAgentStepsObs` (verdict files don't emit it natively): recovered from
|
|
145
|
+
* the per-criterion verdict `refs` (`path:"name"` → `{obs, value:<tool>}`) and
|
|
146
|
+
* the `tool@<obs>` tokens the judge embeds in its critiques. Lets the §2 judge
|
|
147
|
+
* lane map a step to the criterion verdicts that EXAMINED it via `ref.obs ===
|
|
148
|
+
* step.obs` (the precise key) on top of the tool-name fallback. Undefined when
|
|
149
|
+
* no obs id could be recovered for the step (then tool-name match still applies). */
|
|
150
|
+
obs?: string;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** WS-1 — one per-criterion JUDGE verdict for a SINGLE trajectory, read verbatim
|
|
154
|
+
* from the verdict file's `verdicts[]` (critique-before-verdict + grounding refs).
|
|
155
|
+
* Distinct from the FOLDED `CriterionVerdict` (one-per-criterion across the batch):
|
|
156
|
+
* this is the per-trace judgement bound to the §2 drill so EVERY evaluated trace
|
|
157
|
+
* surfaces its judge reasoning (result · critique · refs), whether or not a
|
|
158
|
+
* `judge_steps[]` walk was emitted for it. */
|
|
159
|
+
export interface CriterionTrajectoryVerdict {
|
|
160
|
+
criterionId: string;
|
|
161
|
+
/** pass | fail | uncertain | na */
|
|
162
|
+
result: string;
|
|
163
|
+
confidence?: number;
|
|
164
|
+
confidenceBand?: string;
|
|
165
|
+
critique?: string;
|
|
166
|
+
/** grounding citations — a string OR a structured {obs,path,value} (coerced in the UI). */
|
|
167
|
+
refs?: Array<string | { obs?: string; path?: string; value?: string }>;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Per-trajectory judge-health micro-summary (the §2 footer). */
|
|
171
|
+
export interface JudgeHealthRow {
|
|
172
|
+
contextGathered?: boolean;
|
|
173
|
+
grounded?: number;
|
|
174
|
+
assumed?: number;
|
|
175
|
+
stoppedAtSymptom?: boolean;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** WS-5 — the resolution class for an INDETERMINATE (uncertain) per-criterion verdict.
|
|
179
|
+
* `na` = the criterion's TRIGGER/precondition was ABSENT in this trace (not applicable —
|
|
180
|
+
* DROPPED from the applicable denominator). `needs-evidence` = the precondition WAS
|
|
181
|
+
* present but the judge could not decide (stays in the denominator, carries a concrete
|
|
182
|
+
* next-action). NEVER a bare "indeterminate". */
|
|
183
|
+
export type IndeterminateResolutionClass = "na" | "needs-evidence";
|
|
184
|
+
/** WS-5 — the chain next-action for a `needs-evidence` item (the deterministic decision
|
|
185
|
+
* chain: re-check the code, get a 2nd judge, revise the criterion, or HITL-spot-check). */
|
|
186
|
+
export type IndeterminateNextAction = "code-recheck" | "2nd-judge" | "revise-criterion" | "hitl-spot-check";
|
|
187
|
+
/** WS-5 — one resolved indeterminate per-criterion verdict (§3 resolution chain). */
|
|
188
|
+
export interface IndeterminateResolution {
|
|
189
|
+
criterionId: string;
|
|
190
|
+
resolutionClass: IndeterminateResolutionClass;
|
|
191
|
+
/** present iff `needs-evidence` — the concrete chain next-action. */
|
|
192
|
+
nextAction?: IndeterminateNextAction;
|
|
193
|
+
/** one-line WHY: the missing precondition (na) or the missing signal (needs-evidence). */
|
|
194
|
+
reason: string;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** One per-trace ledger row (§2). The side-by-side fields are present iff the
|
|
198
|
+
* judge emitted `judge_steps[]` for this trajectory. */
|
|
199
|
+
export interface LedgerRow {
|
|
200
|
+
trajectoryId: string;
|
|
201
|
+
route?: string;
|
|
202
|
+
/** PASS | FAIL | INDETERMINATE | INCOMPLETE */
|
|
203
|
+
verdict: string;
|
|
204
|
+
/** dense, na-explicit per-criterion map: cid -> pass|fail|uncertain|na. */
|
|
205
|
+
perCriterion: Record<string, string>;
|
|
206
|
+
failingCriteria: string[];
|
|
207
|
+
root?: string;
|
|
208
|
+
grounding?: string;
|
|
209
|
+
/** WS-1 — the per-criterion JUDGE verdicts for THIS trajectory (result · critique-
|
|
210
|
+
* before-verdict · grounding refs), read verbatim from the verdict file's
|
|
211
|
+
* `verdicts[]`. Bound to the §2 drill ("How the Judge Reasoned") so EVERY trace —
|
|
212
|
+
* PASS or FAIL, walk or no walk — surfaces its judge reasoning. Present whenever
|
|
213
|
+
* the verdict file carried ≥1 per-criterion verdict (i.e. all judged trajectories). */
|
|
214
|
+
criterionVerdicts?: CriterionTrajectoryVerdict[];
|
|
215
|
+
// ── §2 side-by-side (present iff judge_steps[] emitted) ──
|
|
216
|
+
context?: { harness?: string; scenario?: string; exitStates?: string };
|
|
217
|
+
/** Gap A — the RAW triggering INPUT for this trajectory (the thing that fired
|
|
218
|
+
* the subject agent). Sourced from the TRACE (`EvalTrace.input.prompt`, fallback
|
|
219
|
+
* packet `transcript[0]`) keyed by traceId in `buildEvalReportInput` — NOT a
|
|
220
|
+
* judge-emitted field (the input is ground-truth in the trace, no judge echo
|
|
221
|
+
* needed). ABSENT when the trace carried no readable input ⇒ the §2 drill cell
|
|
222
|
+
* renders "—" (never faked). The judge's `context.scenario` LABEL still shows
|
|
223
|
+
* beneath it — the input is the WHAT, the scenario is the route summary. */
|
|
224
|
+
input?: string;
|
|
225
|
+
agentSteps?: AgentStep[];
|
|
226
|
+
judgeSteps?: JudgeStep[];
|
|
227
|
+
localize?: string;
|
|
228
|
+
health?: JudgeHealthRow;
|
|
229
|
+
// ── §9.4.4 v2.2 (present iff the judge emitted them) ──
|
|
230
|
+
/** M2 node-0 train-of-thought (rephrase + given-vs-inferred). */
|
|
231
|
+
understanding?: Understanding;
|
|
232
|
+
/** M3 node-0.5 expected-trajectory (how the target SHOULD have acted). */
|
|
233
|
+
expectedTrajectory?: ExpectedStep[];
|
|
234
|
+
/** M1 the subject profile the judge reasoned under (echoed / reconstructed). */
|
|
235
|
+
subjectProfile?: SubjectProfile;
|
|
236
|
+
/** §9.4.5 E3 — the trace's wall-clock timestamp (ISO), when the source carried it.
|
|
237
|
+
* Powers the eval-HEALTH temporal heatmap (correctness over time). ABSENT ⇒ that
|
|
238
|
+
* trajectory falls into the data-pending bucket (structure renders, never faked). */
|
|
239
|
+
timestamp?: string;
|
|
240
|
+
/** WS-5 — per-criterion INDETERMINATE resolution for THIS trace (keyed by criterionId):
|
|
241
|
+
* the deterministic precondition gate's verdict (`na` precondition-absent vs
|
|
242
|
+
* `needs-evidence` + next-action). Present only for criteria whose raw verdict was
|
|
243
|
+
* uncertain on this trace. */
|
|
244
|
+
indeterminateResolution?: Record<string, IndeterminateResolution>;
|
|
245
|
+
/** WS-2 — how this trajectory's judgement RESOLVED, made honest + visible (a trace
|
|
246
|
+
* with no §2 walk is NOT "unjudged"). One of:
|
|
247
|
+
* • `judge-walk` — the judge emitted the §9.4 walk (agentSteps/judgeSteps)
|
|
248
|
+
* • `judged-walk-not-captured` — the trace WAS judged (per-criterion verdicts present)
|
|
249
|
+
* but the emit-contract dropped the structured walk
|
|
250
|
+
* • `truncated` — INCOMPLETE fidelity (node-1 gate); never walked by design
|
|
251
|
+
* Drives the §2 ledger resolution badge + the per-trace drill routing line. */
|
|
252
|
+
resolution?: TrajectoryResolution;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** WS-2 — the resolution class of one ledger row (see `LedgerRow.resolution`). */
|
|
256
|
+
export type TrajectoryResolution = "judge-walk" | "judged-walk-not-captured" | "truncated";
|
|
257
|
+
|
|
258
|
+
/** WS-2 — derive the resolution class for one verdict file. PURE.
|
|
259
|
+
* truncated > judge-walk (walk emitted) > judged-walk-not-captured (judged, walk dropped). */
|
|
260
|
+
export function deriveTrajectoryResolution(f: {
|
|
261
|
+
fidelity?: { complete?: boolean };
|
|
262
|
+
agentSteps?: unknown[];
|
|
263
|
+
judgeSteps?: unknown[];
|
|
264
|
+
verdicts?: unknown[];
|
|
265
|
+
}): TrajectoryResolution {
|
|
266
|
+
if (f.fidelity?.complete === false) return "truncated";
|
|
267
|
+
const hasWalk = (f.judgeSteps?.length ?? 0) > 0 || (f.agentSteps?.length ?? 0) > 0;
|
|
268
|
+
if (hasWalk) return "judge-walk";
|
|
269
|
+
return "judged-walk-not-captured";
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** WS-2 — the human badge label + one-line routing explanation per resolution class. */
|
|
273
|
+
export function resolutionMeta(r: TrajectoryResolution): { badge: string; cls: string; routing: string } {
|
|
274
|
+
switch (r) {
|
|
275
|
+
case "judge-walk":
|
|
276
|
+
return {
|
|
277
|
+
badge: "JUDGE·WALK",
|
|
278
|
+
cls: "res-walk",
|
|
279
|
+
routing:
|
|
280
|
+
"Routed through the full §9.4 judge walk — the target step lane + the anchored judge reasoning were captured, so the side-by-side below is complete.",
|
|
281
|
+
};
|
|
282
|
+
case "judged-walk-not-captured":
|
|
283
|
+
return {
|
|
284
|
+
badge: "judged · walk not captured",
|
|
285
|
+
cls: "res-nowalk",
|
|
286
|
+
routing:
|
|
287
|
+
"This trace WAS judged (per-criterion verdicts + critiques are present below) — the judge simply did not persist the structured §9.4 walk (agentSteps/judgeSteps/M2/M3). The verdict stands; only the step-by-step lane is missing. The emit-completeness gate (§5) tracks this gap; a fresh run with the hardened emit-contract captures the walk natively.",
|
|
288
|
+
};
|
|
289
|
+
case "truncated":
|
|
290
|
+
return {
|
|
291
|
+
badge: "TRUNCATED",
|
|
292
|
+
cls: "res-trunc",
|
|
293
|
+
routing:
|
|
294
|
+
"The node-1 fidelity gate fired — the trace was truncated/unreadable, so it was NEVER walked per-criterion (no fabricated verdicts from a partial trace). Resolved INCOMPLETE by design, not a judge omission.",
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** §9.4.4 R3/M5 — one DETECTED-but-unmatched flag (a node-2.5 candidate). NOT a
|
|
300
|
+
* verdict: a real behaviour the judge flagged with NO matching criterion. Surfaced
|
|
301
|
+
* in §4 CLEARLY SEPARATED from fails-on-existing-criteria, and routed to discover. */
|
|
302
|
+
export interface DetectedFlag {
|
|
303
|
+
/** `eval` = a candidate criterion to mine · `dataset` = a candidate test case. */
|
|
304
|
+
kind: string;
|
|
305
|
+
detection: string;
|
|
306
|
+
trajectoryId: string;
|
|
307
|
+
anchor?: number | string;
|
|
308
|
+
ref?: string | { obs: string; path: string; value: string };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** One §4 finding (one failing/at-risk criterion, aggregated over the ledger). */
|
|
312
|
+
export interface TopFinding {
|
|
313
|
+
criterion: string;
|
|
314
|
+
severity?: string;
|
|
315
|
+
applicable: number;
|
|
316
|
+
failCount: number;
|
|
317
|
+
/** % over the APPLICABLE denominator (GA-D2b — na never counted). */
|
|
318
|
+
prevalencePctOverApplicable: number;
|
|
319
|
+
root?: string;
|
|
320
|
+
/** the trace whose judge-walk powers the verbatim evidence. */
|
|
321
|
+
exampleTraceId?: string;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** One gating-criterion roll (§1 table + §3 subcards). */
|
|
325
|
+
export interface GatingRoll {
|
|
326
|
+
criterion: string;
|
|
327
|
+
severity?: string;
|
|
328
|
+
applicable: number;
|
|
329
|
+
fail: number;
|
|
330
|
+
/** WS-5 — GENUINE under-evidence (precondition present, undecided). NOT N/A. */
|
|
331
|
+
indeterminate?: number;
|
|
332
|
+
/** WS-5 — N/A (precondition/trigger absent) — DROPPED from the applicable denominator. */
|
|
333
|
+
na?: number;
|
|
334
|
+
passRateOverApplicable: number;
|
|
335
|
+
denominatorNote?: string;
|
|
336
|
+
root?: string;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/** WS-5 — one per-criterion INDETERMINATE resolution roll (§3 resolution chain). Covers
|
|
340
|
+
* ALL criteria (not just gating). The applicable denominator EXCLUDES `na`. */
|
|
341
|
+
export interface CriterionResolution {
|
|
342
|
+
criterion: string;
|
|
343
|
+
severity?: string;
|
|
344
|
+
/** applicable = pass + fail + needsEvidence (N/A excluded — denominator-honest). */
|
|
345
|
+
applicable: number;
|
|
346
|
+
pass: number;
|
|
347
|
+
fail: number;
|
|
348
|
+
/** N/A (precondition absent) — shown SEPARATELY, never in the applicable denominator. */
|
|
349
|
+
na: number;
|
|
350
|
+
/** genuine under-evidence (precondition present, undecided) — carries next-actions. */
|
|
351
|
+
needsEvidence: number;
|
|
352
|
+
passRateOverApplicable: number;
|
|
353
|
+
/** the dominant N/A reason (the missing precondition), when na > 0. */
|
|
354
|
+
naReason?: string;
|
|
355
|
+
/** aggregated next-actions for the needsEvidence items (action → count + a reason). */
|
|
356
|
+
nextActions: Array<{ action: IndeterminateNextAction; count: number; reason: string }>;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
export interface EvalReportInput {
|
|
360
|
+
subject: { name: string; org?: string; source?: string; models?: string[] };
|
|
361
|
+
/** PRD R-4 — render audience. Default "client" (leak-safe): the §5 Self-Eval [INTERNAL]
|
|
362
|
+
* tab + nav button are NOT emitted. "internal" (admin opt-in) shows them. */
|
|
363
|
+
audience?: "client" | "internal";
|
|
364
|
+
/** the v2 *evaluate Scorecard (GATE + variance). */
|
|
365
|
+
scorecard: Scorecard;
|
|
366
|
+
/** the folded per-criterion verdicts. */
|
|
367
|
+
verdicts: CriterionVerdict[];
|
|
368
|
+
/** the criteria (mined → DR-2 cards; matrix → statement/severity only). */
|
|
369
|
+
criteria: ReportCriterion[];
|
|
370
|
+
// ── rich (all OPTIONAL — derived by `buildEvalReportInput` from real pipeline
|
|
371
|
+
// data; the renderer degrades gracefully when absent) ──
|
|
372
|
+
/** per-trace ledger (§2). */
|
|
373
|
+
ledger?: LedgerRow[];
|
|
374
|
+
/** coverage contract (§1). */
|
|
375
|
+
coverage?: { judged?: number; triaged?: number; byVerdict?: Record<string, number>; gapNote?: string };
|
|
376
|
+
/** route cohorts (§3 heatmap): counts per cohort + cid×cohort cells. */
|
|
377
|
+
cohorts?: {
|
|
378
|
+
counts: Record<string, number>;
|
|
379
|
+
matrix: Record<string, Record<string, { pass: number; fail: number; indeterminate: number }>>;
|
|
380
|
+
};
|
|
381
|
+
/** the gating-criteria rolls (§1 table + §3 subcards). */
|
|
382
|
+
gatingCriteria?: GatingRoll[];
|
|
383
|
+
/** WS-5 — per-criterion INDETERMINATE resolution chain (ALL criteria): the
|
|
384
|
+
* applicable·pass·fail·N/A·needs-evidence split that resolves the single
|
|
385
|
+
* indeterminate bucket into precondition-absent (N/A, denominator-excluded) vs
|
|
386
|
+
* genuine under-evidence (carries a next-action). §3 Scorecard. */
|
|
387
|
+
criterionResolutions?: CriterionResolution[];
|
|
388
|
+
/** the §4 top findings (verbatim-evidence cards). */
|
|
389
|
+
topFindings?: TopFinding[];
|
|
390
|
+
/** §9.4.4 R3/M5 — the DETECTED-but-unmatched flags (node-2.5 candidates), shown
|
|
391
|
+
* in §4 SEPARATED from criterion fails. A detection routed to discover, never minted. */
|
|
392
|
+
detectedFlags?: DetectedFlag[];
|
|
393
|
+
/** §9.4.4 M1/R2 — the subject profile (identity·purpose·tools·skill·scope) for the
|
|
394
|
+
* INTERNAL calibration tab + the §2 "who is the agent" band. */
|
|
395
|
+
subjectProfile?: SubjectProfile;
|
|
396
|
+
/** §9.4.5 E2 — per-tool OBSERVED call-count (from the trace observations / trajectory
|
|
397
|
+
* steps). Drives the entity-hero tool CHIPS' "obs N" badge — honest in the trace-only
|
|
398
|
+
* (no-code-access) case where the only tool evidence is what the traces show. */
|
|
399
|
+
toolObservations?: Record<string, number>;
|
|
400
|
+
/** judge-health (§5 self-eval). */
|
|
401
|
+
judgeHealth?: { groundedPct?: number; abstentionRatePct?: number; stoppedAtSymptomPct?: number };
|
|
402
|
+
/** P2 source-map topology (coverage note). */
|
|
403
|
+
sourceMap?: SourceMap;
|
|
404
|
+
/** the EV-051 diagnostics handover (§5 decisions); null when nothing failed. */
|
|
405
|
+
handover?: HandoverBundle | null;
|
|
406
|
+
/** WS-3 — the §5 Self-Eval AGGREGATE judge-calibration derivation (behavior map +
|
|
407
|
+
* the 9 components). Deterministic over the verdict files; ABSENT ⇒ §5 degrades to
|
|
408
|
+
* the legacy calibration view. */
|
|
409
|
+
selfEval?: SelfEvalAggregate;
|
|
410
|
+
/** Overview provenance meta-strip — the RUN config (run-id · date · source · JUDGE
|
|
411
|
+
* substrate · pinned judge model · temp-0 · C-PIN). Distinct from the TARGET model
|
|
412
|
+
* (`subject.models`). All OPTIONAL: a genuinely-unavailable field is MARKED, never
|
|
413
|
+
* faked; substrate/temp/C-PIN carry sensible engine defaults. */
|
|
414
|
+
runConfig?: RunConfig;
|
|
415
|
+
/** the ONLY non-deterministic input (masked for byte-identity). */
|
|
416
|
+
generatedAt: string;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/** §Overview-redesign — the run-configuration the provenance meta-strip renders. The
|
|
420
|
+
* JUDGE substrate + pinned judge model are DISTINCT from the TARGET model under eval. */
|
|
421
|
+
export interface RunConfig {
|
|
422
|
+
/** the *evaluate run id (the artifact dot-root key). */
|
|
423
|
+
runId?: string;
|
|
424
|
+
/** the run date/timestamp (defaults to `generatedAt` when absent). */
|
|
425
|
+
date?: string;
|
|
426
|
+
/** the trace SOURCE platform (langfuse · otel · local-ndjson · …). */
|
|
427
|
+
source?: string;
|
|
428
|
+
/** the JUDGE substrate: agent-dispatch (host runtime) · ai-sdk · code. */
|
|
429
|
+
judgeSubstrate?: string;
|
|
430
|
+
/** the PINNED judge model (C-PIN) — NOT the target model. */
|
|
431
|
+
judgeModel?: string;
|
|
432
|
+
/** the pinned judge temperature (C-PIN ⇒ 0). */
|
|
433
|
+
temperature?: number;
|
|
434
|
+
/** C-PIN engaged (pinned model + temp ⇒ byte-identical reruns). */
|
|
435
|
+
cPin?: boolean;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// ── helpers ─────────────────────────────────────────────────────────────────
|
|
439
|
+
|
|
440
|
+
function sevCls(sev: string | undefined): string {
|
|
441
|
+
const s = String(sev ?? "med").toLowerCase();
|
|
442
|
+
if (s === "crit") return "crit";
|
|
443
|
+
if (s === "high") return "high";
|
|
444
|
+
if (s === "med" || s === "medium") return "med";
|
|
445
|
+
return "low";
|
|
446
|
+
}
|
|
447
|
+
/** Heatmap / pass-rate colour by % (fail<80 · warn<95 · pass). Deterministic. */
|
|
448
|
+
function rateCol(r: number): string {
|
|
449
|
+
return r < 80 ? "var(--fail)" : r < 95 ? "var(--warn)" : "var(--pass)";
|
|
450
|
+
}
|
|
451
|
+
/** Map a per-criterion result string to its display token (uncertain→indeterminate). */
|
|
452
|
+
function resultToken(r: string): string {
|
|
453
|
+
return r === OutcomeVerdict.Uncertain ? "indeterminate" : r;
|
|
454
|
+
}
|
|
455
|
+
function verdictCls(v: string): string {
|
|
456
|
+
const u = v.toUpperCase();
|
|
457
|
+
return u === "PASS" ? "pass" : u === "FAIL" ? "fail" : "inc";
|
|
458
|
+
}
|
|
459
|
+
function criterionById(input: EvalReportInput, id: string): ReportCriterion | undefined {
|
|
460
|
+
return input.criteria.find((c) => c.id === id);
|
|
461
|
+
}
|
|
462
|
+
/** Render structured grounding refs `{obs,path,value}` as compact citations. */
|
|
463
|
+
function refsText(refs: { obs: string; path: string; value: string }[]): string {
|
|
464
|
+
return refs.map((r) => `${r.obs}${r.path ? "/" + r.path : ""}: "${r.value}"`).join(" · ");
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** UI-6 — pass-RATE → an accent class (driven by SUCCESS rate, NOT severity): green-ish
|
|
468
|
+
* high · amber mid · red-ish low. Mirrors the heatmap thresholds (≥95 ok · ≥80 mid ·
|
|
469
|
+
* <80 low) so a criterion card's colour reflects how it actually scored. Deterministic. */
|
|
470
|
+
export function rateClass(r: number): "rate-ok" | "rate-mid" | "rate-low" {
|
|
471
|
+
return r >= 95 ? "rate-ok" : r >= 80 ? "rate-mid" : "rate-low";
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
// ── UI-2 — verdict-state legend (Overview + Scorecard) ───────────────────────
|
|
475
|
+
/**
|
|
476
|
+
* UI-2 — the verdict-state LEGEND: plain-words for every run/trajectory verdict state
|
|
477
|
+
* so a reader never has to guess what INCOMPLETE means. Rendered in the Overview AND the
|
|
478
|
+
* Scorecard. INCOMPLETE is called out as DISTINCT from FAIL (a CRIT/HIGH criterion is
|
|
479
|
+
* uncertain → the run can't be certified, but it is not a fail). PURE.
|
|
480
|
+
*/
|
|
481
|
+
function verdictLegend(): string {
|
|
482
|
+
const items: [string, string, string][] = [
|
|
483
|
+
["pass", "PASS", "every applicable criterion was met — the run is certified as pass."],
|
|
484
|
+
["fail", "FAIL", "at least one gating (CRIT/HIGH) criterion failed."],
|
|
485
|
+
[
|
|
486
|
+
"inc",
|
|
487
|
+
"INCOMPLETE",
|
|
488
|
+
"a CRIT/HIGH criterion is UNCERTAIN (or the trace was too truncated to judge) — the run can't be certified as pass, but it is NOT a fail.",
|
|
489
|
+
],
|
|
490
|
+
["indet", "INDETERMINATE", "judged, but no confident pass/fail on this criterion (not certain either way)."],
|
|
491
|
+
["na", "na", "not applicable — out of scope for this trajectory; never counted as a fail."],
|
|
492
|
+
];
|
|
493
|
+
const cells = items
|
|
494
|
+
.map(
|
|
495
|
+
([cls, k, v]) =>
|
|
496
|
+
`<div class="vl-item"><span class="vl-k ${cls}">${esc(k)}</span><span class="vl-v">${esc(v)}</span></div>`,
|
|
497
|
+
)
|
|
498
|
+
.join("");
|
|
499
|
+
return `<div class="vlegend"><div class="vl-h">▾ what the verdicts mean</div>${cells}</div>`;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// ── UI-7 — the criterion DEFINITION block (explainability) ────────────────────
|
|
503
|
+
/**
|
|
504
|
+
* UI-7 — the criterion DEFINITION (what it is + HOW it's judged), shown BESIDE the
|
|
505
|
+
* verdict so a card reads as explainable, not robotic. Sourced purely from the
|
|
506
|
+
* criterion's EXISTING fields (statement · passCondition · dimension/severity/level ·
|
|
507
|
+
* judgeInputs · provenance · mined discovery). Every field OPTIONAL → graceful. PURE.
|
|
508
|
+
*/
|
|
509
|
+
function criterionDefn(c: ReportCriterion | undefined, fallbackId: string): string {
|
|
510
|
+
if (c === undefined) return `<div class="row">${esc(fallbackId)}</div>`;
|
|
511
|
+
const meta = [
|
|
512
|
+
c.dimension ? `dimension · ${c.dimension}` : "",
|
|
513
|
+
c.severity ? `severity · ${String(c.severity).toUpperCase()}` : "",
|
|
514
|
+
c.level ? `level · ${c.level}` : "",
|
|
515
|
+
c.gating ? "gating" : "",
|
|
516
|
+
]
|
|
517
|
+
.filter(Boolean)
|
|
518
|
+
.join(" · ");
|
|
519
|
+
const judged =
|
|
520
|
+
c.judgeInputs && c.judgeInputs.length > 0
|
|
521
|
+
? `<div class="row"><span class="ref">judged from:</span> ${esc(c.judgeInputs.join(", "))}</div>`
|
|
522
|
+
: "";
|
|
523
|
+
const disc =
|
|
524
|
+
c.discovery !== undefined
|
|
525
|
+
? `<div class="row"><span class="ref">mined:</span> ${esc(c.discovery.targets)} — ${esc(c.discovery.why_problem)}</div>`
|
|
526
|
+
: "";
|
|
527
|
+
return (
|
|
528
|
+
`<div class="row">${esc(c.statement)}</div>` +
|
|
529
|
+
(c.passCondition && c.passCondition !== c.statement
|
|
530
|
+
? `<div class="row"><span class="ref">pass when:</span> ${esc(c.passCondition)}</div>`
|
|
531
|
+
: "") +
|
|
532
|
+
(meta ? `<div class="row defn-meta">${esc(meta)}</div>` : "") +
|
|
533
|
+
judged +
|
|
534
|
+
`<div class="row"><span class="ref">provenance:</span> ${esc(c.provenance?.label ?? "—")}${c.provenance?.detail ? " · " + esc(c.provenance.detail) : ""}</div>` +
|
|
535
|
+
disc
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* WS-3 — a ONE-LINE plain-language gloss for a criterion: de-jargons WHAT it measures
|
|
541
|
+
* and WHY it matters, so a non-author can read the scorecard/findings without parsing
|
|
542
|
+
* the "Pass = …; Fail = …" statement scaffolding or the dimension/refs.
|
|
543
|
+
*
|
|
544
|
+
* SOURCED honestly, in priority order — NEVER fabricates beyond what the criterion
|
|
545
|
+
* already states:
|
|
546
|
+
* 1) the mined discovery rationale — `discovery.targets` (the behaviour it guards)
|
|
547
|
+
* + `discovery.why_problem` (the user/correctness consequence). These are the
|
|
548
|
+
* purpose-built readable fields, used verbatim.
|
|
549
|
+
* 2) DERIVED deterministically from the `statement` by stripping the
|
|
550
|
+
* `Pass = …` / `Fail = …` scaffolding into "Measures: <pass clause>. Why it
|
|
551
|
+
* matters: failing means <fail clause>." (a pure restructure — no new content).
|
|
552
|
+
* 3) the bare statement, when it carries no Pass/Fail scaffolding.
|
|
553
|
+
* Always returns a non-empty string for any criterion with a statement. PURE.
|
|
554
|
+
*/
|
|
555
|
+
export function plainExplainer(c: ReportCriterion | undefined): string {
|
|
556
|
+
if (c === undefined) return "";
|
|
557
|
+
const clean = (s: string): string => s.replace(/\s+/g, " ").trim();
|
|
558
|
+
// (1) mined criteria carry the most readable, purpose-built fields.
|
|
559
|
+
if (c.discovery !== undefined) {
|
|
560
|
+
const what = clean(c.discovery.targets);
|
|
561
|
+
const why = clean(c.discovery.why_problem);
|
|
562
|
+
if (what !== "") return `Measures: ${what}${why !== "" ? `. Why it matters: ${why}` : ""}.`;
|
|
563
|
+
}
|
|
564
|
+
// (2) derive from the statement — strip the Pass/Fail scaffolding (tolerant of
|
|
565
|
+
// `=`/`:` and casing; the pass clause runs up to a `;` or the `Fail` marker).
|
|
566
|
+
const stmt = clean(c.statement);
|
|
567
|
+
if (stmt === "") return "";
|
|
568
|
+
const passM = stmt.match(/pass\s*[:=]\s*([^;]+?)(?:;\s*|\s+fail\s*[:=]|$)/i);
|
|
569
|
+
const failM = stmt.match(/fail\s*[:=]\s*(.+)$/i);
|
|
570
|
+
if (passM) {
|
|
571
|
+
const what = clean(passM[1]);
|
|
572
|
+
const why = failM ? clean(failM[1]).replace(/\.$/, "") : "";
|
|
573
|
+
return `Measures: ${what}${why !== "" ? `. Why it matters: failing means ${why}` : ""}.`;
|
|
574
|
+
}
|
|
575
|
+
// (3) no scaffolding — present the statement as the measure.
|
|
576
|
+
return `Measures: ${stmt}`;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/** WS-3 — the muted plain-language banner element (above the technical block). Empty
|
|
580
|
+
* source ⇒ empty string (no stray banner). Shared by the §3 subcards + §4 findings. */
|
|
581
|
+
function plainBanner(c: ReportCriterion | undefined): string {
|
|
582
|
+
const txt = plainExplainer(c);
|
|
583
|
+
return txt === "" ? "" : `<div class="plain">${esc(txt)}</div>`;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* WS-5 — the deterministic INDETERMINATE-resolution gate. Resolves ONE uncertain
|
|
588
|
+
* per-criterion verdict into the decision chain, NEVER leaving a bare "indeterminate":
|
|
589
|
+
*
|
|
590
|
+
* ① PRECONDITION GATE (deterministic) — is the criterion's TRIGGER/precondition even
|
|
591
|
+
* present in this trace? Signals, in priority:
|
|
592
|
+
* (a) the judge's DENSE na-map marks this criterion `na` on this trace — its
|
|
593
|
+
* explicit, per-trace applicability call (the strongest deterministic signal);
|
|
594
|
+
* (b) the abstain is `blockedBy.kind === "scope"` — the criterion's referent is
|
|
595
|
+
* out of scope for this route;
|
|
596
|
+
* (c) a derivable precondition TRIGGER (tool/event) did NOT fire in the observed
|
|
597
|
+
* agent steps (used when the criterion carries one + the trace has steps).
|
|
598
|
+
* Any ⇒ `na` (precondition absent) — DROPPED from the applicable denominator.
|
|
599
|
+
* ② Otherwise the precondition WAS present but the judge could not decide ⇒
|
|
600
|
+
* `needs-evidence`, with a concrete NEXT-ACTION: code-recheck (a deterministic/hybrid
|
|
601
|
+
* criterion) · 2nd-judge (re-ground a `factual-intent` abstain) · revise-criterion
|
|
602
|
+
* (a `normative` value-call — operator-owned) · hitl-spot-check (default).
|
|
603
|
+
* PURE. `reason` is the judge's own `blockedBy.text` when present (the unbound term),
|
|
604
|
+
* never invented.
|
|
605
|
+
*/
|
|
606
|
+
export function classifyIndeterminate(
|
|
607
|
+
criterion: ReportCriterion | undefined,
|
|
608
|
+
verdict: { result?: string; blockedBy?: { kind?: string; text?: string } } | undefined,
|
|
609
|
+
denseValue: string | undefined,
|
|
610
|
+
agentSteps?: AgentStep[],
|
|
611
|
+
): IndeterminateResolution {
|
|
612
|
+
const cid = criterion?.id ?? "";
|
|
613
|
+
const bb = verdict?.blockedBy;
|
|
614
|
+
const trigger = preconditionTrigger(criterion);
|
|
615
|
+
const norm = (s: string): string => s.trim().toLowerCase();
|
|
616
|
+
const triggerAbsent =
|
|
617
|
+
trigger !== undefined && agentSteps !== undefined && agentSteps.length > 0
|
|
618
|
+
? !agentSteps.some((s) => norm(s.tool ?? "") === norm(trigger))
|
|
619
|
+
: false;
|
|
620
|
+
// ① precondition-absent ⇒ N/A (excluded from the applicable denominator)
|
|
621
|
+
if (denseValue === "na" || bb?.kind === AssumptionKind.Scope || triggerAbsent) {
|
|
622
|
+
const reason =
|
|
623
|
+
bb?.text && bb.text.trim().length > 0
|
|
624
|
+
? bb.text.trim()
|
|
625
|
+
: trigger
|
|
626
|
+
? `trigger '${trigger}' not invoked in this trace`
|
|
627
|
+
: "criterion not applicable to this trace — precondition/trigger absent";
|
|
628
|
+
return { criterionId: cid, resolutionClass: "na", reason };
|
|
629
|
+
}
|
|
630
|
+
// ② precondition present but undecided ⇒ needs-evidence + a chain next-action
|
|
631
|
+
const cm = criterion?.checkMethod;
|
|
632
|
+
const nextAction: IndeterminateNextAction =
|
|
633
|
+
cm === "deterministic" || cm === "hybrid"
|
|
634
|
+
? "code-recheck"
|
|
635
|
+
: bb?.kind === AssumptionKind.FactualIntent
|
|
636
|
+
? "2nd-judge"
|
|
637
|
+
: bb?.kind === AssumptionKind.Normative
|
|
638
|
+
? "revise-criterion"
|
|
639
|
+
: "hitl-spot-check";
|
|
640
|
+
const reason =
|
|
641
|
+
bb?.text && bb.text.trim().length > 0
|
|
642
|
+
? bb.text.trim()
|
|
643
|
+
: "judge abstained — evidence present but inconclusive; needs a confirming pass";
|
|
644
|
+
return { criterionId: cid, resolutionClass: "needs-evidence", nextAction, reason };
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/** WS-5 — the precondition TRIGGER (tool/event) a criterion requires before it applies,
|
|
648
|
+
* when derivable. RESERVED hook: `ReportCriterion` carries no codeEval today, so this
|
|
649
|
+
* returns undefined and the deterministic gate runs off the judge's dense na-map + the
|
|
650
|
+
* typed `blockedBy`. A codeEval-bearing criterion (discover-side) can override later. */
|
|
651
|
+
function preconditionTrigger(_criterion: ReportCriterion | undefined): string | undefined {
|
|
652
|
+
return undefined;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
// ── §9.4.4 R3 — per-tab "what this tab shows" description ─────────────────────
|
|
656
|
+
/** A standardized tab-description banner (R3): every tab states what it shows. */
|
|
657
|
+
function tabDesc(text: string): string {
|
|
658
|
+
return `<div class="tabdesc"><span class="td-i">ⓘ</span> ${esc(text)}</div>`;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// ── §9.4.4 R4 — criterion provenance + human-readable statement + hover ───────
|
|
662
|
+
/**
|
|
663
|
+
* The provenance chip for a criterion (R4): `defined` (authored matrix row) vs
|
|
664
|
+
* `source` (mined from the trace ✓/✗ split), with a hover title carrying the origin.
|
|
665
|
+
*/
|
|
666
|
+
function provChip(c: ReportCriterion | undefined): string {
|
|
667
|
+
const p = c?.provenance;
|
|
668
|
+
if (p === undefined) return "";
|
|
669
|
+
const cls = p.kind === "defined" ? "defined" : "source";
|
|
670
|
+
const title = `${p.label}${p.detail ? " — " + p.detail : ""}`;
|
|
671
|
+
return `<span class="prov ${cls}" title="${esc(title)}">${esc(p.kind)}</span>`;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* UI-10 — the CODE vs JUDGE chip for a criterion, from its `checkMethod` router:
|
|
676
|
+
* `deterministic` → CODE (a deterministic code-eval — zero judge tokens) · `hybrid`
|
|
677
|
+
* → CODE+JUDGE (code pre-filter, then judge on the residual) · else JUDGE (scored by
|
|
678
|
+
* the critique-before-verdict LLM judge). Lets a reader tell at a glance HOW each
|
|
679
|
+
* criterion is evaluated — code-checked vs judged. Absent checkMethod ⇒ no chip.
|
|
680
|
+
*/
|
|
681
|
+
function methodKind(c: ReportCriterion | undefined): "code" | "hybrid" | "judge" | undefined {
|
|
682
|
+
const m = c?.checkMethod;
|
|
683
|
+
if (m === undefined) return undefined;
|
|
684
|
+
return m === "deterministic" ? "code" : m === "hybrid" ? "hybrid" : "judge";
|
|
685
|
+
}
|
|
686
|
+
function methodChip(c: ReportCriterion | undefined): string {
|
|
687
|
+
const kind = methodKind(c);
|
|
688
|
+
if (kind === undefined) return "";
|
|
689
|
+
const label = kind === "code" ? "CODE" : kind === "hybrid" ? "CODE+JUDGE" : "JUDGE";
|
|
690
|
+
const title =
|
|
691
|
+
kind === "code"
|
|
692
|
+
? "code-eval — deterministic check, no LLM judge (zero judge tokens)"
|
|
693
|
+
: kind === "hybrid"
|
|
694
|
+
? "hybrid — deterministic code pre-filter, then LLM judge on the residual"
|
|
695
|
+
: "llm-judge — scored by the critique-before-verdict LLM judge";
|
|
696
|
+
return `<span class="method ${kind}" title="${esc(title)}">${label}</span>`;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* An info-hover badge carrying the criterion's FULL definition (R4): the human-
|
|
701
|
+
* readable statement + pass condition + provenance. Rendered as a `title=`-tooltip
|
|
702
|
+
* on a small ⓘ glyph so every criterion id is self-describing on hover.
|
|
703
|
+
*/
|
|
704
|
+
function critHover(c: ReportCriterion | undefined): string {
|
|
705
|
+
if (c === undefined) return "";
|
|
706
|
+
const kind = methodKind(c);
|
|
707
|
+
const parts = [
|
|
708
|
+
`STATEMENT: ${c.statement}`,
|
|
709
|
+
c.passCondition ? `PASS WHEN: ${c.passCondition}` : "",
|
|
710
|
+
// UI-10 — the code-vs-judge method, spelled out in the hover.
|
|
711
|
+
kind !== undefined ? `EVALUATED BY: ${kind === "code" ? "CODE (deterministic check)" : kind === "hybrid" ? "HYBRID (code pre-filter + judge)" : "LLM JUDGE"}` : "",
|
|
712
|
+
c.provenance ? `PROVENANCE: ${c.provenance.label}${c.provenance.detail ? " (" + c.provenance.detail + ")" : ""}` : "",
|
|
713
|
+
].filter(Boolean);
|
|
714
|
+
return `<span class="ihover" title="${esc(parts.join("\n"))}">ⓘ</span>`;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/**
|
|
718
|
+
* Gap A — the RAW triggering INPUT for a trace: the thing that fired the subject
|
|
719
|
+
* agent. PRIMARY source is `EvalTrace.input.prompt` (the canonical contract field,
|
|
720
|
+
* contracts/eval-types.ts). FALLBACK is the packet-style `transcript[0]` raw user
|
|
721
|
+
* turn carried on some exports (`input.transcript[0].content|text|message`). An
|
|
722
|
+
* absent / non-string / empty value ⇒ `undefined` (the row gets no input; the
|
|
723
|
+
* drill cell renders "—" — never a fabricated input). PURE, read-only.
|
|
724
|
+
*/
|
|
725
|
+
function traceInputText(t: EvalTrace): string | undefined {
|
|
726
|
+
const input = t.input;
|
|
727
|
+
if (input === undefined || input === null) return undefined;
|
|
728
|
+
// PRIMARY: the canonical `input.prompt`.
|
|
729
|
+
const prompt = (input as { prompt?: unknown }).prompt;
|
|
730
|
+
if (typeof prompt === "string" && prompt.trim().length > 0) return prompt;
|
|
731
|
+
// FALLBACK: a packet/transcript-style first user turn (`transcript[0]`).
|
|
732
|
+
const transcript = (input as { transcript?: unknown }).transcript;
|
|
733
|
+
if (Array.isArray(transcript) && transcript.length > 0) {
|
|
734
|
+
const first = transcript[0] as Record<string, unknown> | string | undefined;
|
|
735
|
+
if (typeof first === "string" && first.trim().length > 0) return first;
|
|
736
|
+
if (first !== null && typeof first === "object") {
|
|
737
|
+
for (const key of ["content", "text", "message", "prompt"] as const) {
|
|
738
|
+
const v = first[key];
|
|
739
|
+
if (typeof v === "string" && v.trim().length > 0) return v;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
return undefined;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* WS-6 — reconstruct the subject's SYSTEM PROMPT from a raw trace. The eval
|
|
748
|
+
* `subjectProfile` carries no `systemPrompt` (0/N verdicts), but the agent's static
|
|
749
|
+
* system message rides on every LLM-call observation as the first `role:"system"`
|
|
750
|
+
* message — so it is RECOVERABLE from the trace batch, not confabulated.
|
|
751
|
+
*
|
|
752
|
+
* Scans, in priority order: (1) each observation's `input` — a bare messages ARRAY
|
|
753
|
+
* (e.g. Langfuse `ai.generateText.doGenerate.input = [{role:"system",content:…}, …]`)
|
|
754
|
+
* OR a `{messages:[…]}` wrapper (the `ai.generateText` SPAN shape); (2) the trace-level
|
|
755
|
+
* `input` in those same two shapes. Returns the FIRST `role:"system"` message's text
|
|
756
|
+
* (string content, or the joined `text` parts of an array content). Returns `undefined`
|
|
757
|
+
* when no system message exists — the caller then leaves the prompt UNAVAILABLE
|
|
758
|
+
* (NEVER fabricated). PURE, read-only over the trace. REUSABLE: the discover-side
|
|
759
|
+
* entity hero can import this to mirror the same reconstruction.
|
|
760
|
+
*/
|
|
761
|
+
export function reconstructSystemPrompt(t: EvalTrace): string | undefined {
|
|
762
|
+
// content → text: a message content is a string OR an array of {type,text} parts.
|
|
763
|
+
const contentText = (content: unknown): string => {
|
|
764
|
+
if (typeof content === "string") return content;
|
|
765
|
+
if (Array.isArray(content)) {
|
|
766
|
+
return content
|
|
767
|
+
.map((part) =>
|
|
768
|
+
part !== null && typeof part === "object"
|
|
769
|
+
? String((part as { text?: unknown }).text ?? "")
|
|
770
|
+
: typeof part === "string"
|
|
771
|
+
? part
|
|
772
|
+
: "",
|
|
773
|
+
)
|
|
774
|
+
.join("");
|
|
775
|
+
}
|
|
776
|
+
return "";
|
|
777
|
+
};
|
|
778
|
+
// scan a message ARRAY for the first role:"system" → its (non-empty) content text.
|
|
779
|
+
const fromMessages = (msgs: unknown): string | undefined => {
|
|
780
|
+
if (!Array.isArray(msgs)) return undefined;
|
|
781
|
+
for (const m of msgs) {
|
|
782
|
+
if (m !== null && typeof m === "object" && (m as { role?: unknown }).role === "system") {
|
|
783
|
+
const txt = contentText((m as { content?: unknown }).content).trim();
|
|
784
|
+
if (txt.length > 0) return txt;
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
return undefined;
|
|
788
|
+
};
|
|
789
|
+
// a container is either a bare messages array OR a {messages:[…]} wrapper.
|
|
790
|
+
const fromContainer = (c: unknown): string | undefined => {
|
|
791
|
+
const direct = fromMessages(c);
|
|
792
|
+
if (direct !== undefined) return direct;
|
|
793
|
+
if (c !== null && typeof c === "object" && !Array.isArray(c)) {
|
|
794
|
+
return fromMessages((c as { messages?: unknown }).messages);
|
|
795
|
+
}
|
|
796
|
+
return undefined;
|
|
797
|
+
};
|
|
798
|
+
for (const o of t.observations ?? []) {
|
|
799
|
+
const hit = fromContainer(o.input);
|
|
800
|
+
if (hit !== undefined) return hit;
|
|
801
|
+
}
|
|
802
|
+
return fromContainer(t.input);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* UI-9 (§9.4.5 E3) — the trace WALL-CLOCK timestamp, sourced from the RAW input
|
|
807
|
+
* prompt's `<current_time>…ISO…</current_time>` tag. Real agent harnesses stamp the
|
|
808
|
+
* wall-clock into the prompt they fire the subject with (e.g. a sample subject:
|
|
809
|
+
* `<current_time>2026-05-05T18:39:19.934Z</current_time>`), so the time the run
|
|
810
|
+
* happened is GROUND-TRUTH in the trace — NOT a judge-emit field. The eval-health
|
|
811
|
+
* heatmap buckets pass-rate by this timestamp. TOLERANT: returns the inner ISO when
|
|
812
|
+
* it parses as a real date (never a fabricated stamp); absent/garbage ⇒ undefined
|
|
813
|
+
* (the row lands in the heatmap data-pending bucket). PURE, read-only over traces.
|
|
814
|
+
*/
|
|
815
|
+
export function traceTimestamp(t: EvalTrace): string | undefined {
|
|
816
|
+
const raw = traceInputText(t);
|
|
817
|
+
if (raw === undefined) return undefined;
|
|
818
|
+
const m = raw.match(/<current_time>\s*([^<]+?)\s*<\/current_time>/i);
|
|
819
|
+
if (m === null) return undefined;
|
|
820
|
+
const iso = m[1].trim();
|
|
821
|
+
// validate: a real, date-parseable stamp of ISO-ish length — never fake.
|
|
822
|
+
if (iso.length < 10 || Number.isNaN(Date.parse(iso))) return undefined;
|
|
823
|
+
return iso;
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
/** A criterion's human-readable one-line statement (R4), truncated for table rows. */
|
|
827
|
+
function critStatement(c: ReportCriterion | undefined, max = 88): string {
|
|
828
|
+
const s = c?.statement ?? "";
|
|
829
|
+
return s.length > max ? s.slice(0, max - 1) + "…" : s;
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/** Coerce a tolerant context band (string | string[]) to readable strings — real
|
|
833
|
+
* judges emit exit-states / scenarios as a LIST; join them for the §2 band. */
|
|
834
|
+
function coerceContext(
|
|
835
|
+
ctx: { harness?: unknown; scenario?: unknown; exitStates?: unknown },
|
|
836
|
+
): { harness?: string; scenario?: string; exitStates?: string } {
|
|
837
|
+
// TOLERANT — real judges emit exitStates/scenario as a string, a string[], OR a
|
|
838
|
+
// structured object (e.g. `{steps: 7}`). Coerce all three to a readable string so
|
|
839
|
+
// the §2 drill never renders a raw "[object Object]" (WS-1: the rich verdict files
|
|
840
|
+
// that carry an object exitStates are exactly the ones with the judge_steps walk).
|
|
841
|
+
const one = (v: unknown): string | undefined =>
|
|
842
|
+
v === undefined || v === null
|
|
843
|
+
? undefined
|
|
844
|
+
: Array.isArray(v)
|
|
845
|
+
? v.join(" · ")
|
|
846
|
+
: typeof v === "object"
|
|
847
|
+
? Object.entries(v as Record<string, unknown>)
|
|
848
|
+
.map(([k, val]) => `${k}: ${val}`)
|
|
849
|
+
.join(", ")
|
|
850
|
+
: String(v);
|
|
851
|
+
return {
|
|
852
|
+
...(ctx.harness !== undefined ? { harness: one(ctx.harness) } : {}),
|
|
853
|
+
...(ctx.scenario !== undefined ? { scenario: one(ctx.scenario) } : {}),
|
|
854
|
+
...(ctx.exitStates !== undefined ? { exitStates: one(ctx.exitStates) } : {}),
|
|
855
|
+
};
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
// ── criterion adapters (mined ⇄ matrix → ReportCriterion) ────────────────────
|
|
859
|
+
|
|
860
|
+
const GATING_SEVERITIES = new Set(["CRIT", "HIGH"]);
|
|
861
|
+
|
|
862
|
+
/** A mined criterion carries the full §5b metadata + §5c DR-2 rationale. Its
|
|
863
|
+
* provenance is `source` (R4): mined from the ✓/✗ trace split. */
|
|
864
|
+
export function fromMinedCriteria(criteria: MinedCriterion[]): ReportCriterion[] {
|
|
865
|
+
return criteria.map((c) => ({
|
|
866
|
+
id: c.id,
|
|
867
|
+
statement: c.statement,
|
|
868
|
+
severity: c.metadata.severity,
|
|
869
|
+
gating: GATING_SEVERITIES.has(c.metadata.severity),
|
|
870
|
+
dimension: c.metadata.dimension,
|
|
871
|
+
level: c.metadata.level,
|
|
872
|
+
appliesTo: c.metadata.applies_to,
|
|
873
|
+
// UI-10 — carry the code-vs-judge router (default llm-judge when the metadata omits it).
|
|
874
|
+
checkMethod: c.metadata.check_method ?? "llm-judge",
|
|
875
|
+
...(c.metadata.judge_inputs !== undefined ? { judgeInputs: c.metadata.judge_inputs } : {}),
|
|
876
|
+
provenance: {
|
|
877
|
+
kind: "source" as const,
|
|
878
|
+
label: "mined from trace ✓/✗ split",
|
|
879
|
+
...(c.discovery !== undefined
|
|
880
|
+
? { detail: `grounding ${c.discovery.evidence.grounding} · seen in ${c.discovery.evidence.seen_in_traces} trace(s)` }
|
|
881
|
+
: {}),
|
|
882
|
+
},
|
|
883
|
+
discovery: c.discovery,
|
|
884
|
+
}));
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
/** A matrix criterion carries statement + pass condition + severity. Its provenance
|
|
888
|
+
* is `defined` (R4): an authored eval-matrix row. */
|
|
889
|
+
export function fromMatrixCriteria(criteria: MatrixCriterion[]): ReportCriterion[] {
|
|
890
|
+
return criteria.map((c) => ({
|
|
891
|
+
id: c.criterionId,
|
|
892
|
+
statement: c.statement,
|
|
893
|
+
severity: c.severity,
|
|
894
|
+
gating: GATING_SEVERITIES.has(c.severity),
|
|
895
|
+
passCondition: c.passCondition,
|
|
896
|
+
// UI-10 — carry the code-vs-judge router (default llm-judge when the matrix row omits it).
|
|
897
|
+
checkMethod: c.checkMethod ?? "llm-judge",
|
|
898
|
+
provenance: { kind: "defined" as const, label: "defined eval-matrix criterion" },
|
|
899
|
+
...(c.dimension !== undefined ? { dimension: c.dimension } : {}),
|
|
900
|
+
...(c.judgeInputs !== undefined ? { judgeInputs: c.judgeInputs } : {}),
|
|
901
|
+
}));
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
// ── D-1 terminal eval-cards ─────────────────────────────────────────────────
|
|
905
|
+
|
|
906
|
+
/**
|
|
907
|
+
* Render per-criterion terminal cards from the Scorecard: criterion · verdict ·
|
|
908
|
+
* severity · critique snippet · the §5c DR-2 grounding tag (when mined). Plain
|
|
909
|
+
* text, deterministic (criteria in `verdicts` order). The D-1 dogfood fix.
|
|
910
|
+
*/
|
|
911
|
+
export function renderEvalCards(input: EvalReportInput): string {
|
|
912
|
+
const lines: string[] = [];
|
|
913
|
+
const g = input.scorecard.gate;
|
|
914
|
+
lines.push(`╔═ EVAL CARDS — ${input.subject.name} ═══`);
|
|
915
|
+
lines.push(`║ GATE: ${g.passed ? "PASS" : (g.runVerdict ?? "fail").toUpperCase()} (${g.passCount}/${g.total} pass, ${g.gatedBy.length} gating)`);
|
|
916
|
+
lines.push("╠══════════════════════════════════════");
|
|
917
|
+
for (const v of input.verdicts) {
|
|
918
|
+
const crit = criterionById(input, v.criterionId);
|
|
919
|
+
const sev = crit?.severity ?? "MED";
|
|
920
|
+
// UI-10 — the code-vs-judge tag, so the terminal cards distinguish code-checked
|
|
921
|
+
// criteria from judged ones (CODE · HYBRID · JUDGE). Default JUDGE when absent.
|
|
922
|
+
const mkind = methodKind(crit);
|
|
923
|
+
const method = mkind === "code" ? "CODE" : mkind === "hybrid" ? "HYBRID" : "JUDGE";
|
|
924
|
+
const grounding = crit?.discovery?.evidence.grounding ?? Grounding.Inferred;
|
|
925
|
+
const prevalence = crit?.discovery?.evidence.prevalence ?? "n/a";
|
|
926
|
+
const snippet = v.critique.length > 80 ? v.critique.slice(0, 77) + "…" : v.critique;
|
|
927
|
+
lines.push(`║ ▸ ${v.criterionId} [${sev}·${method}] ${resultToken(v.result).toUpperCase()} (conf ${v.confidence})`);
|
|
928
|
+
lines.push(`║ grounding: ${grounding} (${prevalence})`);
|
|
929
|
+
lines.push(`║ ${snippet}`);
|
|
930
|
+
}
|
|
931
|
+
lines.push("╚══════════════════════════════════════");
|
|
932
|
+
return lines.join("\n");
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
// ── wiring: build the rich report input from real pipeline data ──────────────
|
|
936
|
+
|
|
937
|
+
function ledgerVerdict(perCriterion: Record<string, string>): string {
|
|
938
|
+
const vals = Object.values(perCriterion).filter((v) => v !== "na");
|
|
939
|
+
if (vals.includes(OutcomeVerdict.Fail)) return "FAIL";
|
|
940
|
+
if (vals.includes(OutcomeVerdict.Uncertain)) return "INDETERMINATE";
|
|
941
|
+
if (vals.length === 0) return "INCOMPLETE";
|
|
942
|
+
return "PASS";
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
/** Optional §9.4 trajectory-level fields the judge MAY emit (additive). */
|
|
946
|
+
interface MatrixVerdictFileRich extends MatrixVerdictFile {
|
|
947
|
+
route?: string;
|
|
948
|
+
context?: { harness?: string | string[]; scenario?: string | string[]; exitStates?: string | string[] };
|
|
949
|
+
agentSteps?: AgentStep[];
|
|
950
|
+
judgeSteps?: JudgeStep[];
|
|
951
|
+
localize?: string;
|
|
952
|
+
health?: JudgeHealthRow;
|
|
953
|
+
// §9.4.4 v2.2
|
|
954
|
+
understanding?: Understanding;
|
|
955
|
+
expectedTrajectory?: ExpectedStep[];
|
|
956
|
+
subjectProfile?: SubjectProfile;
|
|
957
|
+
/** §9.4.5 E3 — the trace wall-clock timestamp (ISO), when the source carried it.
|
|
958
|
+
* Common spellings tolerated below (`timestamp` · `traceTimestamp` · `startTime`). */
|
|
959
|
+
timestamp?: string;
|
|
960
|
+
traceTimestamp?: string;
|
|
961
|
+
startTime?: string;
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
/**
|
|
965
|
+
* Best-effort carry of an observation id onto each `AgentStep` so the §2 judge
|
|
966
|
+
* lane can map a step to the criterion verdicts that EXAMINED it by the precise
|
|
967
|
+
* `ref.obs === step.obs` key (on top of the tool-name fallback). The verdict
|
|
968
|
+
* files DON'T emit a per-step obs id — but the per-criterion verdict `refs` and
|
|
969
|
+
* critiques DO carry obs ids bound to a tool:
|
|
970
|
+
* (a) `refs[]` with `path:"name"` → `{obs:<id>, value:<toolName>}`.
|
|
971
|
+
* (b) `tool@<obsid>` tokens the judge embeds in `critique` text
|
|
972
|
+
* (e.g. "sendMessage@04ff44e20dbacb37") — the ONLY source for tools that
|
|
973
|
+
* were cited only by an `output.*` ref (which carries no tool name).
|
|
974
|
+
* We gather tool → ordered-unique obs ids from both, then assign positionally per
|
|
975
|
+
* step (nth call of a tool → nth obs id). A tool with no recovered obs id leaves
|
|
976
|
+
* the step's obs UNDEFINED (honest — the tool-name match still covers it). PURE.
|
|
977
|
+
*/
|
|
978
|
+
export function enrichAgentStepsObs(f: {
|
|
979
|
+
agentSteps?: AgentStep[];
|
|
980
|
+
verdicts?: Array<{ critique?: string; refs?: Array<{ obs?: string; path?: string; value?: string }> }>;
|
|
981
|
+
}): AgentStep[] | undefined {
|
|
982
|
+
const steps = f.agentSteps;
|
|
983
|
+
if (steps === undefined) return undefined;
|
|
984
|
+
const toolObs = new Map<string, string[]>();
|
|
985
|
+
const push = (tool: string, obs: string): void => {
|
|
986
|
+
if (!tool || !obs) return;
|
|
987
|
+
const list = toolObs.get(tool) ?? [];
|
|
988
|
+
if (!list.includes(obs)) {
|
|
989
|
+
list.push(obs);
|
|
990
|
+
toolObs.set(tool, list);
|
|
991
|
+
}
|
|
992
|
+
};
|
|
993
|
+
for (const v of f.verdicts ?? []) {
|
|
994
|
+
for (const rf of v.refs ?? []) {
|
|
995
|
+
if (rf && typeof rf === "object" && rf.path === "name" && rf.value && rf.obs) {
|
|
996
|
+
push(String(rf.value), String(rf.obs));
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
// `tool@<obs>` tokens in the critique (obs ids are 6+ hex chars).
|
|
1000
|
+
const crit = typeof v.critique === "string" ? v.critique : "";
|
|
1001
|
+
const re = /([A-Za-z_]\w*)@([0-9a-f]{6,})/g;
|
|
1002
|
+
let m: RegExpExecArray | null;
|
|
1003
|
+
while ((m = re.exec(crit)) !== null) push(m[1], m[2]);
|
|
1004
|
+
}
|
|
1005
|
+
if (toolObs.size === 0) return steps;
|
|
1006
|
+
const counter: Record<string, number> = {};
|
|
1007
|
+
return steps.map((s) => {
|
|
1008
|
+
const tool = s.tool ?? "";
|
|
1009
|
+
const list = toolObs.get(tool);
|
|
1010
|
+
if (list === undefined || list.length === 0) return s;
|
|
1011
|
+
const i = counter[tool] ?? 0;
|
|
1012
|
+
counter[tool] = i + 1;
|
|
1013
|
+
const obs = list[i];
|
|
1014
|
+
return obs !== undefined ? { ...s, obs } : s;
|
|
1015
|
+
});
|
|
1016
|
+
}
|
|
1017
|
+
|
|
1018
|
+
/**
|
|
1019
|
+
* Derive the rich `EvalReportInput` fields (ledger · coverage · cohorts · gating
|
|
1020
|
+
* rolls · top findings · judge-health) from the REAL pipeline outputs — the
|
|
1021
|
+
* per-trajectory Judge-Agent verdict files + the folded scorecard + the criteria.
|
|
1022
|
+
* Invents NO data: every cell is read from a verdict file or the scorecard.
|
|
1023
|
+
*
|
|
1024
|
+
* §9.4-judge-emit CLOSED (T2): the JUDGE AGENT now EMITS the side-by-side fields
|
|
1025
|
+
* (`judgeSteps`, `agentSteps`, `context`, `localize`, `health`, `route`), a dense
|
|
1026
|
+
* na-explicit `denseMap`, a `fidelity` gate, and per-verdict `confidenceBand` —
|
|
1027
|
+
* mandated in `evaluator.md #mode-judge-trajectory`. The renderer uses an emitted
|
|
1028
|
+
* `denseMap` verbatim when present and otherwise synthesizes one from `verdicts[]`
|
|
1029
|
+
* (na for any unjudged row), so a judge that omits the walk still renders. PURE.
|
|
1030
|
+
*/
|
|
1031
|
+
export function buildEvalReportInput(params: {
|
|
1032
|
+
subject: EvalReportInput["subject"];
|
|
1033
|
+
scorecard: Scorecard;
|
|
1034
|
+
verdicts: CriterionVerdict[];
|
|
1035
|
+
criteria: ReportCriterion[];
|
|
1036
|
+
matrixVerdictFiles?: MatrixVerdictFile[];
|
|
1037
|
+
sourceMap?: SourceMap;
|
|
1038
|
+
handover?: HandoverBundle | null;
|
|
1039
|
+
generatedAt: string;
|
|
1040
|
+
triaged?: number;
|
|
1041
|
+
/** §9.4.4 M1/R2 — the subject profile (given or reconstructed) for the report. */
|
|
1042
|
+
subjectProfile?: SubjectProfile;
|
|
1043
|
+
/** Overview provenance meta-strip — the run config (judge substrate · pinned model · …). */
|
|
1044
|
+
runConfig?: RunConfig;
|
|
1045
|
+
/**
|
|
1046
|
+
* Gap A — the INGESTED trace batch (the EvalTraces under evaluation). Used ONLY
|
|
1047
|
+
* to source each ledger row's RAW triggering INPUT (`EvalTrace.input.prompt`,
|
|
1048
|
+
* keyed by trace id == `MatrixVerdictFile.trajectoryId`). The input is
|
|
1049
|
+
* ground-truth in the trace — NOT a judge-emit field, so this stays non-protocol
|
|
1050
|
+
* (the judge schema is untouched). ABSENT ⇒ rows carry no `input` and the §2
|
|
1051
|
+
* drill cell renders "—" (graceful, never faked). PURE — read-only over traces.
|
|
1052
|
+
*/
|
|
1053
|
+
traces?: EvalTrace[];
|
|
1054
|
+
}): EvalReportInput {
|
|
1055
|
+
const files = (params.matrixVerdictFiles ?? []) as MatrixVerdictFileRich[];
|
|
1056
|
+
const critIds = params.criteria.map((c) => c.id);
|
|
1057
|
+
// Gap A — index the raw triggering INPUT by trace id (== trajectoryId). Read
|
|
1058
|
+
// `EvalTrace.input.prompt`; an absent/empty input leaves the row without `input`
|
|
1059
|
+
// (the drill renders "—"). NO judge echo — the input is read straight from the
|
|
1060
|
+
// trace the verdict was produced over.
|
|
1061
|
+
const inputByTrace = new Map<string, string>();
|
|
1062
|
+
// UI-9 (§9.4.5 E3) — index the wall-clock timestamp by trace id (== trajectoryId),
|
|
1063
|
+
// extracted from the trace's `<current_time>` prompt stamp. This is the REAL data
|
|
1064
|
+
// path the eval-health heatmap needs; verdict files carry no timestamp, but the
|
|
1065
|
+
// ingested traces do. Mirror inputByTrace; NO contract change.
|
|
1066
|
+
const timestampByTrace = new Map<string, string>();
|
|
1067
|
+
for (const t of params.traces ?? []) {
|
|
1068
|
+
const raw = traceInputText(t);
|
|
1069
|
+
if (raw !== undefined) inputByTrace.set(t.id, raw);
|
|
1070
|
+
const ts = traceTimestamp(t);
|
|
1071
|
+
if (ts !== undefined) timestampByTrace.set(t.id, ts);
|
|
1072
|
+
}
|
|
1073
|
+
const sevById = new Map(params.criteria.map((c) => [c.id, c.severity]));
|
|
1074
|
+
const critById = new Map(params.criteria.map((c) => [c.id, c]));
|
|
1075
|
+
const gatingIds = new Set(params.criteria.filter((c) => c.gating).map((c) => c.id));
|
|
1076
|
+
|
|
1077
|
+
// ── ledger: one dense row per trajectory ──
|
|
1078
|
+
const ledger: LedgerRow[] = files.map((f) => {
|
|
1079
|
+
const perCriterion: Record<string, string> = {};
|
|
1080
|
+
for (const cid of critIds) perCriterion[cid] = "na"; // dense, na-explicit
|
|
1081
|
+
// §9.4.2 node 9: prefer the judge's EMITTED dense map (na-explicit) verbatim;
|
|
1082
|
+
// else synthesize from verdicts[] (any unjudged row stays na).
|
|
1083
|
+
const emittedDense = (f as { denseMap?: Record<string, string> }).denseMap;
|
|
1084
|
+
if (emittedDense !== undefined) {
|
|
1085
|
+
for (const cid of critIds) if (emittedDense[cid] !== undefined) perCriterion[cid] = emittedDense[cid];
|
|
1086
|
+
}
|
|
1087
|
+
for (const v of f.verdicts) perCriterion[v.criterionId] = v.result;
|
|
1088
|
+
// WS-5 — the INDETERMINATE resolution chain: deterministically resolve every
|
|
1089
|
+
// uncertain per-criterion verdict into `na` (precondition absent — DROPPED from the
|
|
1090
|
+
// applicable denominator) or `needs-evidence` (+ a next-action). A precondition-
|
|
1091
|
+
// absent uncertain is RECLASSIFIED to `na` in `perCriterion` so EVERY downstream
|
|
1092
|
+
// count (rolls · cohorts · heatmap · trace verdict) becomes denominator-honest —
|
|
1093
|
+
// no bare "indeterminate" that conflates not-applicable with under-evidenced.
|
|
1094
|
+
const verdictByCid = new Map(
|
|
1095
|
+
f.verdicts.map((v) => [v.criterionId, v as { result?: string; blockedBy?: { kind?: string; text?: string } }]),
|
|
1096
|
+
);
|
|
1097
|
+
const indeterminateResolution: Record<string, IndeterminateResolution> = {};
|
|
1098
|
+
for (const cid of critIds) {
|
|
1099
|
+
if (perCriterion[cid] === OutcomeVerdict.Uncertain) {
|
|
1100
|
+
const res = classifyIndeterminate(critById.get(cid), verdictByCid.get(cid), emittedDense?.[cid], f.agentSteps);
|
|
1101
|
+
indeterminateResolution[cid] = res;
|
|
1102
|
+
if (res.resolutionClass === "na") perCriterion[cid] = "na";
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
const failingCriteria = f.verdicts.filter((v) => v.result === OutcomeVerdict.Fail).map((v) => v.criterionId);
|
|
1106
|
+
const grounding = f.verdicts
|
|
1107
|
+
.flatMap((v) => v.refs ?? [])
|
|
1108
|
+
.map((r) => `${r.obs}/${r.path}`)
|
|
1109
|
+
.slice(0, 3)
|
|
1110
|
+
.join(" · ");
|
|
1111
|
+
// #3 EXAMINE fidelity gate: a trajectory whose deterministic fidelity gate
|
|
1112
|
+
// fired (truncated / unterminated) is INCOMPLETE regardless of its dense map —
|
|
1113
|
+
// it was never walked per-criterion.
|
|
1114
|
+
const isIncomplete = (f as { fidelity?: { complete?: boolean } }).fidelity?.complete === false;
|
|
1115
|
+
const row: LedgerRow = {
|
|
1116
|
+
trajectoryId: f.trajectoryId,
|
|
1117
|
+
verdict: isIncomplete ? "INCOMPLETE" : ledgerVerdict(perCriterion),
|
|
1118
|
+
perCriterion,
|
|
1119
|
+
failingCriteria,
|
|
1120
|
+
...(f.route !== undefined ? { route: f.route } : {}),
|
|
1121
|
+
...(grounding ? { grounding } : {}),
|
|
1122
|
+
// WS-1 — bind the per-criterion JUDGE verdicts (result · critique · refs) for
|
|
1123
|
+
// THIS trajectory, read verbatim off the verdict file's `verdicts[]`. This is
|
|
1124
|
+
// the rich "How the Judge Reasoned" payload that EVERY judged trace carries
|
|
1125
|
+
// (PASS or FAIL), independent of whether a judge_steps[] walk was emitted —
|
|
1126
|
+
// so the §2 drill never renders an empty reasoning panel when the verdict file
|
|
1127
|
+
// has data. NO invention: a row with no verdicts simply omits the field.
|
|
1128
|
+
...(f.verdicts.length > 0
|
|
1129
|
+
? { criterionVerdicts: f.verdicts as unknown as CriterionTrajectoryVerdict[] }
|
|
1130
|
+
: {}),
|
|
1131
|
+
// Gap A — the raw triggering INPUT for this trajectory, sourced from the
|
|
1132
|
+
// trace by id (== trajectoryId). ABSENT ⇒ the drill cell renders "—".
|
|
1133
|
+
...(inputByTrace.has(f.trajectoryId) ? { input: inputByTrace.get(f.trajectoryId) } : {}),
|
|
1134
|
+
...(f.context !== undefined ? { context: coerceContext(f.context) } : {}),
|
|
1135
|
+
// Carry recovered obs ids onto each agent step so the §2 judge lane can map
|
|
1136
|
+
// a step → the criterion verdicts that examined it by `ref.obs === step.obs`.
|
|
1137
|
+
...(f.agentSteps !== undefined ? { agentSteps: enrichAgentStepsObs(f) } : {}),
|
|
1138
|
+
...(f.judgeSteps !== undefined ? { judgeSteps: f.judgeSteps } : {}),
|
|
1139
|
+
...(f.localize !== undefined ? { localize: localizeText(f.localize) } : {}),
|
|
1140
|
+
// §9.4.4 v2.2 — node-0 understanding · node-0.5 expected-trajectory · M1 profile.
|
|
1141
|
+
...(f.understanding !== undefined ? { understanding: f.understanding } : {}),
|
|
1142
|
+
...(f.expectedTrajectory !== undefined ? { expectedTrajectory: f.expectedTrajectory } : {}),
|
|
1143
|
+
...(f.subjectProfile !== undefined ? { subjectProfile: f.subjectProfile } : {}),
|
|
1144
|
+
// T4: DERIVE health from the walk when judge_steps were emitted (don't trust
|
|
1145
|
+
// the self-reported `health`); fall back to the self-reported field otherwise.
|
|
1146
|
+
...((f.judgeSteps?.length ?? 0) > 0
|
|
1147
|
+
? { health: deriveWalkHealth(f) }
|
|
1148
|
+
: f.health !== undefined
|
|
1149
|
+
? { health: f.health }
|
|
1150
|
+
: {}),
|
|
1151
|
+
// §9.4.5 E3 / UI-9 — carry the trace timestamp for the eval-HEALTH temporal
|
|
1152
|
+
// heatmap. PRIMARY = the trace's `<current_time>` wall-clock (real ground-truth,
|
|
1153
|
+
// sourced from params.traces); FALLBACK = a verdict-file timestamp (tolerant
|
|
1154
|
+
// spellings) if a judge ever emits one. ABSENT ⇒ this row lands in the
|
|
1155
|
+
// data-pending bucket (never faked).
|
|
1156
|
+
...(((timestampByTrace.get(f.trajectoryId) ?? f.timestamp ?? f.traceTimestamp ?? f.startTime) !== undefined)
|
|
1157
|
+
? { timestamp: timestampByTrace.get(f.trajectoryId) ?? f.timestamp ?? f.traceTimestamp ?? f.startTime }
|
|
1158
|
+
: {}),
|
|
1159
|
+
// WS-5 — the per-criterion indeterminate resolution for this trace (na vs needs-evidence).
|
|
1160
|
+
...(Object.keys(indeterminateResolution).length > 0 ? { indeterminateResolution } : {}),
|
|
1161
|
+
// WS-2 — how this trajectory resolved (judge-walk · judged-walk-not-captured ·
|
|
1162
|
+
// truncated). Makes the "no walk" traces HONEST: judged, not unjudged.
|
|
1163
|
+
resolution: deriveTrajectoryResolution(f),
|
|
1164
|
+
};
|
|
1165
|
+
return row;
|
|
1166
|
+
});
|
|
1167
|
+
|
|
1168
|
+
// ── §9.4.5 E2 — per-tool OBSERVED call-count (from the trajectory agent-steps). In
|
|
1169
|
+
// the trace-only (no-code-access) case this is the ONLY honest tool evidence: how
|
|
1170
|
+
// often each tool was actually seen called across the judged trajectories. PURE. ──
|
|
1171
|
+
const toolObservations: Record<string, number> = {};
|
|
1172
|
+
for (const f of files) {
|
|
1173
|
+
for (const step of f.agentSteps ?? []) {
|
|
1174
|
+
const name = step.tool;
|
|
1175
|
+
if (typeof name === "string" && name.length > 0) {
|
|
1176
|
+
toolObservations[name] = (toolObservations[name] ?? 0) + 1;
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
// ── §9.4.4 R3/M5 — collect the DETECTED-but-unmatched flags (node-2.5 candidates),
|
|
1182
|
+
// surfaced in §4 SEPARATED from criterion fails (a detection, never a verdict). ──
|
|
1183
|
+
const detectedFlags: DetectedFlag[] = [];
|
|
1184
|
+
for (const f of files) {
|
|
1185
|
+
for (const cand of f.candidates ?? []) {
|
|
1186
|
+
detectedFlags.push({
|
|
1187
|
+
kind: cand.kind,
|
|
1188
|
+
detection: cand.detection,
|
|
1189
|
+
trajectoryId: f.trajectoryId,
|
|
1190
|
+
...(cand.anchor !== undefined ? { anchor: cand.anchor } : {}),
|
|
1191
|
+
...(cand.ref !== undefined ? { ref: cand.ref } : {}),
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
// §9.4.4 M1/R2 — the subject profile: prefer the run-supplied one, else the first
|
|
1196
|
+
// one a judge echoed/reconstructed on its verdict file.
|
|
1197
|
+
const subjectProfileBase = params.subjectProfile ?? files.find((f) => f.subjectProfile !== undefined)?.subjectProfile;
|
|
1198
|
+
// WS-6 — reconstruct the SYSTEM PROMPT from the trace batch when the profile lacks
|
|
1199
|
+
// one. The eval subjectProfile never carries `systemPrompt`, but the agent's static
|
|
1200
|
+
// system message rides on each LLM-call observation (role:"system"). Extract it ONCE
|
|
1201
|
+
// from the first trace that yields one, attach it, and TAG it reconstructed (added to
|
|
1202
|
+
// `inferredFields` ⇒ the entity hero shows the "system prompt · reconstructed"
|
|
1203
|
+
// collapsible). NEVER confabulate: if no trace carries a system message, leave it
|
|
1204
|
+
// UNAVAILABLE. If no profile exists at all but a system prompt is recoverable, mint a
|
|
1205
|
+
// MINIMAL reconstructed profile so the recovered prompt still surfaces.
|
|
1206
|
+
let reconstructedSystemPrompt: string | undefined;
|
|
1207
|
+
if (subjectProfileBase?.systemPrompt === undefined || subjectProfileBase.systemPrompt.trim() === "") {
|
|
1208
|
+
for (const t of params.traces ?? []) {
|
|
1209
|
+
const sys = reconstructSystemPrompt(t);
|
|
1210
|
+
if (sys !== undefined) {
|
|
1211
|
+
reconstructedSystemPrompt = sys;
|
|
1212
|
+
break;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
const subjectProfile: SubjectProfile | undefined =
|
|
1217
|
+
reconstructedSystemPrompt !== undefined
|
|
1218
|
+
? ({
|
|
1219
|
+
...(subjectProfileBase ?? { provenance: "reconstructed" }),
|
|
1220
|
+
systemPrompt: reconstructedSystemPrompt,
|
|
1221
|
+
inferredFields: Array.from(
|
|
1222
|
+
new Set([...(subjectProfileBase?.inferredFields ?? []), "systemPrompt"]),
|
|
1223
|
+
),
|
|
1224
|
+
} as SubjectProfile)
|
|
1225
|
+
: subjectProfileBase;
|
|
1226
|
+
|
|
1227
|
+
// ── coverage: byVerdict tally over the ledger ──
|
|
1228
|
+
const byVerdict: Record<string, number> = { PASS: 0, FAIL: 0, INDETERMINATE: 0, INCOMPLETE: 0 };
|
|
1229
|
+
for (const r of ledger) byVerdict[r.verdict] = (byVerdict[r.verdict] ?? 0) + 1;
|
|
1230
|
+
const coverage = {
|
|
1231
|
+
judged: ledger.length,
|
|
1232
|
+
triaged: params.triaged ?? ledger.length,
|
|
1233
|
+
byVerdict,
|
|
1234
|
+
gapNote:
|
|
1235
|
+
ledger.length > 0
|
|
1236
|
+
? `${ledger.length} trajectory verdict(s) folded; every in-scope trajectory carries a per-criterion judgement (0 silently dropped).`
|
|
1237
|
+
: "no per-trajectory verdict files supplied for this render — scorecard-only.",
|
|
1238
|
+
};
|
|
1239
|
+
|
|
1240
|
+
// ── cohorts: criteria × route cohort (route defaults to "all") ──
|
|
1241
|
+
const counts: Record<string, number> = {};
|
|
1242
|
+
const matrix: Record<string, Record<string, { pass: number; fail: number; indeterminate: number }>> = {};
|
|
1243
|
+
for (const r of ledger) {
|
|
1244
|
+
const cohort = r.route ?? "all";
|
|
1245
|
+
counts[cohort] = (counts[cohort] ?? 0) + 1;
|
|
1246
|
+
for (const [cid, res] of Object.entries(r.perCriterion)) {
|
|
1247
|
+
if (res === "na") continue;
|
|
1248
|
+
matrix[cid] ??= {};
|
|
1249
|
+
matrix[cid][cohort] ??= { pass: 0, fail: 0, indeterminate: 0 };
|
|
1250
|
+
if (res === OutcomeVerdict.Pass) matrix[cid][cohort].pass += 1;
|
|
1251
|
+
else if (res === OutcomeVerdict.Fail) matrix[cid][cohort].fail += 1;
|
|
1252
|
+
else matrix[cid][cohort].indeterminate += 1;
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
// ── per-criterion roll (over the ledger). WS-5 — the applicable denominator EXCLUDES
|
|
1257
|
+
// `na` (precondition absent); the remaining `uncertain` are GENUINE needs-evidence
|
|
1258
|
+
// (the precondition-absent uncertains were already reclassified to `na` above). ──
|
|
1259
|
+
function rollOf(cid: string): {
|
|
1260
|
+
applicable: number;
|
|
1261
|
+
pass: number;
|
|
1262
|
+
fail: number;
|
|
1263
|
+
needsEvidence: number;
|
|
1264
|
+
na: number;
|
|
1265
|
+
example?: string;
|
|
1266
|
+
} {
|
|
1267
|
+
let applicable = 0,
|
|
1268
|
+
pass = 0,
|
|
1269
|
+
fail = 0,
|
|
1270
|
+
needsEvidence = 0,
|
|
1271
|
+
na = 0;
|
|
1272
|
+
let example: string | undefined;
|
|
1273
|
+
for (const r of ledger) {
|
|
1274
|
+
const res = r.perCriterion[cid];
|
|
1275
|
+
if (res === undefined) continue;
|
|
1276
|
+
if (res === "na") {
|
|
1277
|
+
na += 1;
|
|
1278
|
+
continue;
|
|
1279
|
+
}
|
|
1280
|
+
applicable += 1;
|
|
1281
|
+
if (res === OutcomeVerdict.Fail) {
|
|
1282
|
+
fail += 1;
|
|
1283
|
+
example ??= r.trajectoryId;
|
|
1284
|
+
} else if (res === OutcomeVerdict.Uncertain) {
|
|
1285
|
+
needsEvidence += 1;
|
|
1286
|
+
example ??= r.trajectoryId;
|
|
1287
|
+
} else pass += 1;
|
|
1288
|
+
}
|
|
1289
|
+
return { applicable, pass, fail, needsEvidence, na, ...(example !== undefined ? { example } : {}) };
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
// #6 — the AUDITABLE second-judge note: surfaces that an INDEPENDENT verifier ran
|
|
1293
|
+
// over a gating fail and what it concluded (upheld vs refuted→downgraded), so the
|
|
1294
|
+
// report shows the refutation result (not just "eligible for"). PURE.
|
|
1295
|
+
function ivNote(folded?: CriterionVerdict): string {
|
|
1296
|
+
const iv = folded?.independentVerify;
|
|
1297
|
+
if (iv === undefined) return "";
|
|
1298
|
+
const who = iv.reviewerId !== undefined ? ` (${iv.reviewerId})` : "";
|
|
1299
|
+
return ` · [INDEPENDENT VERIFY — 2nd judge${who}: ${iv.upheld ? "UPHELD" : "REFUTED → downgraded"}; ${iv.reason}]`;
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
// ── gating-criteria rolls (the gating subset; GA-D2b applicable denominator) ──
|
|
1303
|
+
const gatingCriteria: GatingRoll[] = [...gatingIds]
|
|
1304
|
+
.map((cid) => {
|
|
1305
|
+
const roll = rollOf(cid);
|
|
1306
|
+
const passRate = roll.applicable > 0 ? Math.round((100 * roll.pass) / roll.applicable) : 100;
|
|
1307
|
+
const folded = params.verdicts.find((v) => v.criterionId === cid);
|
|
1308
|
+
return {
|
|
1309
|
+
criterion: cid,
|
|
1310
|
+
severity: sevById.get(cid) ?? "MED",
|
|
1311
|
+
applicable: roll.applicable,
|
|
1312
|
+
fail: roll.fail,
|
|
1313
|
+
indeterminate: roll.needsEvidence,
|
|
1314
|
+
na: roll.na,
|
|
1315
|
+
passRateOverApplicable: passRate,
|
|
1316
|
+
denominatorNote: `N/A excluded — denominator = ${roll.applicable} applicable (na ${roll.na} dropped, GA-D2b)`,
|
|
1317
|
+
...(folded?.critique ? { root: folded.critique + ivNote(folded) } : {}),
|
|
1318
|
+
};
|
|
1319
|
+
})
|
|
1320
|
+
.sort((a, b) => a.passRateOverApplicable - b.passRateOverApplicable);
|
|
1321
|
+
|
|
1322
|
+
// ── WS-5 — per-criterion INDETERMINATE resolution chain (ALL criteria, not just
|
|
1323
|
+
// gating). Splits each criterion's indeterminate into N/A (precondition absent,
|
|
1324
|
+
// denominator-excluded) vs needs-evidence (carries aggregated chain next-actions).
|
|
1325
|
+
// The reasons/next-actions are aggregated from each trace's deterministic
|
|
1326
|
+
// `indeterminateResolution`. ──
|
|
1327
|
+
const criterionResolutions: CriterionResolution[] = params.criteria.map((c) => {
|
|
1328
|
+
const roll = rollOf(c.id);
|
|
1329
|
+
const naReasons: Record<string, number> = {};
|
|
1330
|
+
const actionAgg: Record<string, { count: number; reason: string }> = {};
|
|
1331
|
+
for (const r of ledger) {
|
|
1332
|
+
const res = r.indeterminateResolution?.[c.id];
|
|
1333
|
+
if (res === undefined) continue;
|
|
1334
|
+
if (res.resolutionClass === "na") naReasons[res.reason] = (naReasons[res.reason] ?? 0) + 1;
|
|
1335
|
+
else if (res.nextAction !== undefined) {
|
|
1336
|
+
const a = (actionAgg[res.nextAction] ??= { count: 0, reason: res.reason });
|
|
1337
|
+
a.count += 1;
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
const naReason = Object.entries(naReasons).sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
1341
|
+
return {
|
|
1342
|
+
criterion: c.id,
|
|
1343
|
+
severity: c.severity,
|
|
1344
|
+
applicable: roll.applicable,
|
|
1345
|
+
pass: roll.pass,
|
|
1346
|
+
fail: roll.fail,
|
|
1347
|
+
na: roll.na,
|
|
1348
|
+
needsEvidence: roll.needsEvidence,
|
|
1349
|
+
passRateOverApplicable: roll.applicable > 0 ? Math.round((100 * roll.pass) / roll.applicable) : 100,
|
|
1350
|
+
...(naReason !== undefined ? { naReason } : {}),
|
|
1351
|
+
nextActions: Object.entries(actionAgg)
|
|
1352
|
+
.map(([action, v]) => ({ action: action as IndeterminateNextAction, count: v.count, reason: v.reason }))
|
|
1353
|
+
.sort((a, b) => b.count - a.count),
|
|
1354
|
+
};
|
|
1355
|
+
});
|
|
1356
|
+
|
|
1357
|
+
// ── top findings: every criterion with ≥1 applicable fail, worst-first ──
|
|
1358
|
+
const topFindings: TopFinding[] = params.criteria
|
|
1359
|
+
.map((c) => {
|
|
1360
|
+
const roll = rollOf(c.id);
|
|
1361
|
+
const prev = roll.applicable > 0 ? Math.round((100 * roll.fail) / roll.applicable) : 0;
|
|
1362
|
+
const folded = params.verdicts.find((v) => v.criterionId === c.id);
|
|
1363
|
+
return {
|
|
1364
|
+
criterion: c.id,
|
|
1365
|
+
severity: c.severity,
|
|
1366
|
+
applicable: roll.applicable,
|
|
1367
|
+
failCount: roll.fail,
|
|
1368
|
+
prevalencePctOverApplicable: prev,
|
|
1369
|
+
...(folded?.critique ? { root: folded.critique + ivNote(folded) } : {}),
|
|
1370
|
+
...(roll.example !== undefined ? { exampleTraceId: roll.example } : {}),
|
|
1371
|
+
};
|
|
1372
|
+
})
|
|
1373
|
+
.filter((f) => f.failCount > 0)
|
|
1374
|
+
.sort((a, b) => b.prevalencePctOverApplicable - a.prevalencePctOverApplicable || b.failCount - a.failCount);
|
|
1375
|
+
|
|
1376
|
+
// ── judge-health: grounded% (verdicts citing refs) · abstention% · symptom% ──
|
|
1377
|
+
const allV = files.flatMap((f) => f.verdicts);
|
|
1378
|
+
// UI-12-B — groundedPct is computed over DECIDED verdicts ONLY. An `uncertain`
|
|
1379
|
+
// abstain carries empty refs BY DESIGN (it's blockedBy / na for grounding, NOT
|
|
1380
|
+
// "ungrounded"), so counting it in the denominator dilutes the honest grounded
|
|
1381
|
+
// rate (a clean 100%-of-decided run reads e.g. ~92% just because some rows abstained).
|
|
1382
|
+
// Abstains are surfaced separately as the abstention rate. When there are NO decided
|
|
1383
|
+
// verdicts to ground (all abstained / none judged), groundedPct is undefined →
|
|
1384
|
+
// the cell renders "capture-unavailable" (never a fabricated 100%/0%).
|
|
1385
|
+
const decidedV = allV.filter((v) => v.result !== OutcomeVerdict.Uncertain);
|
|
1386
|
+
const grounded = decidedV.filter((v) => (v.refs?.length ?? 0) > 0).length;
|
|
1387
|
+
const abstained = allV.filter((v) => v.result === OutcomeVerdict.Uncertain).length;
|
|
1388
|
+
const symptom = ledger.filter((r) => r.health?.stoppedAtSymptom).length;
|
|
1389
|
+
const judgeHealth = {
|
|
1390
|
+
...(decidedV.length > 0 ? { groundedPct: Math.round((100 * grounded) / decidedV.length) } : {}),
|
|
1391
|
+
abstentionRatePct: allV.length > 0 ? Math.round((100 * abstained) / allV.length) : 0,
|
|
1392
|
+
stoppedAtSymptomPct: ledger.length > 0 ? Math.round((100 * symptom) / ledger.length) : 0,
|
|
1393
|
+
};
|
|
1394
|
+
|
|
1395
|
+
// WS-3 — derive the §5 Self-Eval AGGREGATE (behavior map + 9 components) from the
|
|
1396
|
+
// SAME parsed files + ledger. Deterministic; the emit-completeness meter (5.7) reads
|
|
1397
|
+
// the WS-1 gate. PURE — no extra dispatch (lean ~0 latency).
|
|
1398
|
+
const selfEval = deriveSelfEval({ ledger, criteria: params.criteria, emit: assessEmitCompleteness(files) });
|
|
1399
|
+
|
|
1400
|
+
return {
|
|
1401
|
+
subject: params.subject,
|
|
1402
|
+
scorecard: params.scorecard,
|
|
1403
|
+
verdicts: params.verdicts,
|
|
1404
|
+
criteria: params.criteria,
|
|
1405
|
+
ledger,
|
|
1406
|
+
selfEval,
|
|
1407
|
+
coverage,
|
|
1408
|
+
cohorts: { counts, matrix },
|
|
1409
|
+
gatingCriteria,
|
|
1410
|
+
criterionResolutions,
|
|
1411
|
+
topFindings,
|
|
1412
|
+
detectedFlags,
|
|
1413
|
+
judgeHealth,
|
|
1414
|
+
...(Object.keys(toolObservations).length > 0 ? { toolObservations } : {}),
|
|
1415
|
+
...(subjectProfile !== undefined ? { subjectProfile } : {}),
|
|
1416
|
+
...(params.sourceMap !== undefined ? { sourceMap: params.sourceMap } : {}),
|
|
1417
|
+
...(params.runConfig !== undefined ? { runConfig: params.runConfig } : {}),
|
|
1418
|
+
handover: params.handover ?? null,
|
|
1419
|
+
generatedAt: params.generatedAt,
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
|
|
1423
|
+
// ── WS-3 §5 Self-Eval — the AGGREGATE judge-calibration derivation ────────────
|
|
1424
|
+
|
|
1425
|
+
/** WS-3 — one behavior cluster in the §5.0 Judge Behavior Map. */
|
|
1426
|
+
export interface BehaviorCluster {
|
|
1427
|
+
label: string;
|
|
1428
|
+
count: number;
|
|
1429
|
+
/** good | warn | bad | neutral — drives the chip tint. */
|
|
1430
|
+
tone: string;
|
|
1431
|
+
note: string;
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
/** WS-3 — the §5 Self-Eval AGGREGATE: a deterministic judge-calibration roll-up over
|
|
1435
|
+
* the WHOLE population of verdicts (NOT per-trace). Each field maps to a wireframe
|
|
1436
|
+
* component; panels that need ground-truth the run does not have are MARKED proxy. */
|
|
1437
|
+
export interface SelfEvalAggregate {
|
|
1438
|
+
/** the headline trust band (0..100 + label + the basis sentence). */
|
|
1439
|
+
trust: { score: number; label: string; basis: string };
|
|
1440
|
+
/** 5.0 Judge Behavior Map — how the judge handled the population. */
|
|
1441
|
+
behaviorMap: {
|
|
1442
|
+
total: number;
|
|
1443
|
+
verdict: { pass: number; fail: number; indeterminate: number; incomplete: number };
|
|
1444
|
+
resolution: { walk: number; noWalk: number; truncated: number };
|
|
1445
|
+
clusters: BehaviorCluster[];
|
|
1446
|
+
/** where fails CONCENTRATE (top criteria by fail count). */
|
|
1447
|
+
concentration: { id: string; statement: string; fails: number; pct: number }[];
|
|
1448
|
+
};
|
|
1449
|
+
/** 5.1 Reference Integrity — grounded-ref presence over decided verdicts. */
|
|
1450
|
+
refIntegrity: { decided: number; grounded: number; ungrounded: number; groundedPct: number; note: string };
|
|
1451
|
+
/** 5.2 Confidence Calibration — verdicts bucketed by confidence band. */
|
|
1452
|
+
confidence: { band: string; count: number; pct: number }[];
|
|
1453
|
+
/** 5.3 Per-Criterion Reliability. */
|
|
1454
|
+
perCriterion: {
|
|
1455
|
+
id: string;
|
|
1456
|
+
statement: string;
|
|
1457
|
+
severity: string;
|
|
1458
|
+
pass: number;
|
|
1459
|
+
fail: number;
|
|
1460
|
+
unc: number;
|
|
1461
|
+
na: number;
|
|
1462
|
+
groundedPct: number;
|
|
1463
|
+
flag: string;
|
|
1464
|
+
}[];
|
|
1465
|
+
/** 5.4 Criterion Value / Applicability Audit. */
|
|
1466
|
+
criterionValue: { id: string; statement: string; severity: string; applicablePct: number; fails: number; verdict: string }[];
|
|
1467
|
+
/** 5.5 Diligence / Trace Coverage. */
|
|
1468
|
+
diligence: { examined: number; total: number; avgGroundingRefs: number; note: string };
|
|
1469
|
+
/** 5.6 Reasoning Transparency (blocked-aware). */
|
|
1470
|
+
transparency: { m2Pct: number; m3Pct: number; walkPct: number; blocked: boolean; note: string };
|
|
1471
|
+
/** 5.7 Emit-Completeness — the WS-1 self-honesty meter. */
|
|
1472
|
+
emit: EmitCompleteness;
|
|
1473
|
+
/** 5.8 Spot-Check / Disagreement queue — the brittle verdicts worth an operator look. */
|
|
1474
|
+
spotCheck: { trace: string; criterion: string; verdict: string; conf: string; reason: string }[];
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
/** WS-3 — derive the §5 Self-Eval aggregate. PURE + deterministic (sorted outputs). */
|
|
1478
|
+
export function deriveSelfEval(args: {
|
|
1479
|
+
ledger: LedgerRow[];
|
|
1480
|
+
criteria: ReportCriterion[];
|
|
1481
|
+
emit: EmitCompleteness;
|
|
1482
|
+
}): SelfEvalAggregate {
|
|
1483
|
+
const { ledger, criteria, emit } = args;
|
|
1484
|
+
const total = ledger.length;
|
|
1485
|
+
const critById = new Map(criteria.map((c) => [c.id, c]));
|
|
1486
|
+
|
|
1487
|
+
// ── verdict + resolution tallies ──
|
|
1488
|
+
const verdict = { pass: 0, fail: 0, indeterminate: 0, incomplete: 0 };
|
|
1489
|
+
const resolution = { walk: 0, noWalk: 0, truncated: 0 };
|
|
1490
|
+
for (const r of ledger) {
|
|
1491
|
+
const v = (r.verdict || "").toUpperCase();
|
|
1492
|
+
if (v === "PASS") verdict.pass++;
|
|
1493
|
+
else if (v === "FAIL") verdict.fail++;
|
|
1494
|
+
else if (v === "INCOMPLETE") verdict.incomplete++;
|
|
1495
|
+
else verdict.indeterminate++;
|
|
1496
|
+
if (r.resolution === "judge-walk") resolution.walk++;
|
|
1497
|
+
else if (r.resolution === "truncated") resolution.truncated++;
|
|
1498
|
+
else resolution.noWalk++;
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
// ── per-criterion roll-up (pass/fail/unc/na + grounding) ──
|
|
1502
|
+
type Agg = { pass: number; fail: number; unc: number; na: number; refd: number; decided: number };
|
|
1503
|
+
const agg = new Map<string, Agg>();
|
|
1504
|
+
for (const c of criteria) agg.set(c.id, { pass: 0, fail: 0, unc: 0, na: 0, refd: 0, decided: 0 });
|
|
1505
|
+
for (const r of ledger) {
|
|
1506
|
+
for (const [cid, res] of Object.entries(r.perCriterion)) {
|
|
1507
|
+
const a = agg.get(cid);
|
|
1508
|
+
if (!a) continue;
|
|
1509
|
+
if (res === "pass") a.pass++;
|
|
1510
|
+
else if (res === "fail") a.fail++;
|
|
1511
|
+
else if (res === "uncertain" || res === "indeterminate") a.unc++;
|
|
1512
|
+
else a.na++;
|
|
1513
|
+
}
|
|
1514
|
+
for (const cv of r.criterionVerdicts ?? []) {
|
|
1515
|
+
const a = agg.get(cv.criterionId);
|
|
1516
|
+
if (!a) continue;
|
|
1517
|
+
const decided = cv.result === "pass" || cv.result === "fail";
|
|
1518
|
+
if (decided) {
|
|
1519
|
+
a.decided++;
|
|
1520
|
+
if ((cv.refs?.length ?? 0) > 0) a.refd++;
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
const perCriterion = criteria.map((c) => {
|
|
1526
|
+
const a = agg.get(c.id)!;
|
|
1527
|
+
const groundedPct = a.decided > 0 ? Math.round((100 * a.refd) / a.decided) : 100;
|
|
1528
|
+
const flag = a.fail > 0 ? "fails-present" : a.pass + a.fail + a.unc === 0 ? "never-applies" : "solid";
|
|
1529
|
+
return { id: c.id, statement: c.statement, severity: c.severity, pass: a.pass, fail: a.fail, unc: a.unc, na: a.na, groundedPct, flag };
|
|
1530
|
+
});
|
|
1531
|
+
|
|
1532
|
+
// 5.4 criterion value: applicable% + ever-fails → keep/low-value verdict.
|
|
1533
|
+
const criterionValue = criteria.map((c) => {
|
|
1534
|
+
const a = agg.get(c.id)!;
|
|
1535
|
+
const applicable = a.pass + a.fail + a.unc;
|
|
1536
|
+
const denom = applicable + a.na;
|
|
1537
|
+
const applicablePct = denom > 0 ? Math.round((100 * applicable) / denom) : 0;
|
|
1538
|
+
const everFails = a.fail > 0;
|
|
1539
|
+
const verdictTag = applicable === 0 ? "never-applies → review" : everFails ? "keep — discriminates" : "keep — guards";
|
|
1540
|
+
return { id: c.id, statement: c.statement, severity: c.severity, applicablePct, fails: a.fail, verdict: verdictTag };
|
|
1541
|
+
});
|
|
1542
|
+
|
|
1543
|
+
// 5.0 concentration — where fails concentrate (top criteria by fail count).
|
|
1544
|
+
const concentration = perCriterion
|
|
1545
|
+
.filter((p) => p.fail > 0)
|
|
1546
|
+
.sort((a, b) => b.fail - a.fail)
|
|
1547
|
+
.slice(0, 6)
|
|
1548
|
+
.map((p) => ({ id: p.id, statement: p.statement, fails: p.fail, pct: total > 0 ? Math.round((100 * p.fail) / total) : 0 }));
|
|
1549
|
+
|
|
1550
|
+
// 5.1 ref integrity — grounded-ref presence over decided verdicts (population).
|
|
1551
|
+
let decided = 0;
|
|
1552
|
+
let groundedRefs = 0;
|
|
1553
|
+
let totalRefs = 0;
|
|
1554
|
+
for (const r of ledger) {
|
|
1555
|
+
for (const cv of r.criterionVerdicts ?? []) {
|
|
1556
|
+
if (cv.result === "pass" || cv.result === "fail") {
|
|
1557
|
+
decided++;
|
|
1558
|
+
const n = cv.refs?.length ?? 0;
|
|
1559
|
+
totalRefs += n;
|
|
1560
|
+
if (n > 0) groundedRefs++;
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
const refIntegrity = {
|
|
1565
|
+
decided,
|
|
1566
|
+
grounded: groundedRefs,
|
|
1567
|
+
ungrounded: decided - groundedRefs,
|
|
1568
|
+
groundedPct: decided > 0 ? Math.round((100 * groundedRefs) / decided) : 100,
|
|
1569
|
+
note:
|
|
1570
|
+
`${totalRefs} structured {obs,path,value} refs cited across ${decided} decided verdicts. ` +
|
|
1571
|
+
"Presence is checked here; full resolveRef-against-trace (GA-1) is the engine readiness gate.",
|
|
1572
|
+
};
|
|
1573
|
+
|
|
1574
|
+
// 5.2 confidence calibration — bucket decided verdicts by confidence band.
|
|
1575
|
+
const bands: { band: string; lo: number; hi: number }[] = [
|
|
1576
|
+
{ band: "0.9–1.0", lo: 0.9, hi: 1.01 },
|
|
1577
|
+
{ band: "0.7–0.9", lo: 0.7, hi: 0.9 },
|
|
1578
|
+
{ band: "0.5–0.7", lo: 0.5, hi: 0.7 },
|
|
1579
|
+
{ band: "<0.5", lo: -1, hi: 0.5 },
|
|
1580
|
+
];
|
|
1581
|
+
const bandCount = bands.map(() => 0);
|
|
1582
|
+
let confTotal = 0;
|
|
1583
|
+
for (const r of ledger) {
|
|
1584
|
+
for (const cv of r.criterionVerdicts ?? []) {
|
|
1585
|
+
const conf = typeof cv.confidence === "number" ? cv.confidence : undefined;
|
|
1586
|
+
if (conf === undefined) continue;
|
|
1587
|
+
confTotal++;
|
|
1588
|
+
const i = bands.findIndex((b) => conf >= b.lo && conf < b.hi);
|
|
1589
|
+
if (i >= 0) bandCount[i]++;
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
const confidence = bands.map((b, i) => ({
|
|
1593
|
+
band: b.band,
|
|
1594
|
+
count: bandCount[i],
|
|
1595
|
+
pct: confTotal > 0 ? Math.round((100 * bandCount[i]) / confTotal) : 0,
|
|
1596
|
+
}));
|
|
1597
|
+
|
|
1598
|
+
// 5.5 diligence — traces examined + avg grounding refs per trace.
|
|
1599
|
+
const examined = ledger.filter((r) => (r.criterionVerdicts?.length ?? 0) > 0).length;
|
|
1600
|
+
const avgGroundingRefs = examined > 0 ? Math.round((10 * totalRefs) / examined) / 10 : 0;
|
|
1601
|
+
const diligence = {
|
|
1602
|
+
examined,
|
|
1603
|
+
total,
|
|
1604
|
+
avgGroundingRefs,
|
|
1605
|
+
note: `${examined}/${total} trajectories carry ≥1 per-criterion judge verdict; avg ${avgGroundingRefs} grounding refs cited per examined trace.`,
|
|
1606
|
+
};
|
|
1607
|
+
|
|
1608
|
+
// 5.6 transparency — read off the emit-completeness (blocked when M2 is 0%).
|
|
1609
|
+
const fieldPct = (f: string): number => emit.fields.find((x) => x.field === f)?.pct ?? 0;
|
|
1610
|
+
const transparency = {
|
|
1611
|
+
m2Pct: fieldPct("understanding"),
|
|
1612
|
+
m3Pct: fieldPct("expectedTrajectory"),
|
|
1613
|
+
walkPct: fieldPct("judgeSteps"),
|
|
1614
|
+
blocked: fieldPct("understanding") === 0,
|
|
1615
|
+
note:
|
|
1616
|
+
fieldPct("understanding") === 0
|
|
1617
|
+
? "BLOCKED — the judge did not persist M2 understanding / M3 expected-trajectory (emit-contract gap, §5.7). A fresh run with the hardened contract captures them."
|
|
1618
|
+
: "the judge's train-of-thought (M2 understanding · M3 expected-trajectory) is captured and auditable.",
|
|
1619
|
+
};
|
|
1620
|
+
|
|
1621
|
+
// 5.8 spot-check queue — the brittle verdicts (fails + low-confidence) worth a look.
|
|
1622
|
+
const spotCheck: SelfEvalAggregate["spotCheck"] = [];
|
|
1623
|
+
for (const r of ledger) {
|
|
1624
|
+
for (const cv of r.criterionVerdicts ?? []) {
|
|
1625
|
+
const conf = typeof cv.confidence === "number" ? cv.confidence : undefined;
|
|
1626
|
+
const lowConf = conf !== undefined && conf < 0.6;
|
|
1627
|
+
const isFail = cv.result === "fail";
|
|
1628
|
+
if (!lowConf && !isFail) continue;
|
|
1629
|
+
spotCheck.push({
|
|
1630
|
+
trace: r.trajectoryId,
|
|
1631
|
+
criterion: critById.get(cv.criterionId)?.statement ?? cv.criterionId,
|
|
1632
|
+
verdict: String(cv.result || "").toUpperCase(),
|
|
1633
|
+
conf: conf !== undefined ? conf.toFixed(2) : "—",
|
|
1634
|
+
reason: isFail ? (lowConf ? "FAIL · low-confidence" : "FAIL") : "low-confidence call",
|
|
1635
|
+
});
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
spotCheck.sort((a, b) => (a.conf === "—" ? 1 : b.conf === "—" ? -1 : Number(a.conf) - Number(b.conf)));
|
|
1639
|
+
const spotCheckTop = spotCheck.slice(0, 20);
|
|
1640
|
+
|
|
1641
|
+
// headline trust band — a conservative composite (grounding · emit-completeness · fail-concentration).
|
|
1642
|
+
const emitPct = emit.completePct;
|
|
1643
|
+
const trustScore = Math.round(0.5 * refIntegrity.groundedPct + 0.5 * emitPct);
|
|
1644
|
+
const trustLabel =
|
|
1645
|
+
transparency.blocked || emitPct < 50
|
|
1646
|
+
? "PARTIAL — reasoning transparency blocked by the emit-contract gap"
|
|
1647
|
+
: trustScore >= 85
|
|
1648
|
+
? "STRONG — grounded + transparent"
|
|
1649
|
+
: "MODERATE — grounded, transparency improving";
|
|
1650
|
+
const trust = {
|
|
1651
|
+
score: trustScore,
|
|
1652
|
+
label: trustLabel,
|
|
1653
|
+
basis: `grounding ${refIntegrity.groundedPct}% of decided verdicts · emit-completeness ${emitPct}% · ${verdict.fail} failing trajectories across ${total}.`,
|
|
1654
|
+
};
|
|
1655
|
+
|
|
1656
|
+
return {
|
|
1657
|
+
trust,
|
|
1658
|
+
behaviorMap: { total, verdict, resolution, clusters: behaviorClusters(verdict, resolution, total), concentration },
|
|
1659
|
+
refIntegrity,
|
|
1660
|
+
confidence,
|
|
1661
|
+
perCriterion,
|
|
1662
|
+
criterionValue,
|
|
1663
|
+
diligence,
|
|
1664
|
+
transparency,
|
|
1665
|
+
emit,
|
|
1666
|
+
spotCheck: spotCheckTop,
|
|
1667
|
+
};
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
/** WS-3 — derive the §5.0 behavior clusters from the population tallies. PURE. */
|
|
1671
|
+
function behaviorClusters(
|
|
1672
|
+
verdict: { pass: number; fail: number; indeterminate: number; incomplete: number },
|
|
1673
|
+
resolution: { walk: number; noWalk: number; truncated: number },
|
|
1674
|
+
total: number,
|
|
1675
|
+
): BehaviorCluster[] {
|
|
1676
|
+
const pct = (n: number): number => (total > 0 ? Math.round((100 * n) / total) : 0);
|
|
1677
|
+
const out: BehaviorCluster[] = [
|
|
1678
|
+
{ label: "Passed (criteria satisfied)", count: verdict.pass, tone: "good", note: `${pct(verdict.pass)}% of the population judged PASS on all applicable criteria.` },
|
|
1679
|
+
{ label: "Failed (≥1 criterion)", count: verdict.fail, tone: "bad", note: `${pct(verdict.fail)}% carry ≥1 failing criterion — the discriminating cases.` },
|
|
1680
|
+
{ label: "Abstained (indeterminate)", count: verdict.indeterminate, tone: "warn", note: `${pct(verdict.indeterminate)}% abstained — inputs could not decide (blockedBy), routed to calibration not gate.` },
|
|
1681
|
+
{ label: "Truncated (INCOMPLETE)", count: verdict.incomplete, tone: "neutral", note: `${pct(verdict.incomplete)}% gated INCOMPLETE by the node-1 fidelity short-circuit (never walked).` },
|
|
1682
|
+
{ label: "Walk captured", count: resolution.walk, tone: "good", note: `${pct(resolution.walk)}% persisted the full §9.4 judge walk (target lane + anchored reasoning).` },
|
|
1683
|
+
{ label: "Walk dropped (judged)", count: resolution.noWalk, tone: "warn", note: `${pct(resolution.noWalk)}% were judged but did NOT persist the structured walk — the emit-contract gap (§5.7).` },
|
|
1684
|
+
];
|
|
1685
|
+
return out.filter((c) => c.count > 0).sort((a, b) => b.count - a.count);
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
// ── §1 Overview ──────────────────────────────────────────────────────────────
|
|
1689
|
+
|
|
1690
|
+
/**
|
|
1691
|
+
* Overview-redesign — the full-width ENTITY HERO card: rich subject identity from the
|
|
1692
|
+
* §9.4.4 M1 subject profile (name · entityType · purpose · tools · skill · scope ·
|
|
1693
|
+
* version · code-access · system-prompt). HONEST: a field the profile cannot establish
|
|
1694
|
+
* renders the `unknown` sentinel (never confabulated); an INFERRED (reconstructed)
|
|
1695
|
+
* field carries an `inferred` honesty chip; the system prompt is COLLAPSED. PURE.
|
|
1696
|
+
*/
|
|
1697
|
+
function entityHero(input: EvalReportInput): string {
|
|
1698
|
+
const p = input.subjectProfile;
|
|
1699
|
+
const inferred = new Set(p?.inferredFields ?? []);
|
|
1700
|
+
const name = p?.identity ?? input.subject.name;
|
|
1701
|
+
const codeAccess = p?.provenance ?? "reconstructed";
|
|
1702
|
+
// §9.4.5 E1 — TRACE-ONLY when there is no code access (reconstructed profile, or no
|
|
1703
|
+
// profile at all). Drives the honest "CODE-ACCESS: NONE · trace-only" header + the
|
|
1704
|
+
// UNAVAILABLE (never-confabulated) marking of harness + system-prompt.
|
|
1705
|
+
const isTraceOnly = codeAccess !== "given";
|
|
1706
|
+
const nTraces =
|
|
1707
|
+
input.sourceMap?.traceCount ?? input.coverage?.judged ?? input.ledger?.length ?? 0;
|
|
1708
|
+
|
|
1709
|
+
// a value cell: shows the value, or the `unknown` sentinel when absent; an `inferred`
|
|
1710
|
+
// chip flags a reconstructed (not-given) field so given-vs-inferred is honest.
|
|
1711
|
+
const cell = (value: string | undefined, key?: string): string => {
|
|
1712
|
+
const has = typeof value === "string" && value.trim().length > 0 && value !== PROFILE_UNKNOWN;
|
|
1713
|
+
const body = has ? esc(value as string) : `<span class="unk">${esc(PROFILE_UNKNOWN)}</span>`;
|
|
1714
|
+
const inf = key !== undefined && inferred.has(key) ? ` <span class="inf">inferred</span>` : "";
|
|
1715
|
+
return `${body}${inf}`;
|
|
1716
|
+
};
|
|
1717
|
+
|
|
1718
|
+
// §9.4.5 E2 — tools as diagnostics-style CHIPS with a per-tool OBSERVED call-count
|
|
1719
|
+
// ("obs N", from the trajectory agent-steps) instead of a flat comma list. In the
|
|
1720
|
+
// trace-only case this is the only honest tool evidence (what was actually called).
|
|
1721
|
+
const obs = input.toolObservations ?? {};
|
|
1722
|
+
const tools = p?.tools ?? [];
|
|
1723
|
+
const toolsInferred = inferred.has("tools");
|
|
1724
|
+
const toolChips =
|
|
1725
|
+
tools.length > 0
|
|
1726
|
+
? tools
|
|
1727
|
+
.map((t) => {
|
|
1728
|
+
const n = obs[t] ?? 0;
|
|
1729
|
+
const count =
|
|
1730
|
+
n > 0
|
|
1731
|
+
? ` <b class="obs" title="observed ${n} call(s) across the judged trajectories">obs ${n}</b>`
|
|
1732
|
+
: ` <b class="obs none" title="present in the inventory; 0 calls observed in the judged trajectories">obs 0</b>`;
|
|
1733
|
+
return `<span class="tchip">${esc(t)}${count}</span>`;
|
|
1734
|
+
})
|
|
1735
|
+
.join("")
|
|
1736
|
+
: `<span class="unk">${esc(PROFILE_UNKNOWN)}</span>`;
|
|
1737
|
+
const toolsCell =
|
|
1738
|
+
`<div class="tchips">${toolChips}${toolsInferred ? ` <span class="inf">inferred</span>` : ""}</div>`;
|
|
1739
|
+
|
|
1740
|
+
const targetModels = (input.subject.models ?? []).join(", ");
|
|
1741
|
+
|
|
1742
|
+
// §9.4.5 E1 — harness is UNAVAILABLE (never confabulated) in the trace-only case.
|
|
1743
|
+
const harnessHas = typeof p?.harness === "string" && p.harness.trim().length > 0 && p.harness !== PROFILE_UNKNOWN;
|
|
1744
|
+
const harnessCell = harnessHas
|
|
1745
|
+
? cell(p?.harness, "harness")
|
|
1746
|
+
: `<span class="unk">${esc(PROFILE_UNKNOWN)}</span> <span class="recon-note">UNAVAILABLE — ${isTraceOnly ? "reconstructed, never confabulated" : "not supplied"}</span>`;
|
|
1747
|
+
|
|
1748
|
+
const rows: [string, string][] = [
|
|
1749
|
+
["purpose", cell(p?.purpose, "purpose")],
|
|
1750
|
+
["tools", toolsCell],
|
|
1751
|
+
["skill", cell(p?.skill, "skill")],
|
|
1752
|
+
["scope", cell(p?.scope, "scope")],
|
|
1753
|
+
["harness", harnessCell],
|
|
1754
|
+
["source", cell(input.subject.source)],
|
|
1755
|
+
["org", cell(input.subject.org)],
|
|
1756
|
+
["target model(s)", cell(targetModels)],
|
|
1757
|
+
];
|
|
1758
|
+
const grid = rows.map(([k, v]) => `<div class="hk">${esc(k)}</div><div class="hv">${v}</div>`).join("");
|
|
1759
|
+
|
|
1760
|
+
// version + entityType + code-access provenance ride in the hero header.
|
|
1761
|
+
const verChip = p?.version ? `<span class="hchip">v ${esc(p.version)}</span>` : "";
|
|
1762
|
+
const typeChip = `<span class="hchip type">${cell(p?.entityType, "entityType")}</span>`;
|
|
1763
|
+
// §9.4.5 E1 — code-access chip: GIVEN reads green; trace-only reads the explicit
|
|
1764
|
+
// "NONE · trace-only" so a reader never mistakes a reconstructed profile for a known one.
|
|
1765
|
+
const accessChip = isTraceOnly
|
|
1766
|
+
? `<span class="hchip access recon">code-access: NONE · trace-only</span>`
|
|
1767
|
+
: `<span class="hchip access given">code-access: given</span>`;
|
|
1768
|
+
|
|
1769
|
+
// system prompt — present when GIVEN (code access) OR RECONSTRUCTED from the traces
|
|
1770
|
+
// (UI-14). `inferred.has("systemPrompt")` distinguishes the two for the provenance tag.
|
|
1771
|
+
const sysHas = typeof p?.systemPrompt === "string" && p.systemPrompt.trim().length > 0;
|
|
1772
|
+
const sysReconstructed = inferred.has("systemPrompt");
|
|
1773
|
+
|
|
1774
|
+
// §9.4.5 E1 — the trace-reconstructed banner: only when trace-only. States the N traces
|
|
1775
|
+
// the identity/tools were inferred from. The system prompt is now RECONSTRUCTED from the
|
|
1776
|
+
// batch when an LLM call carried it (UI-14); only harness stays UNAVAILABLE in that case.
|
|
1777
|
+
const reconLine = isTraceOnly
|
|
1778
|
+
? `<div class="hrecon">⟲ TRACE-RECONSTRUCTED · inferred from ${esc(nTraces)} trace(s) — identity + tools${sysHas ? " + system-prompt" : ""} observed from the batch; harness${sysHas ? "" : " + system-prompt"} UNAVAILABLE (never confabulated).</div>`
|
|
1779
|
+
: "";
|
|
1780
|
+
|
|
1781
|
+
// system prompt — COLLAPSED (it can be tens of KB); carries a provenance tag —
|
|
1782
|
+
// "RECONSTRUCTED · from traces" when derived from a GENERATION message list, or
|
|
1783
|
+
// "given" under code access. Marked UNAVAILABLE only when genuinely not found
|
|
1784
|
+
// (never confabulated).
|
|
1785
|
+
const sysProvTag = sysReconstructed
|
|
1786
|
+
? `<span class="hsys-prov recon">RECONSTRUCTED · from traces</span>`
|
|
1787
|
+
: `<span class="hsys-prov given">given · code access</span>`;
|
|
1788
|
+
const sysPrompt = sysHas
|
|
1789
|
+
? `<details class="hsys"><summary>system prompt ${sysProvTag}</summary><pre>${esc(p?.systemPrompt)}</pre></details>`
|
|
1790
|
+
: `<div class="hsys-na">system prompt — <span class="unk">${esc(PROFILE_UNKNOWN)}</span> · UNAVAILABLE (${isTraceOnly ? "reconstructed, no code access — never confabulated" : "not supplied"})</div>`;
|
|
1791
|
+
|
|
1792
|
+
// honesty footer: which fields are inferred (reconstructed), not given.
|
|
1793
|
+
const infList = [...inferred];
|
|
1794
|
+
const honesty =
|
|
1795
|
+
infList.length > 0
|
|
1796
|
+
? `<div class="hhonesty"><span class="inf">inferred</span> ${esc(infList.join(" · "))} — reconstructed from the trace batch, not given. The rest is GIVEN (code/metadata access).</div>`
|
|
1797
|
+
: `<div class="hhonesty"><span class="inf given">all given</span> every field supplied with code/metadata access — nothing reconstructed.</div>`;
|
|
1798
|
+
|
|
1799
|
+
return (
|
|
1800
|
+
`<div class="hero">` +
|
|
1801
|
+
`<div class="hero-top"><div class="hname">${esc(name)}</div>${typeChip}${verChip}${accessChip}</div>` +
|
|
1802
|
+
reconLine +
|
|
1803
|
+
`<div class="hero-grid">${grid}</div>` +
|
|
1804
|
+
sysPrompt +
|
|
1805
|
+
honesty +
|
|
1806
|
+
`</div>`
|
|
1807
|
+
);
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
/**
|
|
1811
|
+
* Overview-redesign — the thin PROVENANCE meta-strip under the header: the RUN config
|
|
1812
|
+
* (run-id · date · source · JUDGE substrate · pinned judge model · temp-0 · C-PIN).
|
|
1813
|
+
* The JUDGE substrate + model are DISTINCT from the TARGET model under eval. Defaults
|
|
1814
|
+
* that are ALWAYS true for this engine (agent-dispatch host runtime · temp 0 · C-PIN)
|
|
1815
|
+
* fill in when no run-config is threaded; a genuinely-unavailable field is MARKED. PURE.
|
|
1816
|
+
*/
|
|
1817
|
+
function provenanceStrip(input: EvalReportInput): string {
|
|
1818
|
+
const rc = input.runConfig ?? {};
|
|
1819
|
+
const substrate = rc.judgeSubstrate ?? "agent-dispatch";
|
|
1820
|
+
const temp = rc.temperature ?? 0;
|
|
1821
|
+
const cPin = rc.cPin !== false; // C-PIN is the engine default
|
|
1822
|
+
const date = rc.date ?? input.generatedAt;
|
|
1823
|
+
const source = rc.source ?? input.subject.source;
|
|
1824
|
+
const unk = `<span class="unk">${esc(PROFILE_UNKNOWN)}</span>`;
|
|
1825
|
+
const v = (val: string | undefined): string =>
|
|
1826
|
+
typeof val === "string" && val.trim().length > 0 ? esc(val) : unk;
|
|
1827
|
+
const item = (label: string, body: string): string =>
|
|
1828
|
+
`<span class="ps-item"><span class="ps-k">${esc(label)}</span><span class="ps-v">${body}</span></span>`;
|
|
1829
|
+
return (
|
|
1830
|
+
`<div class="provstrip">` +
|
|
1831
|
+
item("run-id", v(rc.runId)) +
|
|
1832
|
+
item("date", v(date)) +
|
|
1833
|
+
item("source", v(source)) +
|
|
1834
|
+
item("judge substrate", esc(substrate)) +
|
|
1835
|
+
item("judge model", v(rc.judgeModel)) +
|
|
1836
|
+
item("temp", esc(`temp ${temp}`)) +
|
|
1837
|
+
`<span class="ps-item"><span class="ps-badge ${cPin ? "on" : "off"}">${cPin ? "C-PIN" : "no C-PIN"}</span></span>` +
|
|
1838
|
+
`</div>`
|
|
1839
|
+
);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
/**
|
|
1843
|
+
* Overview-redesign — the segmented COVERAGE FUNNEL: ingested → triaged → judged →
|
|
1844
|
+
* outcomes. INCOMPLETE is a FIRST-CLASS outcome (a distinct segment from indeterminate:
|
|
1845
|
+
* one is "trace too truncated to judge", the other is "judged but not certain"). Counts
|
|
1846
|
+
* read from `coverage.{triaged,judged,byVerdict}` + the source-map trace count. PURE.
|
|
1847
|
+
*/
|
|
1848
|
+
function coverageFunnel(input: EvalReportInput): string {
|
|
1849
|
+
const cov = input.coverage ?? {};
|
|
1850
|
+
const bv = cov.byVerdict ?? {};
|
|
1851
|
+
const judged = cov.judged ?? input.ledger?.length ?? 0;
|
|
1852
|
+
const triaged = cov.triaged ?? judged;
|
|
1853
|
+
const ingested = input.sourceMap?.traceCount ?? triaged;
|
|
1854
|
+
const stage = (label: string, n: number, note: string): string =>
|
|
1855
|
+
`<div class="fstage"><div class="fn">${esc(n)}</div><div class="fl">${esc(label)}</div><div class="fnote">${esc(note)}</div></div>`;
|
|
1856
|
+
const arrow = `<div class="farrow">›</div>`;
|
|
1857
|
+
// the OUTCOME segment — four FIRST-CLASS buckets, INCOMPLETE separate from indeterminate.
|
|
1858
|
+
const outcomes: [string, number, string][] = [
|
|
1859
|
+
["pass", bv.PASS ?? 0, "pass"],
|
|
1860
|
+
["fail", bv.FAIL ?? 0, "fail"],
|
|
1861
|
+
["indeterminate", bv.INDETERMINATE ?? 0, "indet"],
|
|
1862
|
+
["incomplete", bv.INCOMPLETE ?? 0, "inc"],
|
|
1863
|
+
];
|
|
1864
|
+
const outcomePills = outcomes
|
|
1865
|
+
.map(([l, n, cls]) => `<div class="fpill ${cls}"><b>${esc(n)}</b> ${esc(l)}</div>`)
|
|
1866
|
+
.join("");
|
|
1867
|
+
return (
|
|
1868
|
+
`<div class="funnel">` +
|
|
1869
|
+
stage("Ingested", ingested, "traces pulled from source") +
|
|
1870
|
+
arrow +
|
|
1871
|
+
stage("Triaged", triaged, "tier-0 + fidelity gate") +
|
|
1872
|
+
arrow +
|
|
1873
|
+
stage("Judged", judged, "per-criterion verdicts") +
|
|
1874
|
+
arrow +
|
|
1875
|
+
`<div class="fstage outc"><div class="fl">Outcomes</div><div class="foutc">${outcomePills}</div></div>` +
|
|
1876
|
+
`</div>`
|
|
1877
|
+
);
|
|
1878
|
+
}
|
|
1879
|
+
|
|
1880
|
+
/**
|
|
1881
|
+
* §9.4.5 E3 — the eval-HEALTH temporal heatmap: CORRECTNESS over time (NOT latency —
|
|
1882
|
+
* latency belongs to diagnostics). Each bucket is an hour window; its COLOUR is the
|
|
1883
|
+
* pass-rate over the trajectories judged in that window (fail-clustering pops as a red
|
|
1884
|
+
* window → a deploy/incident regression signal), and its NUMBER is the count of
|
|
1885
|
+
* trajectories judged in the bucket. Computed from the ledger verdicts + the per-trace
|
|
1886
|
+
* timestamps (E3 data path). When NO trajectory carries a timestamp, the heatmap renders
|
|
1887
|
+
* its STRUCTURE with a single data-pending cell + an explicit note (never faked). PURE.
|
|
1888
|
+
*/
|
|
1889
|
+
function evalHealthHeatmap(input: EvalReportInput): string {
|
|
1890
|
+
const ledger = input.ledger ?? [];
|
|
1891
|
+
const timed = ledger.filter((r) => typeof r.timestamp === "string" && (r.timestamp as string).length >= 13);
|
|
1892
|
+
const head = `<h3>eval-health over time — correctness, not latency</h3>`;
|
|
1893
|
+
|
|
1894
|
+
if (timed.length === 0) {
|
|
1895
|
+
// structure + data-pending: the wiring is live, the batch just lacks timestamps.
|
|
1896
|
+
return (
|
|
1897
|
+
head +
|
|
1898
|
+
`<div class="ehm" data-pending="1">` +
|
|
1899
|
+
`<div class="ehm-cell skip" title="awaiting timestamped traces">·</div>` +
|
|
1900
|
+
`</div>` +
|
|
1901
|
+
`<div class="ehm-pending">⏲ data-pending — no per-trace timestamps in this batch. Correctness-over-time wiring is active; it renders real pass-rate windows as soon as the source carries trace timestamps.</div>`
|
|
1902
|
+
);
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
// bucket by hour window (ISO "YYYY-MM-DDTHH"), deterministic chronological order.
|
|
1906
|
+
const buckets = new Map<string, { judged: number; pass: number; fail: number }>();
|
|
1907
|
+
for (const r of timed) {
|
|
1908
|
+
const key = (r.timestamp as string).slice(0, 13); // YYYY-MM-DDTHH
|
|
1909
|
+
const b = buckets.get(key) ?? { judged: 0, pass: 0, fail: 0 };
|
|
1910
|
+
b.judged += 1;
|
|
1911
|
+
const v = r.verdict.toUpperCase();
|
|
1912
|
+
if (v === "PASS") b.pass += 1;
|
|
1913
|
+
else if (v === "FAIL") b.fail += 1; // INCOMPLETE/INDETERMINATE count as judged, not in the rate
|
|
1914
|
+
buckets.set(key, b);
|
|
1915
|
+
}
|
|
1916
|
+
const keys = [...buckets.keys()].sort();
|
|
1917
|
+
const cells = keys
|
|
1918
|
+
.map((k) => {
|
|
1919
|
+
const b = buckets.get(k)!;
|
|
1920
|
+
const denom = b.pass + b.fail;
|
|
1921
|
+
const rate = denom > 0 ? Math.round((100 * b.pass) / denom) : -1;
|
|
1922
|
+
const cls = rate < 0 ? "skip" : rate < 80 ? "fail" : rate < 95 ? "indet" : "pass";
|
|
1923
|
+
const hour = k.slice(11, 13);
|
|
1924
|
+
const day = k.slice(5, 10);
|
|
1925
|
+
const title = `${esc(k)}:00 — ${b.judged} judged · ${b.pass} pass / ${b.fail} fail${rate >= 0 ? ` · ${rate}% pass-rate` : " · rate n/a"}`;
|
|
1926
|
+
return (
|
|
1927
|
+
`<div class="ehm-col">` +
|
|
1928
|
+
`<div class="ehm-cell ${cls}" title="${title}">${esc(b.judged)}</div>` +
|
|
1929
|
+
`<div class="ehm-lab">${esc(day)}<br>${esc(hour)}h</div>` +
|
|
1930
|
+
`</div>`
|
|
1931
|
+
);
|
|
1932
|
+
})
|
|
1933
|
+
.join("");
|
|
1934
|
+
return (
|
|
1935
|
+
head +
|
|
1936
|
+
`<div class="ehm-legend">colour = pass-rate per window (<span class="sw fail"></span><80% <span class="sw indet"></span><95% <span class="sw pass"></span>≥95%) · number = trajectories judged</div>` +
|
|
1937
|
+
`<div class="ehm">${cells}</div>`
|
|
1938
|
+
);
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
function overviewTab(input: EvalReportInput): string {
|
|
1942
|
+
const g = input.scorecard.gate;
|
|
1943
|
+
const sm = input.sourceMap;
|
|
1944
|
+
const cov = input.coverage ?? {};
|
|
1945
|
+
const runVerdict = g.runVerdict ?? (g.passed ? "pass" : "fail");
|
|
1946
|
+
const indetCount = g.indeterminateBy?.length ?? 0;
|
|
1947
|
+
const verdictClass = runVerdict === "pass" ? "pass" : runVerdict === "incomplete" ? "skip" : "fail";
|
|
1948
|
+
const verdictText =
|
|
1949
|
+
runVerdict === "pass"
|
|
1950
|
+
? "GATE PASS — no CRIT/HIGH criterion failed or was indeterminate."
|
|
1951
|
+
: runVerdict === "incomplete"
|
|
1952
|
+
? `GATE INCOMPLETE — ${indetCount} CRIT/HIGH criterion/criteria indeterminate (${(g.indeterminateBy ?? []).map((x) => esc(x.criterionId)).join(", ")}). No CRIT/HIGH fail, but the run cannot be certified — NOT a pass.`
|
|
1953
|
+
: `GATE FAIL — ${g.gatedBy.length} gating criterion/criteria failed (${g.gatedBy.map((x) => esc(x.criterionId)).join(", ")}).`;
|
|
1954
|
+
|
|
1955
|
+
// 6-tile big-stat row (theme.css .big-stat .s/.v/.l) — status-acuity: the OUTCOME
|
|
1956
|
+
// tiles (pass/fail/indeterminate) carry a status tint + left-accent so the eye
|
|
1957
|
+
// separates outcomes from the neutral COUNT tiles (criteria/trajectories/traces).
|
|
1958
|
+
const tileRows: [string, string, string][] = [
|
|
1959
|
+
["Criteria", String(g.total), "count"],
|
|
1960
|
+
["Pass", String(g.passCount), "pass"],
|
|
1961
|
+
["Fail", String(g.gatedBy.length), "fail"],
|
|
1962
|
+
["Indeterminate", String(indetCount), "indet"],
|
|
1963
|
+
["Trajectories", String(input.ledger?.length ?? 0), "count"],
|
|
1964
|
+
["Traces (coverage)", String(sm?.traceCount ?? cov.judged ?? 0), "count"],
|
|
1965
|
+
];
|
|
1966
|
+
const tiles = tileRows
|
|
1967
|
+
.map(([l, v, kind]) => `<div class="s ${kind}"><div class="v">${esc(v)}</div><div class="l">${esc(l)}</div></div>`)
|
|
1968
|
+
.join("");
|
|
1969
|
+
|
|
1970
|
+
const coverageNote = cov.gapNote
|
|
1971
|
+
? `<div class="note"><span class="tag">★ COVERAGE CONTRACT</span> ${esc(cov.gapNote)}</div>`
|
|
1972
|
+
: sm
|
|
1973
|
+
? `<div class="note"><span class="tag">★ COVERAGE CONTRACT</span> ${esc(sm.traceCount)} trace(s) · ${esc(sm.nullOutputTraces)} with null trace-output (read from GENERATION, SV-1) · platform <code>${esc(sm.platform)}</code>.</div>`
|
|
1974
|
+
: "";
|
|
1975
|
+
|
|
1976
|
+
// gating-criteria table — pass-rate over applicable (R4: statement + provenance + hover)
|
|
1977
|
+
const gating = input.gatingCriteria ?? [];
|
|
1978
|
+
const gatingRows =
|
|
1979
|
+
gating.length > 0
|
|
1980
|
+
? gating
|
|
1981
|
+
.map((d) => {
|
|
1982
|
+
const c = criterionById(input, d.criterion);
|
|
1983
|
+
return (
|
|
1984
|
+
`<tr><td class="cn"><b class="mono">${esc(d.criterion)}</b> ${critHover(c)} ${provChip(c)} ${methodChip(c)}` +
|
|
1985
|
+
`<div class="cstmt">${esc(critStatement(c))}</div></td>` +
|
|
1986
|
+
`<td>${esc(d.applicable)}</td><td>${esc(d.fail)}</td>` +
|
|
1987
|
+
`<td style="color:${rateCol(d.passRateOverApplicable)};font-weight:600">${esc(d.passRateOverApplicable)}%</td></tr>`
|
|
1988
|
+
);
|
|
1989
|
+
})
|
|
1990
|
+
.join("")
|
|
1991
|
+
: `<tr><td colspan="4" style="color:var(--dim)">no gating criteria</td></tr>`;
|
|
1992
|
+
const gatingTable =
|
|
1993
|
+
`<h3>gating criteria — pass-rate over applicable</h3>` +
|
|
1994
|
+
`<table><thead><tr><th>criterion (statement · provenance)</th><th>applicable</th><th>fail</th><th>pass-rate</th></tr></thead><tbody>${gatingRows}</tbody></table>`;
|
|
1995
|
+
|
|
1996
|
+
// top-findings teaser
|
|
1997
|
+
const teaser = (input.topFindings ?? [])
|
|
1998
|
+
.slice(0, 5)
|
|
1999
|
+
.map(
|
|
2000
|
+
(fd) =>
|
|
2001
|
+
`<div class="teaser-row"><span class="sev ${sevCls(fd.severity)}">${esc((fd.severity ?? "med").toUpperCase())}</span>` +
|
|
2002
|
+
`<b class="mono">${esc(fd.criterion)}</b>` +
|
|
2003
|
+
`<span class="teaser-num">${esc(fd.failCount)}/${esc(fd.applicable)} fail · ${esc(fd.prevalencePctOverApplicable)}%</span></div>`,
|
|
2004
|
+
)
|
|
2005
|
+
.join("");
|
|
2006
|
+
const teaserBlock = (input.topFindings ?? []).length > 0 ? `<h3>top findings</h3>${teaser}` : "";
|
|
2007
|
+
|
|
2008
|
+
return (
|
|
2009
|
+
`<h2>① Overview</h2><div class="sub">subject identity · provenance · coverage · gate · top findings</div>` +
|
|
2010
|
+
tabDesc("What this tab shows: WHO the subject is (entity hero), HOW the run was judged (provenance strip — judge substrate + pinned model, distinct from the target), the coverage funnel (ingested → triaged → judged → outcomes, INCOMPLETE first-class), the GATE verdict, the gating criteria with their pass-rate over the applicable denominator, and a teaser of the top findings.") +
|
|
2011
|
+
entityHero(input) +
|
|
2012
|
+
provenanceStrip(input) +
|
|
2013
|
+
coverageFunnel(input) +
|
|
2014
|
+
`<div class="verdict ${verdictClass}"><strong>${esc(verdictText)}</strong></div>` +
|
|
2015
|
+
verdictLegend() +
|
|
2016
|
+
`<div class="big-stat">${tiles}</div>` +
|
|
2017
|
+
coverageNote +
|
|
2018
|
+
evalHealthHeatmap(input) +
|
|
2019
|
+
gatingTable +
|
|
2020
|
+
teaserBlock
|
|
2021
|
+
);
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
// ── §2 Trajectory · Judge Behaviour (ledger + drill) ─────────────────────────
|
|
2025
|
+
|
|
2026
|
+
function trajectoryTab(input: EvalReportInput): string {
|
|
2027
|
+
const ledger = input.ledger ?? [];
|
|
2028
|
+
if (ledger.length === 0) {
|
|
2029
|
+
return (
|
|
2030
|
+
`<h2>② Trajectory · Judge Behaviour</h2>` +
|
|
2031
|
+
`<div class="sub">per-trace judge ledger — no per-trajectory verdict files supplied for this render.</div>`
|
|
2032
|
+
);
|
|
2033
|
+
}
|
|
2034
|
+
const walks = ledger.filter((r) => r.judgeSteps && r.judgeSteps.length > 0).length;
|
|
2035
|
+
return (
|
|
2036
|
+
`<h2>② Trajectory · Judge Behaviour</h2>` +
|
|
2037
|
+
tabDesc("What this tab shows: the Target-Agent trajectory ‖ Judge trajectory SIDE-BY-SIDE. Click a row to drill in — the agent's steps render on the LEFT lane, the judge's reasoning walk (anchored to each step) on the RIGHT, with the judge's gather-context, expected-trajectory and root localization.") +
|
|
2038
|
+
`<div class="sub">all ${esc(ledger.length)} — click a row; ✦ = full agent‖judge walk (${esc(walks)} with judge_steps)</div>` +
|
|
2039
|
+
`<div class="lfilter">` +
|
|
2040
|
+
`<b class="on" data-flt="all">all</b><b data-flt="FAIL">fail</b><b data-flt="INDETERMINATE">indeterminate</b><b data-flt="PASS">pass</b>` +
|
|
2041
|
+
`<input id="q" placeholder="search route / trace id…"><span class="sp" id="cnt"></span></div>` +
|
|
2042
|
+
`<table class="ledger"><thead><tr><th>trace</th><th>route</th><th>verdict</th><th>resolution</th><th>pass</th><th>fail</th><th>indet</th><th>failing criteria</th></tr></thead><tbody id="lrows"></tbody></table>` +
|
|
2043
|
+
`<div id="virt" class="mono virt"></div>` +
|
|
2044
|
+
`<div id="drill"></div>`
|
|
2045
|
+
);
|
|
2046
|
+
}
|
|
2047
|
+
|
|
2048
|
+
// ── §3 Eval Scorecard (heatmap + subcards + calibration) ─────────────────────
|
|
2049
|
+
|
|
2050
|
+
function heatmapHtml(input: EvalReportInput): string {
|
|
2051
|
+
const cohorts = input.cohorts;
|
|
2052
|
+
if (!cohorts || Object.keys(cohorts.counts).length === 0) {
|
|
2053
|
+
return `<div class="sub">criteria × route cohort — no per-trajectory cohort data for this render.</div>`;
|
|
2054
|
+
}
|
|
2055
|
+
const cohortKeys = Object.entries(cohorts.counts)
|
|
2056
|
+
.sort((a, b) => b[1] - a[1])
|
|
2057
|
+
.slice(0, 7)
|
|
2058
|
+
.map(([c]) => c);
|
|
2059
|
+
const head =
|
|
2060
|
+
`<span class="lab"></span>` +
|
|
2061
|
+
cohortKeys.map((c) => `<span class="colh">${esc(c)}<br>${esc(cohorts.counts[c] ?? 0)}</span>`).join("");
|
|
2062
|
+
const cids = Object.keys(cohorts.matrix).slice(0, 14);
|
|
2063
|
+
const rows = cids.map((cid) => {
|
|
2064
|
+
let cells = `<span class="lab" title="${esc(cid)}">${esc(cid.slice(0, 26))}</span>`;
|
|
2065
|
+
for (const co of cohortKeys) {
|
|
2066
|
+
const cell = cohorts.matrix[cid]?.[co];
|
|
2067
|
+
if (!cell) {
|
|
2068
|
+
cells += `<span class="cell skip"></span>`;
|
|
2069
|
+
continue;
|
|
2070
|
+
}
|
|
2071
|
+
const tot = cell.pass + cell.fail + cell.indeterminate;
|
|
2072
|
+
if (tot === 0) {
|
|
2073
|
+
cells += `<span class="cell skip"></span>`;
|
|
2074
|
+
continue;
|
|
2075
|
+
}
|
|
2076
|
+
const r = Math.round((100 * cell.pass) / tot);
|
|
2077
|
+
const cls = r >= 95 ? "pass" : r >= 80 ? "indet" : "fail";
|
|
2078
|
+
cells += `<span class="cell ${cls}" title="${esc(cid)} × ${esc(co)}: ${cell.pass}/${tot} pass">${r}</span>`;
|
|
2079
|
+
}
|
|
2080
|
+
return cells;
|
|
2081
|
+
});
|
|
2082
|
+
// UI-3 — wrap in a horizontal-scroll container so long cohort names / many cohorts
|
|
2083
|
+
// never overflow the page; the grid columns use minmax(0,…) so a long unbreakable
|
|
2084
|
+
// header word can't force the track wider than its share (it ellipsis-clips instead).
|
|
2085
|
+
return (
|
|
2086
|
+
`<div class="hm-scroll">` +
|
|
2087
|
+
`<div class="hm" style="grid-template-columns:150px repeat(${cohortKeys.length},minmax(58px,1fr))">` +
|
|
2088
|
+
head +
|
|
2089
|
+
rows.join("") +
|
|
2090
|
+
`</div>` +
|
|
2091
|
+
`</div>`
|
|
2092
|
+
);
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
/**
|
|
2096
|
+
* UI-5 — the success-rate + key-metrics SUMMARY row under the §3 heatmap. The heatmap
|
|
2097
|
+
* alone "dangled" into the subcards with no headline numbers; this anchors it with a
|
|
2098
|
+
* compact metrics strip (criteria pass-rate · trajectory pass-rate · indeterminate ·
|
|
2099
|
+
* incomplete · grounded%). All numbers derive from the scorecard gate + the folded
|
|
2100
|
+
* coverage + judge-health — nothing fabricated; an empty source renders 0/—. PURE.
|
|
2101
|
+
*/
|
|
2102
|
+
function scorecardMetrics(input: EvalReportInput): string {
|
|
2103
|
+
const g = input.scorecard.gate;
|
|
2104
|
+
const bv = input.coverage?.byVerdict ?? {};
|
|
2105
|
+
const judged = input.coverage?.judged ?? input.ledger?.length ?? 0;
|
|
2106
|
+
const critPassRate = g.total > 0 ? Math.round((100 * g.passCount) / g.total) : 100;
|
|
2107
|
+
const trajPass = bv.PASS ?? 0;
|
|
2108
|
+
const trajRate = judged > 0 ? Math.round((100 * trajPass) / judged) : 0;
|
|
2109
|
+
const grounded = input.judgeHealth?.groundedPct;
|
|
2110
|
+
const tile = (label: string, value: string, cls: string, note: string): string =>
|
|
2111
|
+
`<div class="scm ${cls}"><div class="scm-v">${esc(value)}</div><div class="scm-l">${esc(label)}</div><div class="scm-n">${esc(note)}</div></div>`;
|
|
2112
|
+
return (
|
|
2113
|
+
`<div class="sc-metrics">` +
|
|
2114
|
+
tile("criteria pass-rate", `${critPassRate}%`, rateClass(critPassRate), `${g.passCount}/${g.total} criteria pass`) +
|
|
2115
|
+
tile("trajectory pass-rate", `${trajRate}%`, rateClass(trajRate), `${trajPass}/${judged} trajectories pass`) +
|
|
2116
|
+
tile("needs-evidence", String(bv.INDETERMINATE ?? 0), (bv.INDETERMINATE ?? 0) > 0 ? "rate-mid" : "rate-ok", "precondition present, undecided (N/A excluded)") +
|
|
2117
|
+
tile("incomplete", String(bv.INCOMPLETE ?? 0), (bv.INCOMPLETE ?? 0) > 0 ? "rate-mid" : "rate-ok", "too truncated to judge") +
|
|
2118
|
+
// UI-12-B — grounded is HONEST about capture: 0 / undefined (judge emitted 0
|
|
2119
|
+
// structured refs) renders "capture-unavailable" (data-pending), NEVER a silent
|
|
2120
|
+
// green 0%; a real >0 value is colour-coded by rate (low = critical).
|
|
2121
|
+
(grounded === undefined || grounded === 0
|
|
2122
|
+
? tile("grounded", "capture-unavailable", "rate-na pending", "judge emitted 0 structured refs — capture-unavailable")
|
|
2123
|
+
: tile("grounded", `${grounded}%`, rateClass(grounded), "verdicts citing evidence")) +
|
|
2124
|
+
`</div>`
|
|
2125
|
+
);
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
/**
|
|
2129
|
+
* WS-3 follow-up — a compact per-criterion KEY rendered DIRECTLY UNDER the §3
|
|
2130
|
+
* heatmap. The heatmap rows are raw technical criterion ids (e.g.
|
|
2131
|
+
* `send-delivery-success`); this legend maps EVERY criterion on the scorecard axis —
|
|
2132
|
+
* gating AND non-gating — to its one-line plain-language gloss (reusing the same
|
|
2133
|
+
* `plainExplainer` the Findings cards use), so no scorecard criterion is left without
|
|
2134
|
+
* "what it measures · why it matters". Covers ALL `input.criteria` (not the gating
|
|
2135
|
+
* subset). PURE — empty criteria ⇒ empty string.
|
|
2136
|
+
*/
|
|
2137
|
+
function criterionLegend(input: EvalReportInput): string {
|
|
2138
|
+
const crits = input.criteria ?? [];
|
|
2139
|
+
if (crits.length === 0) return "";
|
|
2140
|
+
const rows = crits
|
|
2141
|
+
.map((c) => {
|
|
2142
|
+
const sev = sevCls(c.severity);
|
|
2143
|
+
return (
|
|
2144
|
+
`<div class="plain plain-leg">` +
|
|
2145
|
+
`<span class="pl-id mono">${esc(c.id)}</span>` +
|
|
2146
|
+
`<span class="sev ${sev}">${esc(String(c.severity ?? "med").toUpperCase())}</span>` +
|
|
2147
|
+
`<span class="pl-tx">${esc(plainExplainer(c))}</span>` +
|
|
2148
|
+
`</div>`
|
|
2149
|
+
);
|
|
2150
|
+
})
|
|
2151
|
+
.join("");
|
|
2152
|
+
return (
|
|
2153
|
+
`<div class="plain-legend">` +
|
|
2154
|
+
`<div class="pl-h">what each criterion measures — plain-language key (all ${crits.length})</div>` +
|
|
2155
|
+
rows +
|
|
2156
|
+
`</div>`
|
|
2157
|
+
);
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
function subcardsHtml(input: EvalReportInput): string {
|
|
2161
|
+
const gating = input.gatingCriteria ?? [];
|
|
2162
|
+
if (gating.length === 0) return `<div class="sub">no gating criteria to calibrate.</div>`;
|
|
2163
|
+
return gating
|
|
2164
|
+
.map((d) => {
|
|
2165
|
+
const sev = sevCls(d.severity);
|
|
2166
|
+
const c = criterionById(input, d.criterion);
|
|
2167
|
+
// UI-6 — the card colour is driven by SUCCESS RATE (green-ish high · amber mid ·
|
|
2168
|
+
// red-ish low), on a purplish base — NOT a uniform severity red. Severity stays
|
|
2169
|
+
// legible as its own pill in the header.
|
|
2170
|
+
const rate = rateClass(d.passRateOverApplicable);
|
|
2171
|
+
// WS-5 — the indeterminate RESOLUTION split: N/A (precondition absent, excluded
|
|
2172
|
+
// from the denominator) vs needs-evidence (precondition present, undecided). No
|
|
2173
|
+
// bare "indeterminate".
|
|
2174
|
+
const whyAb =
|
|
2175
|
+
(d.indeterminate ?? 0) > 0 || (d.na ?? 0) > 0
|
|
2176
|
+
? `<div class="nest"><div class="nest-h">▾ indeterminate resolution</div><div class="row">` +
|
|
2177
|
+
`<span class="rz na">N/A ${esc(d.na ?? 0)}</span> precondition absent — excluded from denominator · ` +
|
|
2178
|
+
`<span class="rz ne">needs-evidence ${esc(d.indeterminate ?? 0)}</span> precondition present, undecided</div></div>`
|
|
2179
|
+
: "";
|
|
2180
|
+
return (
|
|
2181
|
+
`<div class="subc ${rate}"><div class="subc-h"><span class="sev ${sev}">${esc((d.severity ?? "med").toUpperCase())}</span>` +
|
|
2182
|
+
`<b>${esc(d.criterion)}</b> ${critHover(c)} ${provChip(c)} ${methodChip(c)}<span class="chip">gate</span>` +
|
|
2183
|
+
`<span class="vp" style="margin-left:auto;color:${rateCol(d.passRateOverApplicable)};border-color:${rateCol(d.passRateOverApplicable)}">${esc(d.passRateOverApplicable)}%</span></div>` +
|
|
2184
|
+
// OPERATOR-REQUESTED: the one-line plain-language "Measures / Why it matters" gloss
|
|
2185
|
+
// INLINE in every gating subcard (not only in the §3 legend) — so reading a gating
|
|
2186
|
+
// criterion's detail explains, in plain language, what it measures right there.
|
|
2187
|
+
plainBanner(c) +
|
|
2188
|
+
// UI-7 — the full criterion DEFINITION in-card (statement · pass condition ·
|
|
2189
|
+
// dimension/severity/level · judged-from inputs · provenance · mined details).
|
|
2190
|
+
`<div class="nest"><div class="nest-h">▾ criterion definition — what it is & how it's judged</div>` +
|
|
2191
|
+
criterionDefn(c, d.criterion) +
|
|
2192
|
+
`</div>` +
|
|
2193
|
+
`<div class="nest"><div class="nest-h">▾ grounding</div><div class="row"><span class="ref">obs:</span> ${esc(d.fail)}/${esc(d.applicable)} applicable fail · ${esc((d.denominatorNote ?? "").slice(0, 90))}</div></div>` +
|
|
2194
|
+
`<div class="nest"><div class="nest-h">▾ verdict reasoning</div><div class="row">${esc((d.root ?? "all applicable pass").slice(0, 160))}</div></div>` +
|
|
2195
|
+
whyAb +
|
|
2196
|
+
// UI-8 — calibrate tags grouped for MUTUAL EXCLUSION (radio semantics): the
|
|
2197
|
+
// keep/revise/retire group and the verify/eliminate group each allow ONE active
|
|
2198
|
+
// selection (the client JS clears siblings within a `data-calgroup`).
|
|
2199
|
+
`<div class="cal"><span class="l">calibrate</span>` +
|
|
2200
|
+
`<span class="calgroup" data-calgroup="keep-revise-retire"><b>keep</b><b>revise</b><b>retire</b></span>` +
|
|
2201
|
+
`<span style="color:var(--dim)">·</span>` +
|
|
2202
|
+
`<span class="calgroup" data-calgroup="verify-eliminate"><b>verify</b><b>eliminate</b></span>` +
|
|
2203
|
+
`</div></div>`
|
|
2204
|
+
);
|
|
2205
|
+
})
|
|
2206
|
+
.join("");
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
/** WS-5 — a readable label for a chain next-action. */
|
|
2210
|
+
function nextActionLabel(a: string): string {
|
|
2211
|
+
return a === "code-recheck"
|
|
2212
|
+
? "code-recheck"
|
|
2213
|
+
: a === "2nd-judge"
|
|
2214
|
+
? "2nd-judge"
|
|
2215
|
+
: a === "revise-criterion"
|
|
2216
|
+
? "revise-criterion"
|
|
2217
|
+
: "HITL-spot-check";
|
|
2218
|
+
}
|
|
2219
|
+
|
|
2220
|
+
/**
|
|
2221
|
+
* WS-5 — the per-criterion INDETERMINATE resolution chain (§3). For EVERY criterion
|
|
2222
|
+
* (gating + non-gating) it shows the denominator-honest split:
|
|
2223
|
+
* applicable · pass · fail · N/A (precondition absent) · needs-evidence (+ next-action)
|
|
2224
|
+
* N/A is shown SEPARATELY and is NEVER in the applicable/fail denominator. A
|
|
2225
|
+
* needs-evidence count carries its aggregated chain next-actions + the missing-signal
|
|
2226
|
+
* reason — so no criterion ever shows a bare, actionless "indeterminate". PURE.
|
|
2227
|
+
*/
|
|
2228
|
+
function criterionResolutionTable(input: EvalReportInput): string {
|
|
2229
|
+
const rows = input.criterionResolutions ?? [];
|
|
2230
|
+
if (rows.length === 0) return "";
|
|
2231
|
+
const body = rows
|
|
2232
|
+
.map((r) => {
|
|
2233
|
+
const sev = sevCls(r.severity);
|
|
2234
|
+
const naCell =
|
|
2235
|
+
r.na > 0
|
|
2236
|
+
? `<span class="rz na">N/A ${esc(r.na)}</span>` +
|
|
2237
|
+
(r.naReason ? `<span class="rz-reason">precondition absent — ${esc(r.naReason.slice(0, 120))}</span>` : "")
|
|
2238
|
+
: `<span class="rz na zero">N/A 0</span>`;
|
|
2239
|
+
const neCell =
|
|
2240
|
+
r.needsEvidence > 0
|
|
2241
|
+
? `<span class="rz ne">needs-evidence ${esc(r.needsEvidence)}</span>` +
|
|
2242
|
+
r.nextActions
|
|
2243
|
+
.map(
|
|
2244
|
+
(a) =>
|
|
2245
|
+
`<span class="rz-act">→ ${esc(nextActionLabel(a.action))} ×${esc(a.count)}` +
|
|
2246
|
+
`<span class="rz-reason">${esc(a.reason.slice(0, 120))}</span></span>`,
|
|
2247
|
+
)
|
|
2248
|
+
.join("")
|
|
2249
|
+
: `<span class="rz ne zero">needs-evidence 0</span>`;
|
|
2250
|
+
return (
|
|
2251
|
+
`<div class="rzrow ${sev}">` +
|
|
2252
|
+
`<div class="rz-h"><span class="sev ${sev}">${esc(String(r.severity ?? "med").toUpperCase())}</span>` +
|
|
2253
|
+
`<b class="mono">${esc(r.criterion)}</b>` +
|
|
2254
|
+
`<span class="rz-rate" style="color:${rateCol(r.passRateOverApplicable)}">${esc(r.passRateOverApplicable)}% pass / applicable</span></div>` +
|
|
2255
|
+
`<div class="rz-stats">` +
|
|
2256
|
+
`<span class="rz applicable">applicable ${esc(r.applicable)}</span>` +
|
|
2257
|
+
`<span class="rz pass">pass ${esc(r.pass)}</span>` +
|
|
2258
|
+
`<span class="rz fail">fail ${esc(r.fail)}</span>` +
|
|
2259
|
+
naCell +
|
|
2260
|
+
neCell +
|
|
2261
|
+
`</div></div>`
|
|
2262
|
+
);
|
|
2263
|
+
})
|
|
2264
|
+
.join("");
|
|
2265
|
+
return (
|
|
2266
|
+
`<h3>indeterminate resolution chain — applicable · pass · fail · N/A (precondition absent) · needs-evidence</h3>` +
|
|
2267
|
+
`<div class="sub">every indeterminate is RESOLVED by a deterministic precondition gate: <b>N/A</b> (the criterion's trigger/precondition was absent in the trace — DROPPED from the applicable denominator) vs <b>needs-evidence</b> (precondition present but undecided — carries a concrete next-action: code-recheck · 2nd-judge · revise-criterion · HITL-spot-check). No bare indeterminate.</div>` +
|
|
2268
|
+
`<div class="rztable">${body}</div>`
|
|
2269
|
+
);
|
|
2270
|
+
}
|
|
2271
|
+
|
|
2272
|
+
function scorecardTab(input: EvalReportInput): string {
|
|
2273
|
+
return (
|
|
2274
|
+
`<h2>③ Eval Scorecard</h2>` +
|
|
2275
|
+
tabDesc("What this tab shows: the DEFINED criteria × results. Each criterion carries its STATEMENT, its provenance (defined vs source-mined), and a hover with the full definition; the heatmap is criteria × route cohort, with nested subcards + inline calibration on the applicable-denominator-honest pass-rate.") +
|
|
2276
|
+
`<div class="sub">cohort heatmap · nested subcards · inline calibration · GA-D2b applicable-denominator honest</div>` +
|
|
2277
|
+
verdictLegend() +
|
|
2278
|
+
`<h3>criteria × route cohort — cell = % pass over (pass+fail+indet)</h3>` +
|
|
2279
|
+
heatmapHtml(input) +
|
|
2280
|
+
// WS-3 follow-up — the per-criterion plain-language KEY directly under the heatmap,
|
|
2281
|
+
// covering ALL criteria (gating + non-gating) so every technical heatmap id has a
|
|
2282
|
+
// "what it measures · why it matters" gloss.
|
|
2283
|
+
criterionLegend(input) +
|
|
2284
|
+
// UI-5 — anchor the heatmap with a success-rate + key-metrics summary row (no dangle).
|
|
2285
|
+
scorecardMetrics(input) +
|
|
2286
|
+
`<h3>gating criteria — nested subcards + HITL calibration</h3>` +
|
|
2287
|
+
subcardsHtml(input) +
|
|
2288
|
+
// WS-5 — the INDETERMINATE resolution chain for ALL criteria (gating + non-gating):
|
|
2289
|
+
// splits the single indeterminate bucket into N/A (precondition absent, excluded)
|
|
2290
|
+
// vs needs-evidence (carries a next-action). interest-logging-integrity's 283 land here.
|
|
2291
|
+
criterionResolutionTable(input)
|
|
2292
|
+
);
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
// ── §4 Findings (verbatim evidence + judge chain + agree/revise/refute) ───────
|
|
2296
|
+
|
|
2297
|
+
const KIND_ORDER: Record<string, number> = { context: 0, examine: 1, detect: 2, bind: 3, ground: 4, critique: 5, decide: 6, verify: 7 };
|
|
2298
|
+
|
|
2299
|
+
/** One §4 finding from the top-findings roll; uses DR-2 / judge-walk for evidence. */
|
|
2300
|
+
function findingFromTop(input: EvalReportInput, fd: TopFinding): string {
|
|
2301
|
+
const crit = criterionById(input, fd.criterion);
|
|
2302
|
+
const sev = sevCls(fd.severity ?? crit?.severity);
|
|
2303
|
+
const ex = fd.exampleTraceId ?? "";
|
|
2304
|
+
const row = input.ledger?.find((r) => r.trajectoryId === ex);
|
|
2305
|
+
const grounds = (row?.judgeSteps ?? []).filter((s) => s.ref);
|
|
2306
|
+
// a judge-step ref may be a string OR a structured {obs,path,value} — coerce.
|
|
2307
|
+
const refToStr = (rf: JudgeStep["ref"]): string =>
|
|
2308
|
+
rf === undefined ? "" : typeof rf === "string" ? rf : [rf.obs, rf.path, rf.value].filter(Boolean).join(":");
|
|
2309
|
+
const evItems =
|
|
2310
|
+
grounds.length > 0
|
|
2311
|
+
? grounds
|
|
2312
|
+
.slice(0, 4)
|
|
2313
|
+
.map((s) => { const r = refToStr(s.ref); return `<li><span class="ref">${esc(r.split(":")[0])}</span> ${esc(r.split(":").slice(1).join(":").slice(0, 120))}</li>`; })
|
|
2314
|
+
.join("")
|
|
2315
|
+
: crit?.discovery && crit.discovery.evidence.refs.length > 0
|
|
2316
|
+
? crit.discovery.evidence.refs
|
|
2317
|
+
.slice(0, 4)
|
|
2318
|
+
.map((r) => `<li><span class="ref">${esc(r.obs)}</span> ${esc(((r.path ? r.path + ": " : "") + r.value).slice(0, 120))}</li>`)
|
|
2319
|
+
.join("")
|
|
2320
|
+
: `<li><span class="ref">trace</span> <code>${esc(ex || "(suite)")}</code> — search §2 for the judge walk</li>`;
|
|
2321
|
+
const chain =
|
|
2322
|
+
(row?.judgeSteps ?? []).length > 0
|
|
2323
|
+
? [...(row?.judgeSteps ?? [])]
|
|
2324
|
+
.filter((s) => s.kind !== "context")
|
|
2325
|
+
.sort((a, b) => (KIND_ORDER[a.kind] ?? 9) - (KIND_ORDER[b.kind] ?? 9))
|
|
2326
|
+
.map((s) => `<span class="step">${esc(s.kind)}</span>`)
|
|
2327
|
+
.join(" → ")
|
|
2328
|
+
: "detect → ground → critique → decide → verify";
|
|
2329
|
+
const link =
|
|
2330
|
+
row?.judgeSteps && row.judgeSteps.length > 0 ? ` <span class="lnk" data-drill="${esc(ex)}">[open §2 side-by-side ▸]</span>` : "";
|
|
2331
|
+
const gate = sev === "crit" || sev === "high" ? `<span class="gatetag">GATING</span>` : "";
|
|
2332
|
+
const prev = fd.prevalencePctOverApplicable;
|
|
2333
|
+
const assumptionsLine =
|
|
2334
|
+
crit?.discovery && crit.discovery.assumptions.length > 0
|
|
2335
|
+
? crit.discovery.assumptions.map((a) => `${esc(a.text)} <span class="badge b-skip">${esc(a.status)}</span>`).join(" · ")
|
|
2336
|
+
: `<span class="none">grounded</span> — judge-health surfaced (see §5)`;
|
|
2337
|
+
const whyProblem = crit?.discovery ? `<div class="k">why a<br>problem</div><div class="v">${esc(crit.discovery.why_problem)}</div>` : "";
|
|
2338
|
+
return (
|
|
2339
|
+
`<div class="find ${sev === "crit" || sev === "high" ? "gate" : ""}">` +
|
|
2340
|
+
`<div class="find-h"><span class="sev ${sev}">${esc((fd.severity ?? "med").toUpperCase())}</span><b>${esc(fd.criterion)}</b>${gate}` +
|
|
2341
|
+
`<span class="verd fail" style="margin-left:auto">${esc(fd.failCount)}/${esc(fd.applicable)} fail · ${esc(prev)}%</span></div>` +
|
|
2342
|
+
// WS-3 — plain-language gloss (what · why) above the technical claim/evidence grid.
|
|
2343
|
+
plainBanner(crit) +
|
|
2344
|
+
`<div class="fg">` +
|
|
2345
|
+
`<div class="k">claim</div><div class="v">Criterion <code>${esc(fd.criterion)}</code> fails on ${esc(fd.failCount)} of ${esc(fd.applicable)} applicable trajectories.</div>` +
|
|
2346
|
+
whyProblem +
|
|
2347
|
+
`<div class="k">evidence<br>(verbatim)</div><div class="v" data-pii="evidence"><ul>${evItems}</ul></div>` +
|
|
2348
|
+
`<div class="k">judge<br>reasoning</div><div class="v reason">${chain}${link}</div>` +
|
|
2349
|
+
`<div class="k">root</div><div class="v">${esc(fd.root ?? "")}</div>` +
|
|
2350
|
+
`<div class="k">prevalence</div><div class="v"><b>${esc(prev)}%</b> over applicable (${esc(fd.applicable)}) <span class="prev"><i style="width:${Math.min(prev, 100)}%"></i></span> <span class="mono na-note">na excluded (GA-D2b)</span></div>` +
|
|
2351
|
+
`<div class="k">assumptions</div><div class="v">${assumptionsLine}</div>` +
|
|
2352
|
+
`</div>` +
|
|
2353
|
+
`<div class="areview"><span class="l">⊕ alignment review:</span><b class="agree">✓ agree</b><b>~ revise</b><b class="refute">✗ refute</b><input placeholder="reviewer note…"></div>` +
|
|
2354
|
+
`</div>`
|
|
2355
|
+
);
|
|
2356
|
+
}
|
|
2357
|
+
|
|
2358
|
+
/** Fallback DR-2 finding card (no ledger / no top-findings — mined criteria only). */
|
|
2359
|
+
function findingFromCriterion(c: ReportCriterion, verdict: CriterionVerdict | undefined): string {
|
|
2360
|
+
const sev = sevCls(c.severity);
|
|
2361
|
+
const d = c.discovery;
|
|
2362
|
+
if (!d) {
|
|
2363
|
+
return (
|
|
2364
|
+
`<div class="find"><div class="find-h"><span class="sev ${sev}">${esc(c.severity)}</span><b>${esc(c.id)}</b>` +
|
|
2365
|
+
`${verdict ? `<span class="verd ${verdictCls(resultToken(verdict.result))}" style="margin-left:auto">${esc(resultToken(verdict.result))}</span>` : ""}</div>` +
|
|
2366
|
+
plainBanner(c) +
|
|
2367
|
+
`<div class="fg"><div class="k">statement</div><div class="v">${esc(c.statement)}</div>` +
|
|
2368
|
+
`${verdict ? `<div class="k">critique</div><div class="v">${esc(verdict.critique)}</div>` : ""}</div></div>`
|
|
2369
|
+
);
|
|
2370
|
+
}
|
|
2371
|
+
const ev = d.evidence;
|
|
2372
|
+
const refs = ev.refs.length > 0 ? `<li><span class="ref">refs</span> <code>${esc(refsText(ev.refs))}</code></li>` : "";
|
|
2373
|
+
const assumptions = d.assumptions.map((a) => `${esc(a.text)} <span class="badge b-skip">${esc(a.status)}</span>`).join(" · ");
|
|
2374
|
+
return (
|
|
2375
|
+
`<div class="find ${sev === "crit" || sev === "high" ? "gate" : ""}">` +
|
|
2376
|
+
`<div class="find-h"><span class="sev ${sev}">${esc(c.severity)}</span><b>${esc(c.id)}</b>` +
|
|
2377
|
+
`${verdict ? `<span class="verd ${verdictCls(resultToken(verdict.result))}" style="margin-left:auto">${esc(resultToken(verdict.result))}</span>` : ""}</div>` +
|
|
2378
|
+
plainBanner(c) +
|
|
2379
|
+
`<div class="fg">` +
|
|
2380
|
+
`<div class="k">targets</div><div class="v">${esc(d.targets)}</div>` +
|
|
2381
|
+
`<div class="k">why a<br>problem</div><div class="v">${esc(d.why_problem)}</div>` +
|
|
2382
|
+
`<div class="k">evidence<br>(verbatim)</div><div class="v" data-pii="evidence">grounding <strong>${esc(ev.grounding)}</strong> · seen-in-traces ${esc(ev.seen_in_traces)} · prevalence <strong>${esc(ev.prevalence)}</strong><ul>${refs}</ul></div>` +
|
|
2383
|
+
`<div class="k">judge<br>reasoning</div><div class="v reason">${esc(d.reasoning)}</div>` +
|
|
2384
|
+
`<div class="k">assumptions</div><div class="v">${assumptions}</div>` +
|
|
2385
|
+
`</div>` +
|
|
2386
|
+
`<div class="areview"><span class="l">⊕ alignment review:</span><b class="agree">✓ agree</b><b>~ revise</b><b class="refute">✗ refute</b><input placeholder="reviewer note…"></div>` +
|
|
2387
|
+
`</div>`
|
|
2388
|
+
);
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
function findingsTab(input: EvalReportInput): string {
|
|
2392
|
+
const top = input.topFindings ?? [];
|
|
2393
|
+
let body: string;
|
|
2394
|
+
if (top.length > 0) {
|
|
2395
|
+
body = top.map((fd) => findingFromTop(input, fd)).join("");
|
|
2396
|
+
} else {
|
|
2397
|
+
// no ledger-derived findings — fall back to DR-2 cards for failing/at-risk criteria.
|
|
2398
|
+
const verdictById = new Map(input.verdicts.map((v) => [v.criterionId, v]));
|
|
2399
|
+
const failing = input.criteria.filter((c) => {
|
|
2400
|
+
const v = verdictById.get(c.id);
|
|
2401
|
+
return v ? v.result !== OutcomeVerdict.Pass : c.discovery !== undefined;
|
|
2402
|
+
});
|
|
2403
|
+
const pool = failing.length > 0 ? failing : input.criteria;
|
|
2404
|
+
body = pool.map((c) => findingFromCriterion(c, verdictById.get(c.id))).join("");
|
|
2405
|
+
}
|
|
2406
|
+
// §9.4.4 R3/M5 — DETECTED-but-unmatched flags, CLEARLY SEPARATED from the
|
|
2407
|
+
// fails-on-existing-criteria above. These are detections the judge flagged with NO
|
|
2408
|
+
// matching criterion — routed to *discover-evals, never minted into evals mid-judging.
|
|
2409
|
+
const flags = input.detectedFlags ?? [];
|
|
2410
|
+
const refToStr = (rf: DetectedFlag["ref"]): string =>
|
|
2411
|
+
rf === undefined ? "" : typeof rf === "string" ? rf : [rf.obs, rf.path, rf.value].filter(Boolean).join(":");
|
|
2412
|
+
const detectedSection =
|
|
2413
|
+
flags.length > 0
|
|
2414
|
+
? `<h3 class="detected-h">⚠ Detected — unmatched (no defined criterion) <span class="badge b-skip">NOT SCORED</span></h3>` +
|
|
2415
|
+
`<div class="sub">M5 JUDGE-WHAT-IS: these behaviours were DETECTED + FLAGGED but have no matching criterion — routed to <code>*discover-evals</code> / <code>*build-dataset</code>, never scored or minted into evals mid-judging.</div>` +
|
|
2416
|
+
flags
|
|
2417
|
+
.map((fl) => {
|
|
2418
|
+
const r = refToStr(fl.ref);
|
|
2419
|
+
return (
|
|
2420
|
+
`<div class="detected"><div class="detected-top"><span class="dkind">${esc(fl.kind)}</span>` +
|
|
2421
|
+
`<b>${esc(fl.detection)}</b><span class="dtrace mono">${esc(fl.trajectoryId)}${fl.anchor !== undefined ? " · step " + esc(fl.anchor) : ""}</span></div>` +
|
|
2422
|
+
(r ? `<div class="detected-ev mono"><span class="ref">${esc(r.split(":")[0])}</span> ${esc(r.split(":").slice(1).join(":"))}</div>` : "") +
|
|
2423
|
+
`</div>`
|
|
2424
|
+
);
|
|
2425
|
+
})
|
|
2426
|
+
.join("")
|
|
2427
|
+
: `<h3 class="detected-h">⚠ Detected — unmatched (no defined criterion)</h3><div class="sub">none — every detected behaviour had a matching defined criterion this run.</div>`;
|
|
2428
|
+
|
|
2429
|
+
return (
|
|
2430
|
+
`<h2>④ Findings</h2>` +
|
|
2431
|
+
tabDesc("What this tab shows: TWO clearly separated groups — (1) FAILURES on the existing DEFINED criteria (with verbatim evidence + judge reasoning chain), and (2) DETECTED-but-unmatched flags the judge raised with no matching criterion (routed to discover, never scored).") +
|
|
2432
|
+
`<div class="sub">alignment-auditable — verbatim evidence + judge reasoning chain</div>` +
|
|
2433
|
+
`<h3 class="fails-h">✗ Failures on defined criteria</h3>` +
|
|
2434
|
+
body +
|
|
2435
|
+
detectedSection +
|
|
2436
|
+
`<div class="note"><span class="tag">★ AUDITABLE</span> Each finding: verbatim evidence (obs/diff refs from the example trace's judge walk), the judge reasoning chain, prevalence over the applicable denominator, agree/revise/refute to overturn. Open §2 to see the full side-by-side.</div>`
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
// ── §5 Self-Eval · Calibration [INTERNAL — stripped on publish] ───────────────
|
|
2441
|
+
|
|
2442
|
+
const PHASE_LENS_ORDER = ["gather", "expect", "context", "examine", "detect", "bind", "ground", "critique", "decide", "verify", "localize"];
|
|
2443
|
+
|
|
2444
|
+
/** §9.4.4 R2/M1 — the subject-profile card for the INTERNAL calibration lens. */
|
|
2445
|
+
function subjectProfileCard(p: SubjectProfile | undefined): string {
|
|
2446
|
+
if (p === undefined) {
|
|
2447
|
+
return `<div class="sub">no subject profile supplied for this run — the judge reconstructed identity at reason-time.</div>`;
|
|
2448
|
+
}
|
|
2449
|
+
const inferred = new Set(p.inferredFields ?? []);
|
|
2450
|
+
const tag = (field: string): string => (inferred.has(field) ? ` <span class="badge b-skip">inferred</span>` : "");
|
|
2451
|
+
const rows: [string, string, string][] = [
|
|
2452
|
+
["identity", p.identity, "identity"],
|
|
2453
|
+
["purpose", p.purpose, "purpose"],
|
|
2454
|
+
["scope", p.scope, "scope"],
|
|
2455
|
+
["skill", p.skill ?? "—", "skill"],
|
|
2456
|
+
["tools", (p.tools ?? []).join(", ") || "—", "tools"],
|
|
2457
|
+
["harness", p.harness, "harness"],
|
|
2458
|
+
["version", p.version ?? "—", "version"],
|
|
2459
|
+
];
|
|
2460
|
+
const prov = `<span class="prov ${p.provenance === "given" ? "defined" : "source"}">${esc(p.provenance)}</span>`;
|
|
2461
|
+
const body = rows
|
|
2462
|
+
.map(([k, v, field]) => `<div class="pk">${esc(k)}${tag(field)}</div><div class="pv">${esc(v)}</div>`)
|
|
2463
|
+
.join("");
|
|
2464
|
+
return (
|
|
2465
|
+
`<div class="sub">M1 — who the judge understood the agent to be (${prov}; <span class="badge b-skip">inferred</span> = reconstructed, not given · harness <code>unknown</code> is MARKED, never confabulated).</div>` +
|
|
2466
|
+
`<div class="profile-grid">${body}</div>`
|
|
2467
|
+
);
|
|
2468
|
+
}
|
|
2469
|
+
|
|
2470
|
+
/** §9.4.4 R2/M4 — the PHASE-BLOCK lens: the judge's reasoning grouped by DAG node
|
|
2471
|
+
* across the whole run — the calibration surface the builder reads like an AI eng. */
|
|
2472
|
+
function phaseLens(input: EvalReportInput): string {
|
|
2473
|
+
const ledger = input.ledger ?? [];
|
|
2474
|
+
const byPhase = new Map<string, { count: number; samples: string[] }>();
|
|
2475
|
+
for (const r of ledger) {
|
|
2476
|
+
for (const s of r.judgeSteps ?? []) {
|
|
2477
|
+
const k = s.kind;
|
|
2478
|
+
const entry = byPhase.get(k) ?? { count: 0, samples: [] };
|
|
2479
|
+
entry.count += 1;
|
|
2480
|
+
if (s.text && entry.samples.length < 3) entry.samples.push(s.text);
|
|
2481
|
+
byPhase.set(k, entry);
|
|
2482
|
+
}
|
|
2483
|
+
}
|
|
2484
|
+
if (byPhase.size === 0) {
|
|
2485
|
+
return `<div class="sub">no judge_steps emitted this run — phase-block lens unavailable (scorecard-only render).</div>`;
|
|
2486
|
+
}
|
|
2487
|
+
const orderedKinds = [
|
|
2488
|
+
...PHASE_LENS_ORDER.filter((k) => byPhase.has(k)),
|
|
2489
|
+
...[...byPhase.keys()].filter((k) => !PHASE_LENS_ORDER.includes(k)),
|
|
2490
|
+
];
|
|
2491
|
+
return (
|
|
2492
|
+
`<div class="phase-lens">` +
|
|
2493
|
+
orderedKinds
|
|
2494
|
+
.map((k) => {
|
|
2495
|
+
const e = byPhase.get(k)!;
|
|
2496
|
+
const samples = e.samples.map((t) => `<div class="ps-row">${esc(t.slice(0, 140))}</div>`).join("");
|
|
2497
|
+
return (
|
|
2498
|
+
`<div class="phase-blk"><div class="phase-h"><span class="k ${esc(k)}">${esc(k)}</span><span class="phase-n">${esc(e.count)} step(s)</span></div>${samples}</div>`
|
|
2499
|
+
);
|
|
2500
|
+
})
|
|
2501
|
+
.join("") +
|
|
2502
|
+
`</div>`
|
|
2503
|
+
);
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
/** §9.4.4 R2/M2+M3 — an example node-0 understanding (train-of-thought) + node-0.5
|
|
2507
|
+
* expected-trajectory, from the first trajectory that carries them. */
|
|
2508
|
+
function reasoningExample(input: EvalReportInput): string {
|
|
2509
|
+
const ledger = input.ledger ?? [];
|
|
2510
|
+
const withU = ledger.find((r) => r.understanding !== undefined);
|
|
2511
|
+
const withE = ledger.find((r) => r.expectedTrajectory && r.expectedTrajectory.length > 0);
|
|
2512
|
+
if (withU === undefined && withE === undefined) {
|
|
2513
|
+
return `<div class="sub">no node-0 understanding / node-0.5 expected-trajectory emitted this run.</div>`;
|
|
2514
|
+
}
|
|
2515
|
+
let out = "";
|
|
2516
|
+
if (withU?.understanding !== undefined) {
|
|
2517
|
+
const u = withU.understanding;
|
|
2518
|
+
const given = (u.given ?? []).map((g) => `<li>${esc(g)}</li>`).join("");
|
|
2519
|
+
const inferred = (u.inferred ?? []).map((g) => `<li>${esc(g)}</li>`).join("");
|
|
2520
|
+
out +=
|
|
2521
|
+
`<div class="re-blk"><div class="re-h">node-0 GATHER — understanding (M2) · <span class="mono">${esc(withU.trajectoryId)}</span></div>` +
|
|
2522
|
+
`<div class="re-rephrase">“${esc(u.rephrase)}”</div>` +
|
|
2523
|
+
(given ? `<div class="re-sub">given</div><ul class="re-ul">${given}</ul>` : "") +
|
|
2524
|
+
(inferred ? `<div class="re-sub">inferred</div><ul class="re-ul">${inferred}</ul>` : "") +
|
|
2525
|
+
`</div>`;
|
|
2526
|
+
}
|
|
2527
|
+
if (withE?.expectedTrajectory !== undefined) {
|
|
2528
|
+
const steps = withE.expectedTrajectory
|
|
2529
|
+
.map((s: ExpectedStep, i: number) => `<li><b>${esc(s.step ?? i + 1)}.</b> ${esc(s.expected)}${s.rationale ? ` <span class="re-rat">— ${esc(s.rationale)}</span>` : ""}</li>`)
|
|
2530
|
+
.join("");
|
|
2531
|
+
out +=
|
|
2532
|
+
`<div class="re-blk"><div class="re-h">node-0.5 EXPECTED-TRAJECTORY (M3) · <span class="mono">${esc(withE.trajectoryId)}</span></div>` +
|
|
2533
|
+
`<div class="re-sub">how the target SHOULD have acted — built BEFORE examine</div><ol class="re-ul">${steps}</ol></div>`;
|
|
2534
|
+
}
|
|
2535
|
+
return out;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
// (UI-12-B groundedHealthCell removed in the §5 self-eval rebuild (WS-3): the per-cell
|
|
2539
|
+
// judge-health grounding readout was superseded by the aggregate Reference Integrity
|
|
2540
|
+
// panel (§5.1), which keeps the same no-false-green honesty. `rateClass` stays — used
|
|
2541
|
+
// by the §3 pass-rate tiles + the per-criterion rows.)
|
|
2542
|
+
|
|
2543
|
+
/**
|
|
2544
|
+
* WS-4 — one parsed routed-failure. The EV-051 handover encodes each routed item as a
|
|
2545
|
+
* deterministic acceptance string (`routeFailures`):
|
|
2546
|
+
* `<criterionId> [<severity>/<flag>] FAILED on trace <traceId>: <critique>`
|
|
2547
|
+
* so the §5 handover is reconstructed STRUCTURALLY (WHAT · WHY · WHERE · TARGET) from
|
|
2548
|
+
* it — no new data, just a parse. A string that doesn't match the format degrades to
|
|
2549
|
+
* `{criterionId: <raw>}` (rendered as WHAT, never dropped).
|
|
2550
|
+
*/
|
|
2551
|
+
interface RoutedItem {
|
|
2552
|
+
criterionId: string;
|
|
2553
|
+
severity: string;
|
|
2554
|
+
flag: string;
|
|
2555
|
+
traceId: string;
|
|
2556
|
+
critique: string;
|
|
2557
|
+
raw: string;
|
|
2558
|
+
}
|
|
2559
|
+
function parseRoutedCriterion(c: string): RoutedItem {
|
|
2560
|
+
const m = c.match(/^(.+?)\s+\[(.+?)\/(.+?)\]\s+FAILED on trace\s+(.+?):\s+([\s\S]*)$/);
|
|
2561
|
+
if (m) return { criterionId: m[1], severity: m[2], flag: m[3], traceId: m[4], critique: m[5].trim(), raw: c };
|
|
2562
|
+
return { criterionId: c, severity: "", flag: "", traceId: "", critique: "", raw: c };
|
|
2563
|
+
}
|
|
2564
|
+
|
|
2565
|
+
/**
|
|
2566
|
+
* WS-4 — the refined EV-051 "Routed to diagnostics" handover. Restructures the raw
|
|
2567
|
+
* acceptance-criteria dump into a clean, COPY-PASTEABLE handover block: a meta strip
|
|
2568
|
+
* (subject · target stage · escalation · produced-by) + one card per routed item with
|
|
2569
|
+
* WHAT (the criterion/finding) · WHY (the judge critique) · WHERE (the locus — trace +
|
|
2570
|
+
* dimension) · TARGET (diagnostics). Each card carries a per-item "copy" button (this
|
|
2571
|
+
* item as markdown) and the block ends with a "copy full handover as markdown" button —
|
|
2572
|
+
* the operator hands the bundle straight to diagnostics. PURE; deterministic markdown.
|
|
2573
|
+
*/
|
|
2574
|
+
function routedHandover(input: EvalReportInput): string {
|
|
2575
|
+
const h = input.handover;
|
|
2576
|
+
if (h === null || h === undefined) {
|
|
2577
|
+
return `<div class="sub">No failures routed — nothing handed to diagnostics this run (judge-only: the evaluator flags + routes, it never fixes).</div>`;
|
|
2578
|
+
}
|
|
2579
|
+
const subjName = h.subject?.name ?? input.subject.name;
|
|
2580
|
+
const target = h.intent?.command ?? "*diagnose";
|
|
2581
|
+
const items = h.acceptance.criteria.map(parseRoutedCriterion);
|
|
2582
|
+
|
|
2583
|
+
// per-item markdown (one routed finding, copy-pasteable into diagnostics).
|
|
2584
|
+
const itemMd = (p: RoutedItem, crit: ReportCriterion | undefined): string =>
|
|
2585
|
+
[
|
|
2586
|
+
`## ${p.criterionId}${p.severity ? ` [${p.severity}/${p.flag}]` : ""}`,
|
|
2587
|
+
`- **WHAT:** ${crit?.statement ?? p.criterionId}`,
|
|
2588
|
+
`- **WHY:** ${p.critique || "(no critique recorded)"}`,
|
|
2589
|
+
`- **WHERE:** ${p.traceId ? `trace ${p.traceId}` : "—"}${crit?.dimension ? ` · ${crit.dimension}` : ""}`,
|
|
2590
|
+
`- **TARGET:** diagnostics (${target})`,
|
|
2591
|
+
].join("\n");
|
|
2592
|
+
|
|
2593
|
+
const cards = items
|
|
2594
|
+
.map((p, i) => {
|
|
2595
|
+
const crit = criterionById(input, p.criterionId);
|
|
2596
|
+
const sev = sevCls(p.severity || crit?.severity);
|
|
2597
|
+
const sevTxt = String(p.severity || crit?.severity || "MED").toUpperCase();
|
|
2598
|
+
const where = p.traceId
|
|
2599
|
+
? `trace <code>${esc(p.traceId)}</code>${crit?.dimension ? ` · <span class="hov-dim">${esc(crit.dimension)}</span>` : ""}`
|
|
2600
|
+
: "—";
|
|
2601
|
+
return (
|
|
2602
|
+
`<div class="hov">` +
|
|
2603
|
+
`<div class="hov-h"><span class="hov-n">#${i + 1}</span><span class="sev ${sev}">${esc(sevTxt)}</span>` +
|
|
2604
|
+
`<b>${esc(p.criterionId)}</b>${p.flag ? `<span class="chip">${esc(p.flag)}</span>` : ""}` +
|
|
2605
|
+
`<button class="copy-md hov-copy" data-md="${esc(itemMd(p, crit))}">copy item ▸</button></div>` +
|
|
2606
|
+
`<div class="hov-g">` +
|
|
2607
|
+
`<div class="hk">WHAT</div><div class="hv">${esc(crit?.statement ?? p.criterionId)}</div>` +
|
|
2608
|
+
`<div class="hk">WHY</div><div class="hv">${esc(p.critique || "—")}</div>` +
|
|
2609
|
+
`<div class="hk">WHERE</div><div class="hv">${where}</div>` +
|
|
2610
|
+
`<div class="hk">TARGET</div><div class="hv">→ diagnostics (<code>${esc(target)}</code>)</div>` +
|
|
2611
|
+
`</div></div>`
|
|
2612
|
+
);
|
|
2613
|
+
})
|
|
2614
|
+
.join("");
|
|
2615
|
+
|
|
2616
|
+
// full-bundle markdown (the whole routed set, ready to hand to diagnostics).
|
|
2617
|
+
const fullMd = [
|
|
2618
|
+
`# EV-051 — Routed to diagnostics`,
|
|
2619
|
+
``,
|
|
2620
|
+
`**Subject:** ${subjName} · **Stage:** ${h.adl_stage} · **Escalation:** ${h.escalation_policy} · **By:** ${h.provenance.produced_by ?? "evaluator"}`,
|
|
2621
|
+
``,
|
|
2622
|
+
`**Goal:** ${h.acceptance.goal}`,
|
|
2623
|
+
``,
|
|
2624
|
+
...items.flatMap((p) => [itemMd(p, criterionById(input, p.criterionId)), ``]),
|
|
2625
|
+
].join("\n");
|
|
2626
|
+
|
|
2627
|
+
return (
|
|
2628
|
+
`<div class="sub">EV-051 — failing criteria ROUTED to diagnostics (judge-only; the evaluator flags + routes, it never fixes). Hand the block below straight to <code>*diagnose</code>.</div>` +
|
|
2629
|
+
`<div class="hov-meta">` +
|
|
2630
|
+
`<span class="hm-c"><span class="l">subject</span><span class="v">${esc(subjName)}</span></span>` +
|
|
2631
|
+
`<span class="hm-c"><span class="l">target stage</span><span class="v"><code>${esc(target)}</code></span></span>` +
|
|
2632
|
+
`<span class="hm-c"><span class="l">routed</span><span class="v">${items.length} finding(s)</span></span>` +
|
|
2633
|
+
`<span class="hm-c"><span class="l">escalation</span><span class="v">${esc(h.escalation_policy)}</span></span>` +
|
|
2634
|
+
`<span class="hm-c"><span class="l">produced by</span><span class="v">${esc(h.provenance.produced_by)}</span></span>` +
|
|
2635
|
+
`</div>` +
|
|
2636
|
+
`<div class="hov-list">${cards}</div>` +
|
|
2637
|
+
`<button class="copy-md" data-md="${esc(fullMd)}">⧉ Copy full handover as markdown</button>`
|
|
2638
|
+
);
|
|
2639
|
+
}
|
|
2640
|
+
|
|
2641
|
+
function selfEvalTab(input: EvalReportInput): string {
|
|
2642
|
+
const se = input.selfEval;
|
|
2643
|
+
const decisions = routedHandover(input);
|
|
2644
|
+
const header =
|
|
2645
|
+
`<h2>⑤ Self-Eval · Judge Calibration <span class="badge b-skip">INTERNAL</span></h2>` +
|
|
2646
|
+
tabDesc("What this tab shows [INTERNAL — stripped on publish]: the AGGREGATE judge-calibration surface — the human-in-the-loop for the JUDGE. It audits the METHODOLOGY the judge followed across the WHOLE population (not one trace): ① is the judge applying the method · ② do the criteria make sense / add value · ③ did it read enough traces (diligence) · ④ are results grounded · ⑤ are the refs real or fabricated. Accuracy-first ordering.") +
|
|
2647
|
+
`<div class="sub" data-strip="strip-for-client">internal — calibrate the judge · stripped on publish</div>`;
|
|
2648
|
+
|
|
2649
|
+
if (se === undefined) {
|
|
2650
|
+
// graceful fallback — no aggregate derived (older run): keep the EV-051 decisions.
|
|
2651
|
+
return header + `<h3>Routed to diagnostics (EV-051)</h3>` + decisions;
|
|
2652
|
+
}
|
|
2653
|
+
|
|
2654
|
+
const checksLegend =
|
|
2655
|
+
`<div class="se-checks"><div class="se-checks-h">the 5 operator checks this tab answers</div><div class="se-checks-g">` +
|
|
2656
|
+
[
|
|
2657
|
+
["①", "methodology adherence", "is the judge applying the judging method (bind · ground · critique-before-verdict · abstain-on-silence)?"],
|
|
2658
|
+
["②", "criterion value", "do the criteria make sense, apply, and discriminate — or are they dead weight?"],
|
|
2659
|
+
["③", "diligence", "did the judge read enough of each trace, or pick one sample and stop?"],
|
|
2660
|
+
["④", "grounded", "are the results anchored in real trace evidence (refs), not vibes?"],
|
|
2661
|
+
["⑤", "refs real", "do the cited {obs,path,value} refs actually resolve in the trace — real or fabricated?"],
|
|
2662
|
+
]
|
|
2663
|
+
.map(([n, k, v]) => `<div class="se-check"><span class="se-cn">${n}</span><span class="se-ck">${esc(k)}</span><span class="se-cv">${esc(v)}</span></div>`)
|
|
2664
|
+
.join("") +
|
|
2665
|
+
`</div></div>`;
|
|
2666
|
+
|
|
2667
|
+
return (
|
|
2668
|
+
header +
|
|
2669
|
+
checksLegend +
|
|
2670
|
+
selfEvalTrustBand(se) +
|
|
2671
|
+
selfEvalBehaviorMap(se) +
|
|
2672
|
+
selfEvalRefIntegrity(se) +
|
|
2673
|
+
selfEvalConfidence(se) +
|
|
2674
|
+
selfEvalPerCriterion(se) +
|
|
2675
|
+
selfEvalCriterionValue(se) +
|
|
2676
|
+
selfEvalDiligence(se) +
|
|
2677
|
+
selfEvalTransparency(se) +
|
|
2678
|
+
selfEvalEmit(se) +
|
|
2679
|
+
selfEvalSpotCheck(se) +
|
|
2680
|
+
selfEvalGroundTruthFuture() +
|
|
2681
|
+
// ── judge-reasoning DETAIL (M1 · M2 · M3 · M4) — the per-trace calibration lens
|
|
2682
|
+
// that backs the aggregate above (who the judge thinks the agent is, its
|
|
2683
|
+
// train-of-thought, the expected-trajectory it built). Retained below the
|
|
2684
|
+
// aggregate so a builder can drill from the population view into an example.
|
|
2685
|
+
`<h3 class="se-h"><span class="se-num">5.D</span> Judge reasoning detail (M1 · M2 · M3 · M4)</h3>` +
|
|
2686
|
+
`<div class="sub">the per-trace calibration lens behind the aggregate — who the judge understood the agent to be (M1), its phase-by-phase reasoning (M4), and an example understanding + expected-trajectory (M2 · M3).</div>` +
|
|
2687
|
+
subjectProfileCard(input.subjectProfile) +
|
|
2688
|
+
phaseLens(input) +
|
|
2689
|
+
reasoningExample(input) +
|
|
2690
|
+
`<h3 class="se-h">Routed to diagnostics (EV-051)</h3>` +
|
|
2691
|
+
decisions
|
|
2692
|
+
);
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2695
|
+
// ── WS-3 §5 Self-Eval — section renderers (PURE string assembly) ──────────────
|
|
2696
|
+
|
|
2697
|
+
function seBar(pct: number, tone = "p"): string {
|
|
2698
|
+
const w = Math.max(0, Math.min(100, pct));
|
|
2699
|
+
return `<span class="sebar"><span class="sebar-f ${tone}" style="width:${w}%"></span></span>`;
|
|
2700
|
+
}
|
|
2701
|
+
|
|
2702
|
+
function selfEvalTrustBand(se: SelfEvalAggregate): string {
|
|
2703
|
+
const t = se.trust;
|
|
2704
|
+
const tone = t.score >= 85 ? "good" : t.score >= 60 ? "warn" : "bad";
|
|
2705
|
+
return (
|
|
2706
|
+
`<div class="se-trust ${tone}"><div class="se-trust-l"><div class="se-trust-score">${esc(t.score)}<span class="se-trust-d">/100</span></div><div class="se-trust-cap">judge trust</div></div>` +
|
|
2707
|
+
`<div class="se-trust-r"><div class="se-trust-label">${esc(t.label)}</div><div class="se-trust-basis">${esc(t.basis)}</div></div></div>`
|
|
2708
|
+
);
|
|
2709
|
+
}
|
|
2710
|
+
|
|
2711
|
+
function selfEvalBehaviorMap(se: SelfEvalAggregate): string {
|
|
2712
|
+
const b = se.behaviorMap;
|
|
2713
|
+
const clusters = b.clusters
|
|
2714
|
+
.map(
|
|
2715
|
+
(c) =>
|
|
2716
|
+
`<div class="se-cl ${esc(c.tone)}"><div class="se-cl-top"><span class="se-cl-n">${esc(c.count)}</span><span class="se-cl-l">${esc(c.label)}</span></div><div class="se-cl-note">${esc(c.note)}</div></div>`,
|
|
2717
|
+
)
|
|
2718
|
+
.join("");
|
|
2719
|
+
const conc = b.concentration.length
|
|
2720
|
+
? `<div class="se-conc"><div class="se-conc-h">where fails CONCENTRATE</div>` +
|
|
2721
|
+
b.concentration
|
|
2722
|
+
.map(
|
|
2723
|
+
(c) =>
|
|
2724
|
+
`<div class="se-conc-r"><span class="se-conc-id" title="${esc(c.statement)}">${esc(c.id)}</span>${seBar(c.pct, "bad")}<span class="se-conc-v">${esc(c.fails)} fails · ${esc(c.pct)}%</span></div>`,
|
|
2725
|
+
)
|
|
2726
|
+
.join("") +
|
|
2727
|
+
`</div>`
|
|
2728
|
+
: `<div class="se-empty">no failing criteria — nothing concentrates.</div>`;
|
|
2729
|
+
return (
|
|
2730
|
+
`<h3 class="se-h"><span class="se-num">5.0</span> Judge Behavior Map — clusters & where it falls short <span class="se-tag agg">AGGREGATE · MULTI-TRAJECTORY</span></h3>` +
|
|
2731
|
+
`<div class="sub">how the judge handled the POPULATION of ${esc(b.total)} trajectories — the cluster sizes + the fail concentration tell you where the judge (and the criteria) are working vs thin.</div>` +
|
|
2732
|
+
`<div class="se-clusters">${clusters}</div>` +
|
|
2733
|
+
conc
|
|
2734
|
+
);
|
|
2735
|
+
}
|
|
2736
|
+
|
|
2737
|
+
function selfEvalRefIntegrity(se: SelfEvalAggregate): string {
|
|
2738
|
+
const r = se.refIntegrity;
|
|
2739
|
+
// honesty (UI-12-B): with ZERO decided verdicts grounding is UNMEASURABLE — render
|
|
2740
|
+
// capture-unavailable (muted), NEVER a false-green default.
|
|
2741
|
+
if (r.decided === 0) {
|
|
2742
|
+
return (
|
|
2743
|
+
`<h3 class="se-h"><span class="se-num">5.1</span> Reference Integrity — real or fabricated? <span class="se-tag gt">check ⑤ · GROUND-TRUTH</span></h3>` +
|
|
2744
|
+
`<div class="se-row"><div class="se-metric"><div class="se-metric-v capture-na">capture-unavailable</div><div class="se-metric-l">no decided verdicts to ground</div></div></div>` +
|
|
2745
|
+
`<div class="se-note">${esc(r.note)}</div>`
|
|
2746
|
+
);
|
|
2747
|
+
}
|
|
2748
|
+
const tone = r.groundedPct >= 90 ? "good" : r.groundedPct >= 70 ? "warn" : "bad";
|
|
2749
|
+
return (
|
|
2750
|
+
`<h3 class="se-h"><span class="se-num">5.1</span> Reference Integrity — real or fabricated? <span class="se-tag gt">check ⑤ · GROUND-TRUTH</span></h3>` +
|
|
2751
|
+
`<div class="se-row"><div class="se-metric ${tone}"><div class="se-metric-v">${esc(r.groundedPct)}%</div><div class="se-metric-l">decided verdicts carry ≥1 structured ref</div></div>` +
|
|
2752
|
+
`<div class="se-kv"><div><b>${esc(r.grounded)}</b> grounded</div><div><b>${esc(r.ungrounded)}</b> ungrounded</div><div><b>${esc(r.decided)}</b> decided</div></div></div>` +
|
|
2753
|
+
`<div class="se-note">${esc(r.note)}</div>`
|
|
2754
|
+
);
|
|
2755
|
+
}
|
|
2756
|
+
|
|
2757
|
+
function selfEvalConfidence(se: SelfEvalAggregate): string {
|
|
2758
|
+
const rows = se.confidence
|
|
2759
|
+
.map(
|
|
2760
|
+
(c) =>
|
|
2761
|
+
`<div class="se-conf-r"><span class="se-conf-b">${esc(c.band)}</span>${seBar(c.pct, "p")}<span class="se-conf-v">${esc(c.count)} · ${esc(c.pct)}%</span></div>`,
|
|
2762
|
+
)
|
|
2763
|
+
.join("");
|
|
2764
|
+
return (
|
|
2765
|
+
`<h3 class="se-h"><span class="se-num">5.2</span> Confidence Calibration <span class="se-tag proxy">PROXY</span></h3>` +
|
|
2766
|
+
`<div class="sub">how the judge's stated confidence is distributed across decided verdicts. Tracking this against actual correctness (vs a 2nd judge / ground-truth) is the §5.X future extension.</div>` +
|
|
2767
|
+
`<div class="se-conf">${rows || '<div class="se-empty">no per-verdict confidence emitted.</div>'}</div>`
|
|
2768
|
+
);
|
|
2769
|
+
}
|
|
2770
|
+
|
|
2771
|
+
function selfEvalPerCriterion(se: SelfEvalAggregate): string {
|
|
2772
|
+
const rows = se.perCriterion
|
|
2773
|
+
.map((p) => {
|
|
2774
|
+
const flagCls = p.flag === "fails-present" ? "bad" : p.flag === "never-applies" ? "warn" : "good";
|
|
2775
|
+
return (
|
|
2776
|
+
`<tr><td class="se-cid" title="${esc(p.statement)}">${esc(p.id)}</td><td><span class="sev ${esc(p.severity)}">${esc(p.severity)}</span></td>` +
|
|
2777
|
+
`<td class="good">${esc(p.pass)}</td><td class="bad">${esc(p.fail)}</td><td class="warn">${esc(p.unc)}</td><td class="dim">${esc(p.na)}</td>` +
|
|
2778
|
+
`<td>${esc(p.groundedPct)}%</td><td><span class="se-flag ${flagCls}">${esc(p.flag)}</span></td></tr>`
|
|
2779
|
+
);
|
|
2780
|
+
})
|
|
2781
|
+
.join("");
|
|
2782
|
+
return (
|
|
2783
|
+
`<h3 class="se-h"><span class="se-num">5.3</span> Per-Criterion Reliability <span class="se-tag proxy">check ①④ · PROXY</span></h3>` +
|
|
2784
|
+
`<table class="se-table"><thead><tr><th>criterion</th><th>sev</th><th>pass</th><th>fail</th><th>indet</th><th>na</th><th>grnd</th><th>flag</th></tr></thead><tbody>${rows}</tbody></table>`
|
|
2785
|
+
);
|
|
2786
|
+
}
|
|
2787
|
+
|
|
2788
|
+
function selfEvalCriterionValue(se: SelfEvalAggregate): string {
|
|
2789
|
+
const rows = se.criterionValue
|
|
2790
|
+
.map((c, i) => {
|
|
2791
|
+
const cls = c.applicablePct === 0 ? "warn" : "good";
|
|
2792
|
+
return (
|
|
2793
|
+
`<tr><td class="se-cid" title="${esc(c.statement)}">${esc(c.id)}</td><td><span class="sev ${esc(c.severity)}">${esc(c.severity)}</span></td>` +
|
|
2794
|
+
`<td class="${cls}">${esc(c.applicablePct)}%</td><td>${esc(c.fails)} fails</td><td class="se-vverdict">${esc(c.verdict)}</td>` +
|
|
2795
|
+
`<td class="se-hitl"><span class="se-pick" data-vgroup="cval-${i}" data-v="keep">keep</span><span class="se-pick" data-vgroup="cval-${i}" data-v="revise">revise</span><span class="se-pick" data-vgroup="cval-${i}" data-v="retire">retire</span></td></tr>`
|
|
2796
|
+
);
|
|
2797
|
+
})
|
|
2798
|
+
.join("");
|
|
2799
|
+
return (
|
|
2800
|
+
`<h3 class="se-h"><span class="se-num">5.4</span> Criterion Value / Applicability Audit <span class="se-tag hitl">check ② · HITL</span></h3>` +
|
|
2801
|
+
`<div class="sub">does each criterion apply + discriminate, or is it dead weight? A criterion that NEVER applies or NEVER fails is a candidate to retire/revise. Your pick is captured below (export-ready).</div>` +
|
|
2802
|
+
`<table class="se-table"><thead><tr><th>criterion</th><th>sev</th><th>applicable</th><th>fails</th><th>value</th><th>operator</th></tr></thead><tbody>${rows}</tbody></table>`
|
|
2803
|
+
);
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
function selfEvalDiligence(se: SelfEvalAggregate): string {
|
|
2807
|
+
const d = se.diligence;
|
|
2808
|
+
const pct = d.total > 0 ? Math.round((100 * d.examined) / d.total) : 0;
|
|
2809
|
+
const tone = pct >= 95 ? "good" : pct >= 80 ? "warn" : "bad";
|
|
2810
|
+
return (
|
|
2811
|
+
`<h3 class="se-h"><span class="se-num">5.5</span> Diligence / Trace Coverage <span class="se-tag proxy">check ③ · PROXY</span></h3>` +
|
|
2812
|
+
`<div class="se-row"><div class="se-metric ${tone}"><div class="se-metric-v">${esc(d.examined)}/${esc(d.total)}</div><div class="se-metric-l">trajectories examined</div></div>` +
|
|
2813
|
+
`<div class="se-metric"><div class="se-metric-v">${esc(d.avgGroundingRefs)}</div><div class="se-metric-l">avg grounding refs / examined trace</div></div></div>` +
|
|
2814
|
+
`<div class="se-note">${esc(d.note)}</div>`
|
|
2815
|
+
);
|
|
2816
|
+
}
|
|
2817
|
+
|
|
2818
|
+
function selfEvalTransparency(se: SelfEvalAggregate): string {
|
|
2819
|
+
const t = se.transparency;
|
|
2820
|
+
const block = t.blocked
|
|
2821
|
+
? `<div class="se-blocked"><span class="se-blocked-tag">BLOCKED</span> ${esc(t.note)}</div>`
|
|
2822
|
+
: `<div class="se-note">${esc(t.note)}</div>`;
|
|
2823
|
+
return (
|
|
2824
|
+
`<h3 class="se-h"><span class="se-num">5.6</span> Judge Reasoning Transparency (walk · M2 · M3) <span class="se-tag">TRANSPARENCY · check ①</span></h3>` +
|
|
2825
|
+
`<div class="se-row"><div class="se-metric ${t.m2Pct > 0 ? "good" : "bad"}"><div class="se-metric-v">${esc(t.m2Pct)}%</div><div class="se-metric-l">M2 understanding</div></div>` +
|
|
2826
|
+
`<div class="se-metric ${t.m3Pct > 0 ? "good" : "bad"}"><div class="se-metric-v">${esc(t.m3Pct)}%</div><div class="se-metric-l">M3 expected-trajectory</div></div>` +
|
|
2827
|
+
`<div class="se-metric ${t.walkPct > 0 ? "good" : "bad"}"><div class="se-metric-v">${esc(t.walkPct)}%</div><div class="se-metric-l">judge walk emitted</div></div></div>` +
|
|
2828
|
+
block
|
|
2829
|
+
);
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
function selfEvalEmit(se: SelfEvalAggregate): string {
|
|
2833
|
+
const e = se.emit;
|
|
2834
|
+
const meter = (label: string, present: number, total: number): string => {
|
|
2835
|
+
const pct = total > 0 ? Math.round((100 * present) / total) : 0;
|
|
2836
|
+
const tone = pct >= 90 ? "good" : pct >= 40 ? "warn" : "bad";
|
|
2837
|
+
return `<div class="se-emit-r"><span class="se-emit-l">${esc(label)}</span>${seBar(pct, tone === "good" ? "p" : tone)}<span class="se-emit-v">${esc(present)}/${esc(total)}</span></div>`;
|
|
2838
|
+
};
|
|
2839
|
+
const fieldRow = (f: { field: string; present: number }): string =>
|
|
2840
|
+
meter(f.field, f.present, e.eligible);
|
|
2841
|
+
return (
|
|
2842
|
+
`<h3 class="se-h"><span class="se-num">5.7</span> Emit-Completeness — self-honesty meter <span class="se-tag">HONESTY</span></h3>` +
|
|
2843
|
+
`<div class="sub">does the judge persist what it reasoned? A dropped walk does NOT change a verdict — but it STARVES this tab + §2. The engine gate (assessEmitCompleteness) tracks this every run.</div>` +
|
|
2844
|
+
`<div class="se-emit">` +
|
|
2845
|
+
meter("verdict + critique", e.eligible, e.eligible) +
|
|
2846
|
+
e.fields.map(fieldRow).join("") +
|
|
2847
|
+
`</div>` +
|
|
2848
|
+
`<div class="se-note">complete emits (all of M2+M3+agentSteps+judgeSteps): <b>${esc(e.completeEmits)}/${esc(e.eligible)}</b> (${esc(e.completePct)}%).` +
|
|
2849
|
+
(e.exemptIncomplete > 0 ? ` ${esc(e.exemptIncomplete)} INCOMPLETE traces exempt (node-1 short-circuit).` : "") +
|
|
2850
|
+
`</div>`
|
|
2851
|
+
);
|
|
2852
|
+
}
|
|
2853
|
+
|
|
2854
|
+
function selfEvalSpotCheck(se: SelfEvalAggregate): string {
|
|
2855
|
+
if (se.spotCheck.length === 0) {
|
|
2856
|
+
return (
|
|
2857
|
+
`<h3 class="se-h"><span class="se-num">5.8</span> Spot-Check / Disagreement Queue <span class="se-tag hitl">check ④ · HITL</span></h3>` +
|
|
2858
|
+
`<div class="se-empty">no failing or low-confidence verdicts queued — nothing brittle to spot-check.</div>`
|
|
2859
|
+
);
|
|
2860
|
+
}
|
|
2861
|
+
const rows = se.spotCheck
|
|
2862
|
+
.map(
|
|
2863
|
+
(s, i) =>
|
|
2864
|
+
`<div class="se-spot"><span class="se-spot-t mono" title="${esc(s.criterion)}">${esc(s.trace.slice(0, 14))}</span>` +
|
|
2865
|
+
`<span class="se-spot-c">${esc(s.criterion.slice(0, 54))}</span>` +
|
|
2866
|
+
`<span class="verd ${s.verdict === "FAIL" ? "fail" : "inc"}">${esc(s.verdict)}</span>` +
|
|
2867
|
+
`<span class="se-spot-conf">conf ${esc(s.conf)}</span><span class="se-spot-why">${esc(s.reason)}</span>` +
|
|
2868
|
+
`<span class="se-spot-act"><span class="se-pick" data-vgroup="spot-${i}" data-v="agree">agree</span><span class="se-pick" data-vgroup="spot-${i}" data-v="disagree">disagree</span></span></div>`,
|
|
2869
|
+
)
|
|
2870
|
+
.join("");
|
|
2871
|
+
return (
|
|
2872
|
+
`<h3 class="se-h"><span class="se-num">5.8</span> Spot-Check / Disagreement Queue <span class="se-tag hitl">check ④ · HITL</span></h3>` +
|
|
2873
|
+
`<div class="sub">the brittle calls — fails + low-confidence verdicts — sorted weakest-confidence-first. Your agree/disagree is captured (and becomes the ground-truth seed for §5.X).</div>` +
|
|
2874
|
+
`<div class="se-spots">${rows}</div>`
|
|
2875
|
+
);
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2878
|
+
function selfEvalGroundTruthFuture(): string {
|
|
2879
|
+
return (
|
|
2880
|
+
`<h3 class="se-h"><span class="se-num">5.X</span> Judge Accuracy vs Ground-Truth <span class="se-tag future">FUTURE — not built</span></h3>` +
|
|
2881
|
+
`<div class="se-future"><div class="se-future-h">documented extension</div>` +
|
|
2882
|
+
`<div class="se-future-b">The strongest calibration is judge-verdict vs a TRUSTED label. We do not have ground-truth labels in this run, so this is DOCUMENTED, not computed. The seed already exists: §5.8 Spot-Check collects operator agree/disagree per verdict — accumulate those into a labelled set and this panel becomes a real accuracy/precision/recall readout against the judge. Until then, §5.1 (refs real) + §5.2 (confidence) + §5.3 (per-criterion) are the PROXY signals.</div></div>`
|
|
2883
|
+
);
|
|
2884
|
+
}
|
|
2885
|
+
|
|
2886
|
+
// ── report-component CSS (references theme.css tokens) ────────────────────────
|
|
2887
|
+
|
|
2888
|
+
const REPORT_CSS = `
|
|
2889
|
+
.mono{font-family:var(--mono)}
|
|
2890
|
+
/* status-acuity — color-code the OUTCOME big-stat tiles; COUNT tiles stay neutral (faint cyan) */
|
|
2891
|
+
/* ALIGNMENT FIX (parity w/ discover): theme.css ships .big-stat as content-sized flex
|
|
2892
|
+
* (min-width:96px) → ragged tile widths. Override to an EQUAL-column grid so every tile
|
|
2893
|
+
* is the same width AND height (grid row-stretch). */
|
|
2894
|
+
.big-stat{display:grid;grid-template-columns:repeat(auto-fit,minmax(120px,1fr));gap:12px;align-items:start}
|
|
2895
|
+
.big-stat .s{min-width:0;min-height:84px;display:flex;flex-direction:column;justify-content:center}
|
|
2896
|
+
.big-stat .s .v{line-height:1.15}
|
|
2897
|
+
.big-stat .s{border-left:4px solid var(--border-strong)}
|
|
2898
|
+
.big-stat .s.count{border-left-color:var(--cyan);opacity:.95}
|
|
2899
|
+
.big-stat .s.pass{background:var(--pass-bg);border-color:var(--pass);border-left-color:var(--pass)}
|
|
2900
|
+
.big-stat .s.pass .v{color:var(--pass)}
|
|
2901
|
+
.big-stat .s.fail{background:var(--fail-bg);border-color:var(--fail);border-left-color:var(--fail)}
|
|
2902
|
+
.big-stat .s.fail .v{color:var(--fail)}
|
|
2903
|
+
.big-stat .s.indet{background:var(--warn-bg);border-color:var(--warn);border-left-color:var(--warn)}
|
|
2904
|
+
.big-stat .s.indet .v{color:var(--warn)}
|
|
2905
|
+
/* Overview-redesign — entity HERO (full-width subject identity) */
|
|
2906
|
+
/* §9.4.5 E4 — entity-AS-HERO: a primary left-accent + tighter rhythm so the subject
|
|
2907
|
+
* card reads as the page anchor (diagnostics-grade density, dark/branded preserved). */
|
|
2908
|
+
.hero{border:1px solid var(--border-strong);border-left:3px solid var(--primary);background:var(--surf);margin:10px 0 14px;padding:0}
|
|
2909
|
+
.hero-top{display:flex;gap:10px;align-items:center;flex-wrap:wrap;padding:11px 14px;border-bottom:1px solid var(--border);background:var(--surf-2)}
|
|
2910
|
+
.hero .hname{font-size:var(--fs-xl);font-weight:700;color:var(--fg-strong);font-family:var(--mono)}
|
|
2911
|
+
.hero .hchip{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:2px 8px;border:1px solid var(--border);color:var(--muted);text-transform:uppercase;letter-spacing:.04em}
|
|
2912
|
+
.hero .hchip.type{color:var(--cyan);border-color:var(--cyan);background:rgba(69,184,204,.10)}
|
|
2913
|
+
.hero .hchip.access{margin-left:auto}
|
|
2914
|
+
.hero .hchip.access.given{color:var(--pass);border-color:var(--pass);background:var(--pass-bg)}
|
|
2915
|
+
.hero .hchip.access.recon{color:var(--warn);border-color:var(--warn);background:var(--warn-bg)}
|
|
2916
|
+
.hero-grid{display:grid;grid-template-columns:max-content 1fr;gap:1px;background:var(--border)}
|
|
2917
|
+
.hero-grid .hk{background:var(--surf);font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase;letter-spacing:.04em;padding:7px 12px}
|
|
2918
|
+
.hero-grid .hv{background:var(--surf-2);font-size:var(--fs-sm);color:var(--fg);padding:7px 13px;line-height:1.5}
|
|
2919
|
+
.hero .unk{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--dim);text-transform:uppercase;border:1px dashed var(--border-strong);padding:0 5px}
|
|
2920
|
+
.hero .inf{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;color:var(--warn);background:var(--warn-bg);padding:1px 5px;text-transform:uppercase}
|
|
2921
|
+
.hero .inf.given{color:var(--pass);background:var(--pass-bg)}
|
|
2922
|
+
.hero .hsys{margin:9px 12px;border:1px solid var(--border)}
|
|
2923
|
+
.hero .hsys summary{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;color:var(--cyan);cursor:pointer;padding:6px 10px;background:var(--surf);display:flex;gap:8px;align-items:center}
|
|
2924
|
+
.hero .hsys-prov{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:1px 6px;text-transform:uppercase;letter-spacing:.03em}
|
|
2925
|
+
.hero .hsys-prov.recon{color:var(--warn);background:var(--warn-bg);border:1px solid var(--warn)}
|
|
2926
|
+
.hero .hsys-prov.given{color:var(--pass);background:var(--pass-bg);border:1px solid var(--pass)}
|
|
2927
|
+
.hero .hsys pre{margin:0;padding:9px 11px;font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg);white-space:pre-wrap;word-break:break-word;line-height:1.5;background:var(--bg);max-height:420px;overflow:auto}
|
|
2928
|
+
.hero .hsys-na{margin:9px 12px;font-family:var(--mono);font-size:var(--fs-2xs);color:var(--dim)}
|
|
2929
|
+
.hero .hhonesty{padding:8px 13px;border-top:1px solid var(--border);font-size:var(--fs-2xs);color:var(--muted);line-height:1.5}
|
|
2930
|
+
/* §9.4.5 E1 — trace-only honesty: the reconstructed banner + the harness/prompt UNAVAILABLE note */
|
|
2931
|
+
.hero .hrecon{padding:7px 14px;border-bottom:1px solid var(--border);background:var(--warn-bg);font-family:var(--mono);font-size:var(--fs-2xs);font-weight:600;color:var(--warn);letter-spacing:.03em;line-height:1.5}
|
|
2932
|
+
.hero .recon-note{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--warn);text-transform:uppercase;letter-spacing:.03em}
|
|
2933
|
+
/* §9.4.5 E2 — tools as diagnostics-style chips with a per-tool observed call-count */
|
|
2934
|
+
.hero .tchips{display:flex;gap:5px;flex-wrap:wrap;align-items:center}
|
|
2935
|
+
.hero .tchip{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--fg);background:var(--surf-3);border:1px solid var(--border-strong);padding:2px 7px;display:inline-flex;gap:5px;align-items:center;white-space:nowrap}
|
|
2936
|
+
.hero .tchip .obs{font-weight:700;color:var(--cyan);font-size:var(--fs-2xs)}
|
|
2937
|
+
.hero .tchip .obs.none{color:var(--dim)}
|
|
2938
|
+
/* Overview-redesign — provenance META-STRIP (run-config + judge substrate) */
|
|
2939
|
+
.provstrip{display:flex;gap:7px 14px;flex-wrap:wrap;align-items:center;border:1px solid var(--border);border-left:3px solid var(--primary);background:var(--surf-2);padding:7px 12px;margin:10px 0}
|
|
2940
|
+
.provstrip .ps-item{display:inline-flex;gap:5px;align-items:baseline}
|
|
2941
|
+
.provstrip .ps-k{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.04em;color:var(--muted)}
|
|
2942
|
+
.provstrip .ps-v{font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg-strong)}
|
|
2943
|
+
.provstrip .unk{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--dim);text-transform:uppercase}
|
|
2944
|
+
.provstrip .ps-badge{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:2px 8px;border:1px solid;text-transform:uppercase}
|
|
2945
|
+
.provstrip .ps-badge.on{color:var(--primary-soft);border-color:var(--primary);background:rgba(126,71,215,.12)}
|
|
2946
|
+
.provstrip .ps-badge.off{color:var(--dim);border-color:var(--border)}
|
|
2947
|
+
/* Overview-redesign — segmented coverage FUNNEL (INCOMPLETE first-class) */
|
|
2948
|
+
.funnel{display:flex;align-items:stretch;gap:6px;flex-wrap:wrap;margin:14px 0 6px}
|
|
2949
|
+
.funnel .fstage{flex:1;min-width:118px;border:1px solid var(--border);background:var(--surf);padding:9px 11px}
|
|
2950
|
+
.funnel .fstage .fn{font-family:var(--mono);font-size:var(--fs-2xl);font-weight:700;color:var(--fg-strong);line-height:1.05}
|
|
2951
|
+
.funnel .fstage .fl{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--cyan);margin-top:3px}
|
|
2952
|
+
.funnel .fstage .fnote{font-size:var(--fs-2xs);color:var(--dim);margin-top:3px;line-height:1.4}
|
|
2953
|
+
.funnel .farrow{display:flex;align-items:center;color:var(--dim);font-size:var(--fs-lg);font-family:var(--mono)}
|
|
2954
|
+
.funnel .fstage.outc{flex:1.6;min-width:200px}
|
|
2955
|
+
.funnel .foutc{display:flex;gap:5px;flex-wrap:wrap;margin-top:6px}
|
|
2956
|
+
.funnel .fpill{font-family:var(--mono);font-size:var(--fs-2xs);padding:2px 8px;border:1px solid var(--border);color:var(--muted)}
|
|
2957
|
+
.funnel .fpill b{color:var(--fg-strong)}
|
|
2958
|
+
.funnel .fpill.pass{border-color:var(--pass);color:var(--pass)}.funnel .fpill.pass b{color:var(--pass)}
|
|
2959
|
+
.funnel .fpill.fail{border-color:var(--fail);color:var(--fail)}.funnel .fpill.fail b{color:var(--fail)}
|
|
2960
|
+
.funnel .fpill.indet{border-color:var(--warn);color:var(--warn)}.funnel .fpill.indet b{color:var(--warn)}
|
|
2961
|
+
.funnel .fpill.inc{border-color:var(--border-strong);color:var(--muted);border-style:dashed}
|
|
2962
|
+
/* §9.4.5 E3 — eval-HEALTH temporal heatmap (correctness over time, NOT latency) */
|
|
2963
|
+
.ehm{display:flex;gap:4px;flex-wrap:wrap;align-items:flex-start;margin:6px 0 4px}
|
|
2964
|
+
.ehm-col{display:flex;flex-direction:column;align-items:center;gap:3px}
|
|
2965
|
+
.ehm-cell{min-width:30px;height:30px;padding:0 6px;display:flex;align-items:center;justify-content:center;border:1px solid rgba(0,0,0,.35);font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;color:#0a0a12}
|
|
2966
|
+
.ehm-cell.pass{background:var(--hm-pass,rgba(67,195,154,.50))}.ehm-cell.fail{background:var(--hm-fail,rgba(224,102,102,.54))}.ehm-cell.indet{background:var(--hm-indet,rgba(232,166,77,.52))}.ehm-cell.skip{background:var(--surf-3);color:var(--dim)}
|
|
2967
|
+
.ehm-lab{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-align:center;line-height:1.25}
|
|
2968
|
+
.ehm-legend{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);margin:4px 0;display:flex;gap:4px;align-items:center;flex-wrap:wrap}
|
|
2969
|
+
.ehm-legend .sw{display:inline-block;width:11px;height:11px;border:1px solid rgba(0,0,0,.35);vertical-align:middle;margin:0 2px}
|
|
2970
|
+
.ehm-legend .sw.pass{background:var(--hm-pass,rgba(67,195,154,.50))}.ehm-legend .sw.fail{background:var(--hm-fail,rgba(224,102,102,.54))}.ehm-legend .sw.indet{background:var(--hm-indet,rgba(232,166,77,.52))}
|
|
2971
|
+
.ehm-pending{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--dim);border:1px dashed var(--border-strong);background:var(--surf);padding:7px 11px;margin:2px 0 6px;line-height:1.5}
|
|
2972
|
+
/* note callout */
|
|
2973
|
+
.note{border-left:3px solid var(--recommend);background:var(--recommend-bg);padding:7px 11px;font-size:var(--fs-xs);margin:14px 0}
|
|
2974
|
+
.note .tag{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;color:var(--recommend)}
|
|
2975
|
+
/* severity pills + verd */
|
|
2976
|
+
.sev{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:2px 7px;margin-right:4px}
|
|
2977
|
+
.sev.crit{color:var(--fail);background:var(--fail-bg)}.sev.high{color:var(--warn);background:var(--warn-bg)}.sev.med{color:var(--cyan);background:rgba(69,184,204,.10)}.sev.low{color:var(--muted);background:var(--surf-3)}
|
|
2978
|
+
.verd{font-family:var(--mono);font-size:var(--fs-xs);font-weight:700;padding:2px 8px;border:1px solid}
|
|
2979
|
+
.verd.fail{color:var(--fail);border-color:var(--fail)}.verd.pass{color:var(--pass);border-color:var(--pass)}.verd.inc{color:var(--warn);border-color:var(--warn)}
|
|
2980
|
+
.chip{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);border:1px solid var(--border);padding:1px 6px;white-space:nowrap}
|
|
2981
|
+
/* teaser */
|
|
2982
|
+
.teaser-row{display:flex;gap:8px;align-items:baseline;margin:4px 0;font-size:var(--fs-sm)}
|
|
2983
|
+
.teaser-num{color:var(--fail);font-family:var(--mono);font-size:var(--fs-xs)}
|
|
2984
|
+
/* §2 ledger */
|
|
2985
|
+
.lfilter{display:flex;gap:6px;align-items:center;flex-wrap:wrap;margin:9px 0}
|
|
2986
|
+
.lfilter b{font-family:var(--mono);font-size:var(--fs-2xs);padding:3px 8px;border:1px solid var(--border);color:var(--fg);opacity:.8;cursor:pointer}
|
|
2987
|
+
.lfilter b.on{border-color:var(--primary);color:var(--primary-soft);background:rgba(126,71,215,.12)}
|
|
2988
|
+
.lfilter input{background:var(--surf-2);border:1px solid var(--border);color:var(--fg);font-family:var(--mono);font-size:var(--fs-xs);padding:3px 8px;flex:1;max-width:240px}
|
|
2989
|
+
.lfilter .sp{margin-left:auto;font-family:var(--mono);font-size:var(--fs-2xs);color:var(--dim)}
|
|
2990
|
+
.virt{font-size:var(--fs-2xs);color:var(--dim);text-align:center;padding:8px}
|
|
2991
|
+
table.ledger td{cursor:pointer;font-family:var(--mono);font-size:var(--fs-xs)}
|
|
2992
|
+
table.ledger tr:hover td{background:var(--surf-2)}table.ledger tr.sel td{background:rgba(126,71,215,.10)}
|
|
2993
|
+
/* status-acuity — verdict-tinted ledger rows + a status left-accent (eye-catch at a glance) */
|
|
2994
|
+
table.ledger tr.vrow.FAIL td{background:var(--fail-bg)}table.ledger tr.vrow.FAIL td.tid{box-shadow:inset 3px 0 0 var(--fail)}
|
|
2995
|
+
table.ledger tr.vrow.INDETERMINATE td{background:var(--warn-bg)}table.ledger tr.vrow.INDETERMINATE td.tid{box-shadow:inset 3px 0 0 var(--warn)}
|
|
2996
|
+
table.ledger tr.vrow.PASS td.tid{box-shadow:inset 3px 0 0 var(--pass)}
|
|
2997
|
+
table.ledger tr.vrow.INCOMPLETE td.tid{box-shadow:inset 3px 0 0 var(--dim)}
|
|
2998
|
+
table.ledger tr.vrow:hover td{background:var(--surf-2)}table.ledger tr.vrow.sel td{background:rgba(126,71,215,.10)}
|
|
2999
|
+
.tid{color:var(--cyan)}.haswalk{color:var(--primary-soft);font-size:var(--fs-2xs)}
|
|
3000
|
+
.mini{font-family:var(--mono);font-size:var(--fs-2xs);padding:1px 5px}
|
|
3001
|
+
.mini.PASS{color:var(--pass);background:var(--pass-bg)}.mini.FAIL{color:var(--fail);background:var(--fail-bg)}.mini.INDETERMINATE{color:var(--warn);background:var(--warn-bg)}.mini.INCOMPLETE{color:var(--muted);background:var(--surf-3)}
|
|
3002
|
+
/* WS-2 — resolution badge (ledger column) + drill routing line. Makes a no-walk trace
|
|
3003
|
+
HONEST: judged, not unjudged. walk=cyan · walk-dropped=amber · truncated=dim. */
|
|
3004
|
+
.resbadge{font-family:var(--mono);font-size:var(--fs-2xs);padding:1px 6px;border:1px solid;white-space:nowrap;cursor:help}
|
|
3005
|
+
.resbadge.res-walk{color:var(--cyan);border-color:var(--cyan);background:rgba(69,184,204,.10)}
|
|
3006
|
+
.resbadge.res-nowalk{color:var(--warn);border-color:var(--warn);background:var(--warn-bg)}
|
|
3007
|
+
.resbadge.res-trunc{color:var(--muted);border-color:var(--border-strong);background:var(--surf-3)}
|
|
3008
|
+
.routing{display:flex;gap:9px;align-items:baseline;flex-wrap:wrap;margin:0 0 9px;padding:8px 11px;border-left:3px solid var(--border-strong);background:var(--surf-2)}
|
|
3009
|
+
.routing.res-walk{border-left-color:var(--cyan)}
|
|
3010
|
+
.routing.res-nowalk{border-left-color:var(--warn)}
|
|
3011
|
+
.routing.res-trunc{border-left-color:var(--dim)}
|
|
3012
|
+
.routing-k{flex:0 0 auto;font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--dim)}
|
|
3013
|
+
.routing-b{flex:0 0 auto;font-family:var(--mono);font-size:var(--fs-2xs);padding:1px 6px;border:1px solid}
|
|
3014
|
+
.routing-b.res-walk{color:var(--cyan);border-color:var(--cyan)}
|
|
3015
|
+
.routing-b.res-nowalk{color:var(--warn);border-color:var(--warn)}
|
|
3016
|
+
.routing-b.res-trunc{color:var(--muted);border-color:var(--border-strong)}
|
|
3017
|
+
.routing-v{flex:1 1 200px;font-size:var(--fs-sm);color:var(--fg);line-height:1.5}
|
|
3018
|
+
/* §2 side-by-side */
|
|
3019
|
+
.ctx{border:1px solid var(--border);background:var(--surf-2);margin:8px 0 4px}
|
|
3020
|
+
.ctx-h{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--cyan);text-transform:uppercase;letter-spacing:.06em;padding:6px 11px;border-bottom:1px solid var(--border)}
|
|
3021
|
+
.ctx-g{display:grid;grid-template-columns:repeat(3,1fr);gap:1px;background:var(--border)}.ctx-c{background:var(--surf);padding:8px 11px}
|
|
3022
|
+
.ctx-c .l{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase}.ctx-c .v{font-size:var(--fs-xs);margin-top:2px}
|
|
3023
|
+
/* Gap A — raw triggering INPUT in the §2 drill (input + scenario cell). font floor 11px. */
|
|
3024
|
+
.ctx-c .iraw{margin:0;font-family:var(--mono);font-size:var(--fs-2xs);line-height:1.5;color:var(--fg);white-space:pre-wrap;word-break:break-word;background:var(--bg);border:1px solid var(--border);padding:6px 8px}
|
|
3025
|
+
.ctx-c .iraw.clamp{max-height:120px;overflow:auto}
|
|
3026
|
+
.ctx-c .iexp summary{cursor:pointer;font-family:var(--mono);font-size:var(--fs-2xs);color:var(--cyan);text-transform:uppercase;letter-spacing:.04em;padding:2px 0}
|
|
3027
|
+
.ctx-c .iexp[open] summary{margin-bottom:4px}
|
|
3028
|
+
.ctx-c .iscn{margin-top:4px;font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted)}
|
|
3029
|
+
.ctx-c .ival{font-size:var(--fs-xs);color:var(--fg)}
|
|
3030
|
+
.lanehdr{display:grid;grid-template-columns:1fr 42px 1fr}.lanehdr div{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;padding:5px 8px;border-bottom:1px solid var(--border-strong)}
|
|
3031
|
+
.lanehdr .a{color:var(--fg-strong)}.lanehdr .x{text-align:center;color:var(--dim)}.lanehdr .j{color:var(--primary-soft)}
|
|
3032
|
+
.grid2{display:grid;grid-template-columns:1fr 42px 1fr;align-items:stretch}
|
|
3033
|
+
.band{grid-column:1/-1;border:1px solid var(--border);margin:8px 0;padding:7px 11px;background:var(--surf)}.band.loc{border-left:3px solid var(--fail);background:var(--fail-bg)}.band .bh{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;color:var(--fail)}
|
|
3034
|
+
.node{display:flex;flex-direction:column;align-items:center}.node .n{width:22px;height:22px;display:flex;align-items:center;justify-content:center;font-family:var(--mono);font-size:var(--fs-xs);font-weight:600;border:1px solid var(--border-strong);background:var(--surf-2);margin-top:12px}.node .ln{flex:1;width:1px;background:var(--border)}
|
|
3035
|
+
.node.error .n,.node.false-success .n{border-color:var(--fail);color:var(--fail)}.node.warn .n{border-color:var(--warn);color:var(--warn)}.node.ok .n{border-color:var(--pass);color:var(--pass)}
|
|
3036
|
+
.evb{border:1px solid var(--border);background:var(--surf-2);padding:6px 9px;margin:8px 8px 8px 0}.evb.r{margin:8px 0 8px 8px}
|
|
3037
|
+
.evb .top{display:flex;align-items:center;gap:6px}.evb .tool{font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg-strong)}.evb .st{font-family:var(--mono);font-size:var(--fs-2xs);padding:1px 6px;margin-left:auto}
|
|
3038
|
+
.evb .st.ok{color:var(--pass);background:var(--pass-bg)}.evb .st.error,.evb .st.false-success{color:var(--fail);background:var(--fail-bg)}.evb .st.warn{color:var(--warn);background:var(--warn-bg)}
|
|
3039
|
+
.evb .det{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);margin-top:3px;line-height:1.4}
|
|
3040
|
+
.jstep{display:flex;gap:6px;align-items:baseline;font-family:var(--mono);font-size:var(--fs-2xs);margin-top:4px}
|
|
3041
|
+
.jstep .k{font-size:var(--fs-2xs);font-weight:600;padding:1px 5px;min-width:54px;text-align:center}
|
|
3042
|
+
.k.context{color:var(--cyan);background:rgba(69,184,204,.10)}.k.examine,.k.detect{color:var(--warn);background:var(--warn-bg)}.k.bind{color:var(--cyan);background:rgba(69,184,204,.10)}.k.ground{color:var(--pass);background:var(--pass-bg)}.k.critique{color:var(--primary-soft);background:rgba(126,71,215,.12)}.k.decide{color:var(--fail);background:var(--fail-bg)}.k.verify{color:var(--fg-strong);background:var(--surf-3)}
|
|
3043
|
+
.jstep .t{color:var(--fg);opacity:.92;line-height:1.4}.jstep .t .ref{color:var(--cyan)}
|
|
3044
|
+
.jstep.noexam .t{color:var(--dim);font-style:italic}
|
|
3045
|
+
/* §2 judge lane — per-step eval-coverage entry (which criterion examined this step). */
|
|
3046
|
+
.jcov{border-left:2px solid var(--border-strong);background:var(--surf);padding:5px 8px;margin-top:6px}
|
|
3047
|
+
.jcov.pass{border-left-color:var(--pass)}.jcov.fail{border-left-color:var(--fail)}
|
|
3048
|
+
.jcov.uncertain,.jcov.indeterminate{border-left-color:var(--warn)}.jcov.na{border-left-color:var(--border)}
|
|
3049
|
+
.jcov-h{display:flex;gap:6px;align-items:center;flex-wrap:wrap;margin-bottom:3px}
|
|
3050
|
+
.jcov .jm{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:600;letter-spacing:.04em;padding:0 5px;border:1px solid var(--border-strong)}
|
|
3051
|
+
.jcov .jm.code{color:var(--cyan);border-color:var(--cyan)}.jcov .jm.judge{color:var(--primary-soft);border-color:var(--primary-soft)}
|
|
3052
|
+
.jcov .jcid{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--fg-strong);word-break:break-all}
|
|
3053
|
+
.jcov .jcrit{font-size:var(--fs-xs);color:var(--fg);opacity:.92;line-height:1.45}.jcov .jcrit .dim{color:var(--dim)}
|
|
3054
|
+
.jcov .cvrefs{margin-top:4px}
|
|
3055
|
+
.health{display:grid;grid-template-columns:repeat(4,1fr);gap:1px;background:var(--border);margin-top:8px}.health .hc{background:var(--surf);padding:8px 10px}.health .hc .l{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase}.health .hc .v{font-size:var(--fs-lg);font-weight:600;margin-top:2px}.health .hc .v.good{color:var(--pass)}.health .hc .v.warn{color:var(--warn)}
|
|
3056
|
+
.drillbox{border:1px solid var(--border-strong);background:var(--surf-2);margin-top:10px;padding:11px}
|
|
3057
|
+
/* §3 heatmap + subcards + calibration */
|
|
3058
|
+
.hm{display:grid;gap:3px;font-family:var(--mono);font-size:var(--fs-2xs);margin-top:6px}
|
|
3059
|
+
.hm .lab{color:var(--fg);opacity:.82;text-align:right;padding-right:5px;line-height:18px;white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
|
3060
|
+
.hm .colh{color:var(--muted);text-align:center;font-size:var(--fs-2xs);line-height:13px}
|
|
3061
|
+
.hm .cell{height:18px;border:1px solid rgba(0,0,0,.35);display:flex;align-items:center;justify-content:center;color:#0a0a12;font-size:var(--fs-2xs);font-weight:700}
|
|
3062
|
+
/* status-acuity — heatmap fills nudged up for contrast (still toned, not neon) */
|
|
3063
|
+
.hm .cell.pass{background:var(--hm-pass,rgba(67,195,154,.50))}.hm .cell.fail{background:var(--hm-fail,rgba(224,102,102,.54))}.hm .cell.indet{background:var(--hm-indet,rgba(232,166,77,.52))}.hm .cell.skip{background:var(--surf-3)}
|
|
3064
|
+
/* UI-6 — criteria cards: a PURPLISH base (brand primary), accent driven by SUCCESS RATE
|
|
3065
|
+
* (green-ish high · amber mid · red-ish low) — NOT a uniform severity red. */
|
|
3066
|
+
.subc{border:1px solid var(--border);border-left:4px solid var(--primary);background:linear-gradient(180deg,rgba(126,71,215,.12),rgba(126,71,215,.045));margin-top:10px}
|
|
3067
|
+
.subc.rate-ok{border-left-color:var(--pass)}
|
|
3068
|
+
.subc.rate-mid{border-left-color:var(--warn)}
|
|
3069
|
+
.subc.rate-low{border-left-color:var(--fail)}
|
|
3070
|
+
.subc-h{display:flex;gap:8px;align-items:center;padding:8px 11px;border-bottom:1px solid var(--border)}.subc-h b{font-size:var(--fs-sm);color:var(--fg-strong)}.subc-h .vp{font-family:var(--mono);font-size:var(--fs-xs);font-weight:700;padding:1px 7px;border:1px solid}
|
|
3071
|
+
.nest{border:1px solid var(--border);background:var(--surf);margin:7px 11px}.nest-h{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase;padding:4px 9px;border-bottom:1px solid var(--border)}
|
|
3072
|
+
.row{font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg);opacity:.92;margin:4px 9px;line-height:1.5}.row .ref{color:var(--cyan)}.row .w{color:var(--warn)}
|
|
3073
|
+
.cal{display:flex;gap:5px;align-items:center;padding:8px 11px;border-top:1px solid var(--border)}.cal .l{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);margin-right:3px}.cal b{font-family:var(--mono);font-size:var(--fs-2xs);padding:2px 8px;border:1px solid var(--border);color:var(--fg);opacity:.85;cursor:pointer}.cal b.on{border-color:var(--primary);color:var(--primary-soft);background:rgba(126,71,215,.12)}
|
|
3074
|
+
/* §4 findings */
|
|
3075
|
+
.find{border:1px solid var(--border);background:var(--surf-2);margin-top:11px}.find.gate{border-top:2px solid var(--fail);border-left:4px solid var(--fail);background:var(--fail-bg)}
|
|
3076
|
+
.find-h{display:flex;gap:9px;align-items:center;padding:9px 12px;border-bottom:1px solid var(--border);flex-wrap:wrap}.find-h b{font-size:var(--fs-sm);color:var(--fg-strong)}.find-h .gatetag{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--fail);border:1px solid var(--fail);padding:1px 6px}
|
|
3077
|
+
.fg{display:grid;grid-template-columns:92px 1fr;gap:1px;background:var(--border)}.fg .k{background:var(--surf);font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase;padding:8px 10px}
|
|
3078
|
+
.fg .v{background:var(--surf-2);padding:8px 11px;font-size:var(--fs-sm);color:var(--fg)}.fg .v code{font-family:var(--mono);font-size:var(--fs-xs);background:var(--bg);padding:0 4px}.fg .v .ref{color:var(--cyan);font-family:var(--mono);font-size:var(--fs-xs)}.fg .v .none{color:var(--pass)}.fg .v ul{margin:2px 0;padding-left:15px}.fg .v li{margin:2px 0;font-size:var(--fs-xs);font-family:var(--mono)}
|
|
3079
|
+
.fg .v.reason .step{color:var(--primary-soft)}.fg .v .lnk{color:var(--primary-soft);font-family:var(--mono);font-size:var(--fs-2xs);cursor:pointer}
|
|
3080
|
+
.na-note{font-size:var(--fs-2xs);color:var(--dim)}
|
|
3081
|
+
.prev{display:inline-block;background:var(--surf-3);height:8px;width:110px;vertical-align:middle}.prev i{display:block;height:100%;background:var(--fail)}
|
|
3082
|
+
.areview{display:flex;gap:6px;align-items:center;padding:9px 12px;border-top:1px solid var(--border-strong);background:var(--surf)}.areview .l{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--cyan)}.areview b{font-family:var(--mono);font-size:var(--fs-2xs);padding:2px 9px;border:1px solid var(--border);color:var(--fg);opacity:.85;cursor:pointer}.areview b.agree.on{border-color:var(--pass);color:var(--pass)}.areview b.refute.on{border-color:var(--fail);color:var(--fail)}.areview input{flex:1;background:var(--surf-2);border:1px solid var(--border);color:var(--fg);font-family:var(--mono);font-size:var(--fs-xs);padding:3px 7px}
|
|
3083
|
+
/* §4/§2 per-trajectory scorecard grid */
|
|
3084
|
+
.scgrid{display:grid;grid-template-columns:repeat(auto-fill,minmax(220px,1fr));gap:5px;margin-top:6px}
|
|
3085
|
+
.scc{border:1px solid var(--border);background:var(--surf);padding:5px 8px;font-family:var(--mono);font-size:var(--fs-2xs);display:flex;gap:6px;align-items:center}.scc.fail{border-left:2px solid var(--fail)}.scc.uncertain,.scc.indeterminate{border-left:2px solid var(--warn)}.scc .nm{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
3086
|
+
/* WS-3 §5 self-eval AGGREGATE — judge-calibration surface */
|
|
3087
|
+
.se-h{font-size:var(--fs-md);margin:22px 0 4px;display:flex;gap:8px;align-items:baseline;flex-wrap:wrap}
|
|
3088
|
+
.se-num{font-family:var(--mono);font-weight:700;color:var(--primary-soft)}
|
|
3089
|
+
.se-tag{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:1px 6px;border:1px solid var(--border-strong);color:var(--muted);text-transform:uppercase;letter-spacing:.03em}
|
|
3090
|
+
.se-tag.agg{color:var(--cyan);border-color:var(--cyan)}.se-tag.gt{color:var(--pass);border-color:var(--pass)}.se-tag.proxy{color:var(--warn);border-color:var(--warn)}.se-tag.hitl{color:var(--primary-soft);border-color:var(--primary)}.se-tag.future{color:var(--dim);border-color:var(--border)}
|
|
3091
|
+
.se-checks{border:1px solid var(--border);border-left:3px solid var(--primary);background:var(--surf-2);margin:10px 0 4px;padding:9px 12px}
|
|
3092
|
+
.se-checks-h{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--dim);margin-bottom:6px}
|
|
3093
|
+
.se-checks-g{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:6px}
|
|
3094
|
+
.se-check{display:flex;gap:7px;align-items:baseline}.se-cn{color:var(--primary-soft);font-family:var(--mono);font-weight:700}.se-ck{font-weight:600;color:var(--fg-strong);font-size:var(--fs-sm);white-space:nowrap}.se-cv{color:var(--muted);font-size:var(--fs-xs);line-height:1.4}
|
|
3095
|
+
.se-trust{display:flex;gap:16px;align-items:center;border:1px solid var(--border-strong);background:var(--surf);margin:12px 0;padding:14px 16px;border-left:3px solid var(--primary)}
|
|
3096
|
+
.se-trust.good{border-left-color:var(--pass)}.se-trust.warn{border-left-color:var(--warn)}.se-trust.bad{border-left-color:var(--fail)}
|
|
3097
|
+
.se-trust-score{font-family:var(--mono);font-size:var(--fs-2xl);font-weight:700;color:var(--fg-strong);line-height:1}.se-trust-d{font-size:var(--fs-md);color:var(--dim)}
|
|
3098
|
+
.se-trust-cap{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--dim);margin-top:3px}
|
|
3099
|
+
.se-trust.good .se-trust-score{color:var(--pass)}.se-trust.warn .se-trust-score{color:var(--warn)}.se-trust.bad .se-trust-score{color:var(--fail)}
|
|
3100
|
+
.se-trust-label{font-weight:600;color:var(--fg-strong);font-size:var(--fs-md)}.se-trust-basis{color:var(--muted);font-size:var(--fs-sm);margin-top:4px;line-height:1.5}
|
|
3101
|
+
.se-clusters{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:9px;margin:10px 0}
|
|
3102
|
+
.se-cl{border:1px solid var(--border);border-top:3px solid var(--border-strong);background:var(--surf);padding:9px 11px}
|
|
3103
|
+
.se-cl.good{border-top-color:var(--pass)}.se-cl.bad{border-top-color:var(--fail)}.se-cl.warn{border-top-color:var(--warn)}.se-cl.neutral{border-top-color:var(--dim)}
|
|
3104
|
+
.se-cl-top{display:flex;gap:8px;align-items:baseline}.se-cl-n{font-family:var(--mono);font-size:var(--fs-xl);font-weight:700;color:var(--fg-strong)}.se-cl-l{font-size:var(--fs-sm);font-weight:600;color:var(--fg)}
|
|
3105
|
+
.se-cl-note{font-size:var(--fs-xs);color:var(--muted);margin-top:4px;line-height:1.45}
|
|
3106
|
+
.se-conc{border:1px solid var(--border);background:var(--surf-2);padding:9px 12px;margin:6px 0}
|
|
3107
|
+
.se-conc-h{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--dim);margin-bottom:6px}
|
|
3108
|
+
.se-conc-r,.se-conf-r,.se-emit-r{display:flex;gap:9px;align-items:center;margin:3px 0;font-size:var(--fs-xs)}
|
|
3109
|
+
.se-conc-id{font-family:var(--mono);color:var(--cyan);flex:0 0 200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
3110
|
+
.se-conc-v,.se-conf-v,.se-emit-v{font-family:var(--mono);color:var(--muted);flex:0 0 auto;white-space:nowrap}
|
|
3111
|
+
.sebar{flex:1 1 auto;height:11px;background:var(--bg);border:1px solid var(--border);position:relative;overflow:hidden}
|
|
3112
|
+
.sebar-f{position:absolute;left:0;top:0;bottom:0}.sebar-f.p{background:linear-gradient(90deg,#3a2d63,var(--primary))}.sebar-f.good{background:var(--pass)}.sebar-f.warn{background:var(--warn)}.sebar-f.bad{background:var(--fail)}
|
|
3113
|
+
.se-row{display:flex;gap:10px;flex-wrap:wrap;margin:8px 0}
|
|
3114
|
+
.se-metric{border:1px solid var(--border);background:var(--surf);padding:9px 14px;min-width:140px;border-left:3px solid var(--border-strong)}
|
|
3115
|
+
.se-metric.good{border-left-color:var(--pass)}.se-metric.warn{border-left-color:var(--warn)}.se-metric.bad{border-left-color:var(--fail)}
|
|
3116
|
+
.se-metric-v{font-family:var(--mono);font-size:var(--fs-xl);font-weight:700;color:var(--fg-strong)}
|
|
3117
|
+
.se-metric.good .se-metric-v{color:var(--pass)}.se-metric.warn .se-metric-v{color:var(--warn)}.se-metric.bad .se-metric-v{color:var(--fail)}
|
|
3118
|
+
.se-metric-l{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase;margin-top:3px}
|
|
3119
|
+
.se-kv{display:flex;gap:16px;align-items:center;font-size:var(--fs-sm);color:var(--muted)}.se-kv b{color:var(--fg-strong);font-family:var(--mono)}
|
|
3120
|
+
.se-note{font-size:var(--fs-sm);color:var(--muted);margin:5px 0 2px;line-height:1.5}
|
|
3121
|
+
.se-empty{font-family:var(--mono);font-size:var(--fs-xs);color:var(--dim);border:1px dashed var(--border-strong);padding:12px;text-align:center;margin:6px 0}
|
|
3122
|
+
.se-conf-b{font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg);flex:0 0 64px}
|
|
3123
|
+
.se-table{width:100%;border-collapse:collapse;margin:8px 0;font-size:var(--fs-xs)}
|
|
3124
|
+
.se-table th{text-align:left;font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;color:var(--dim);border-bottom:1px solid var(--border);padding:5px 8px}
|
|
3125
|
+
.se-table td{padding:5px 8px;border-bottom:1px solid var(--border);font-family:var(--mono)}
|
|
3126
|
+
.se-table td.good{color:var(--pass)}.se-table td.bad{color:var(--fail)}.se-table td.warn{color:var(--warn)}.se-table td.dim{color:var(--dim)}
|
|
3127
|
+
.se-cid{color:var(--cyan);max-width:230px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
3128
|
+
.se-flag{font-size:var(--fs-2xs);padding:1px 6px;border:1px solid}.se-flag.good{color:var(--pass);border-color:var(--pass)}.se-flag.bad{color:var(--fail);border-color:var(--fail)}.se-flag.warn{color:var(--warn);border-color:var(--warn)}
|
|
3129
|
+
.se-vverdict{color:var(--muted)}
|
|
3130
|
+
.se-hitl,.se-spot-act{display:flex;gap:4px}
|
|
3131
|
+
.se-pick{font-family:var(--mono);font-size:var(--fs-2xs);padding:1px 7px;border:1px solid var(--border);color:var(--muted);cursor:pointer;user-select:none}
|
|
3132
|
+
.se-pick:hover{border-color:var(--primary);color:var(--fg)}
|
|
3133
|
+
.se-pick.on{border-color:var(--primary);color:var(--primary-soft);background:rgba(126,71,215,.14);font-weight:700}
|
|
3134
|
+
.se-blocked{border:1px solid var(--fail);background:var(--fail-bg);padding:9px 12px;margin:6px 0;font-size:var(--fs-sm);color:var(--fg);line-height:1.5}
|
|
3135
|
+
.se-blocked-tag{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;color:var(--fail);border:1px solid var(--fail);padding:1px 6px;margin-right:6px}
|
|
3136
|
+
.se-emit{border:1px solid var(--border);background:var(--surf-2);padding:10px 12px;margin:6px 0}
|
|
3137
|
+
.se-emit-l{font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg);flex:0 0 160px}
|
|
3138
|
+
.se-spots{display:flex;flex-direction:column;gap:5px;margin:6px 0}
|
|
3139
|
+
.se-spot{display:flex;gap:9px;align-items:center;flex-wrap:wrap;border:1px solid var(--border);border-left:3px solid var(--warn);background:var(--surf);padding:6px 10px;font-size:var(--fs-xs)}
|
|
3140
|
+
.se-spot-t{color:var(--cyan);flex:0 0 auto}.se-spot-c{color:var(--fg);flex:1 1 200px}.se-spot-conf{font-family:var(--mono);color:var(--muted)}.se-spot-why{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--warn)}.se-spot-act{margin-left:auto}
|
|
3141
|
+
.se-future{border:1px dashed var(--border-strong);background:var(--surf-2);padding:10px 13px;margin:6px 0;opacity:.92}
|
|
3142
|
+
.se-future-h{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--dim);margin-bottom:5px}
|
|
3143
|
+
.se-future-b{font-size:var(--fs-sm);color:var(--muted);line-height:1.55}
|
|
3144
|
+
/* §5 self-eval health */
|
|
3145
|
+
.hsc{display:grid;grid-template-columns:repeat(3,1fr);gap:1px;background:var(--border);margin-top:10px}.hsc .hc{background:var(--surf);padding:10px 12px}.hsc .hc .l{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase}.hsc .hc .v{font-size:var(--fs-xl);font-weight:700;margin-top:3px;font-family:var(--mono)}.hsc .hc .v.good{color:var(--pass)}.hsc .hc .v.warn{color:var(--warn)}
|
|
3146
|
+
/* UI-12-B — honest grounded health: rate-coloured (low=critical), capture-na=muted (never green) */
|
|
3147
|
+
.hsc .hc .v.rate-ok{color:var(--pass)}.hsc .hc .v.rate-mid{color:var(--warn)}.hsc .hc .v.rate-low{color:var(--fail)}
|
|
3148
|
+
.hsc .hc .v.capture-na{color:var(--dim);font-size:var(--fs-sm);font-weight:600;cursor:help}
|
|
3149
|
+
.copy-md{font-family:var(--mono);font-size:var(--fs-xs);padding:4px 10px;border:1px solid var(--border);background:var(--surf-2);color:var(--fg);cursor:pointer;margin-top:8px}
|
|
3150
|
+
.copy-md:hover{border-color:var(--cyan);color:var(--cyan)}
|
|
3151
|
+
.copy-md.copied{border-color:var(--pass);color:var(--pass)}
|
|
3152
|
+
.copy-md.copyfail{border-color:var(--warn);color:var(--warn)}
|
|
3153
|
+
/* WS-4 — refined EV-051 "routed to diagnostics" handover block */
|
|
3154
|
+
.hov-meta{display:flex;flex-wrap:wrap;gap:7px;margin:8px 0 10px}
|
|
3155
|
+
.hov-meta .hm-c{display:flex;flex-direction:column;gap:1px;border:1px solid var(--border);border-left:3px solid var(--primary);background:var(--surf);padding:5px 10px;min-width:90px}
|
|
3156
|
+
.hov-meta .hm-c .l{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.04em;color:var(--cyan)}
|
|
3157
|
+
.hov-meta .hm-c .v{font-size:var(--fs-sm);color:var(--fg-strong)}
|
|
3158
|
+
.hov-list{display:flex;flex-direction:column;gap:8px;margin:8px 0}
|
|
3159
|
+
.hov{border:1px solid var(--border);border-left:3px solid var(--fail);background:var(--surf)}
|
|
3160
|
+
.hov-h{display:flex;gap:8px;align-items:center;flex-wrap:wrap;padding:7px 11px;border-bottom:1px solid var(--border);background:var(--surf-2)}
|
|
3161
|
+
.hov-h .hov-n{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted)}
|
|
3162
|
+
.hov-h b{font-family:var(--mono);font-size:var(--fs-sm);color:var(--fg-strong)}
|
|
3163
|
+
.hov-copy{margin:0 0 0 auto;padding:2px 9px;font-size:var(--fs-2xs)}
|
|
3164
|
+
.hov-g{display:grid;grid-template-columns:64px 1fr;gap:5px 11px;padding:9px 11px}
|
|
3165
|
+
.hov-g .hk{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:600;text-transform:uppercase;letter-spacing:.04em;color:var(--cyan);padding-top:2px}
|
|
3166
|
+
.hov-g .hv{font-size:var(--fs-sm);color:var(--fg);line-height:1.5;min-width:0;word-break:break-word}
|
|
3167
|
+
.hov-dim{color:var(--muted);font-family:var(--mono);font-size:var(--fs-2xs)}
|
|
3168
|
+
/* §9.4.4 R3 per-tab description */
|
|
3169
|
+
.tabdesc{border:1px solid var(--border);border-left:3px solid var(--cyan);background:var(--surf-2);padding:7px 11px;font-size:var(--fs-xs);color:var(--fg);opacity:.92;line-height:1.5;margin:8px 0 12px}
|
|
3170
|
+
.tabdesc .td-i{color:var(--cyan);font-weight:700;margin-right:5px}
|
|
3171
|
+
/* §9.4.4 R4 criterion provenance + hover + statement */
|
|
3172
|
+
.prov{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:1px 6px;border:1px solid;text-transform:uppercase;cursor:help}
|
|
3173
|
+
.prov.defined{color:var(--cyan);border-color:var(--cyan);background:rgba(69,184,204,.10)}
|
|
3174
|
+
.prov.source{color:var(--primary-soft);border-color:var(--primary);background:rgba(126,71,215,.12)}
|
|
3175
|
+
/* UI-10 — CODE vs JUDGE method chip (how a criterion is evaluated) */
|
|
3176
|
+
.method{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:1px 6px;border:1px solid;text-transform:uppercase;letter-spacing:.03em;cursor:help;white-space:nowrap}
|
|
3177
|
+
.method.code{color:var(--warn);border-color:var(--warn);background:rgba(214,158,46,.12)}
|
|
3178
|
+
.method.hybrid{color:var(--cyan);border-color:var(--cyan);background:rgba(69,184,204,.10)}
|
|
3179
|
+
.method.judge{color:var(--muted);border-color:var(--border-strong);background:var(--surf-2)}
|
|
3180
|
+
.ihover{color:var(--cyan);font-size:var(--fs-xs);cursor:help;border-bottom:1px dotted var(--cyan)}
|
|
3181
|
+
.cstmt{font-size:var(--fs-xs);color:var(--muted);margin-top:3px;line-height:1.4;white-space:normal}
|
|
3182
|
+
td .cstmt{font-family:var(--sans,inherit)}
|
|
3183
|
+
/* §9.4.4 R3/M5 detected-but-unmatched */
|
|
3184
|
+
.fails-h,.detected-h{font-size:var(--fs-sm);margin-top:14px}
|
|
3185
|
+
.detected-h{color:var(--warn)}
|
|
3186
|
+
.detected{border:1px solid var(--warn);border-left:3px solid var(--warn);background:var(--warn-bg);margin-top:8px;padding:7px 11px}
|
|
3187
|
+
.detected-top{display:flex;gap:8px;align-items:baseline;flex-wrap:wrap}.detected-top b{font-size:var(--fs-sm);color:var(--fg-strong)}
|
|
3188
|
+
.dkind{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;color:var(--warn);border:1px solid var(--warn);padding:1px 6px;text-transform:uppercase}
|
|
3189
|
+
.dtrace{font-size:var(--fs-2xs);color:var(--muted);margin-left:auto}
|
|
3190
|
+
.detected-ev{font-size:var(--fs-2xs);color:var(--muted);margin-top:4px}.detected-ev .ref{color:var(--cyan)}
|
|
3191
|
+
/* §9.4.4 R2/M1 internal subject-profile card */
|
|
3192
|
+
.profile-grid{display:grid;grid-template-columns:max-content 1fr;gap:1px;background:var(--border);border:1px solid var(--border);margin:8px 0}
|
|
3193
|
+
.profile-grid .pk{background:var(--surf);font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase;padding:6px 10px}
|
|
3194
|
+
.profile-grid .pv{background:var(--surf-2);font-size:var(--fs-xs);color:var(--fg);padding:6px 11px}
|
|
3195
|
+
/* §9.4.4 R2/M4 phase-block lens */
|
|
3196
|
+
.phase-lens{display:grid;grid-template-columns:repeat(auto-fill,minmax(220px,1fr));gap:6px;margin:8px 0}
|
|
3197
|
+
.phase-blk{border:1px solid var(--border);background:var(--surf);padding:7px 9px}
|
|
3198
|
+
.phase-h{display:flex;gap:6px;align-items:center;margin-bottom:4px}.phase-n{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);margin-left:auto}
|
|
3199
|
+
.ps-row{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--fg);opacity:.88;line-height:1.4;margin-top:3px;border-top:1px solid var(--border);padding-top:3px}
|
|
3200
|
+
.phase-h .k.gather{color:var(--cyan);background:rgba(69,184,204,.10)}.phase-h .k.expect{color:var(--primary-soft);background:rgba(126,71,215,.12)}
|
|
3201
|
+
/* §9.4.4 R2/M2+M3 reasoning example */
|
|
3202
|
+
.re-blk{border:1px solid var(--border);background:var(--surf-2);padding:8px 11px;margin-top:8px}
|
|
3203
|
+
.re-h{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;color:var(--cyan);margin-bottom:5px}
|
|
3204
|
+
.re-rephrase{font-size:var(--fs-sm);color:var(--fg-strong);font-style:italic;line-height:1.5}
|
|
3205
|
+
.re-sub{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;color:var(--muted);margin-top:6px}
|
|
3206
|
+
.re-ul{margin:3px 0;padding-left:18px}.re-ul li{font-size:var(--fs-xs);margin:2px 0;line-height:1.4}.re-rat{color:var(--muted)}
|
|
3207
|
+
/* UI-2 — verdict-state legend (Overview + Scorecard) */
|
|
3208
|
+
.vlegend{border:1px solid var(--border);border-left:3px solid var(--cyan);background:var(--surf-2);padding:8px 11px;margin:10px 0}
|
|
3209
|
+
.vlegend .vl-h{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.05em;color:var(--cyan);margin-bottom:6px}
|
|
3210
|
+
.vlegend .vl-item{display:flex;gap:9px;align-items:baseline;margin:4px 0;line-height:1.45}
|
|
3211
|
+
.vlegend .vl-k{font-family:var(--mono);font-size:var(--fs-2xs);font-weight:700;padding:1px 7px;border:1px solid;min-width:108px;text-align:center;flex:0 0 auto}
|
|
3212
|
+
.vlegend .vl-v{font-size:var(--fs-xs);color:var(--fg);opacity:.92}
|
|
3213
|
+
.vlegend .vl-k.pass{color:var(--pass);border-color:var(--pass);background:var(--pass-bg)}
|
|
3214
|
+
.vlegend .vl-k.fail{color:var(--fail);border-color:var(--fail);background:var(--fail-bg)}
|
|
3215
|
+
.vlegend .vl-k.inc{color:var(--muted);border-color:var(--border-strong);background:var(--surf-3)}
|
|
3216
|
+
.vlegend .vl-k.indet{color:var(--warn);border-color:var(--warn);background:var(--warn-bg)}
|
|
3217
|
+
.vlegend .vl-k.na{color:var(--dim);border-color:var(--border);background:var(--surf)}
|
|
3218
|
+
/* UI-3 — heatmap horizontal-scroll container (no page overflow) */
|
|
3219
|
+
.hm-scroll{overflow-x:auto;max-width:100%;padding-bottom:2px}
|
|
3220
|
+
.hm .colh{word-break:break-word;white-space:normal}
|
|
3221
|
+
/* UI-5 — success-rate + key-metrics summary row under the heatmap */
|
|
3222
|
+
.sc-metrics{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:6px;margin:10px 0 6px}
|
|
3223
|
+
.scm{border:1px solid var(--border);border-top:3px solid var(--border-strong);background:var(--surf);padding:9px 11px}
|
|
3224
|
+
.scm-v{font-family:var(--mono);font-size:var(--fs-2xl);font-weight:700;line-height:1.05;color:var(--fg-strong)}
|
|
3225
|
+
.scm-l{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.04em;color:var(--cyan);margin-top:3px}
|
|
3226
|
+
.scm-n{font-size:var(--fs-2xs);color:var(--dim);margin-top:3px;line-height:1.4}
|
|
3227
|
+
.scm.rate-ok{border-top-color:var(--pass)}.scm.rate-ok .scm-v{color:var(--pass)}
|
|
3228
|
+
.scm.rate-mid{border-top-color:var(--warn)}.scm.rate-mid .scm-v{color:var(--warn)}
|
|
3229
|
+
.scm.rate-low{border-top-color:var(--fail)}.scm.rate-low .scm-v{color:var(--fail)}
|
|
3230
|
+
.scm.rate-na{border-top-color:var(--border)}
|
|
3231
|
+
/* UI-12-B — capture-unavailable grounded tile: long honest label, not a big green number */
|
|
3232
|
+
.scm.pending .scm-v{font-size:var(--fs-sm);color:var(--dim);word-break:break-word}
|
|
3233
|
+
/* UI-7 — criterion-definition meta line */
|
|
3234
|
+
.nest .row.defn-meta{color:var(--cyan);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.03em}
|
|
3235
|
+
/* UI-8 — calibrate groups (mutually-exclusive radio semantics) */
|
|
3236
|
+
.cal .calgroup{display:inline-flex;gap:5px;align-items:center}
|
|
3237
|
+
/* UI-4 — judge-reasoning bands in the §2 drill (why this verdict + full why-chain) */
|
|
3238
|
+
.band.whyverdict{border-left:3px solid var(--primary);background:rgba(126,71,215,.08)}
|
|
3239
|
+
.band.whyverdict .wv-t{font-size:var(--fs-sm);color:var(--fg);line-height:1.5;margin-top:4px}
|
|
3240
|
+
.ctx.whychain{margin:8px 0}
|
|
3241
|
+
.ctx.whychain .wc-body{padding:7px 11px;display:flex;flex-direction:column;gap:2px}
|
|
3242
|
+
.why-note{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);margin-top:4px;line-height:1.45}.why-note .ref{color:var(--cyan)}
|
|
3243
|
+
/* WS-5 — indeterminate resolution chain (§3): per-criterion applicable·pass·fail·N/A·needs-evidence */
|
|
3244
|
+
.rztable{display:flex;flex-direction:column;gap:8px;margin:8px 0}
|
|
3245
|
+
.rzrow{border:1px solid var(--border);border-left:3px solid var(--border-strong);background:var(--surf);padding:8px 11px}
|
|
3246
|
+
.rzrow.crit,.rzrow.high{border-left-color:var(--fail)}.rzrow.med{border-left-color:var(--warn)}.rzrow.low{border-left-color:var(--border-strong)}
|
|
3247
|
+
.rz-h{display:flex;gap:8px;align-items:baseline;flex-wrap:wrap;margin-bottom:5px}
|
|
3248
|
+
.rz-h b{font-size:var(--fs-sm);color:var(--fg-strong)}
|
|
3249
|
+
.rz-rate{margin-left:auto;font-family:var(--mono);font-size:var(--fs-2xs)}
|
|
3250
|
+
.rz-stats{display:flex;flex-wrap:wrap;gap:6px;align-items:center}
|
|
3251
|
+
.rz{font-family:var(--mono);font-size:var(--fs-2xs);padding:2px 7px;border:1px solid var(--border)}
|
|
3252
|
+
.rz.applicable{color:var(--fg-strong);border-color:var(--border-strong)}
|
|
3253
|
+
.rz.pass{color:var(--pass);border-color:var(--pass)}
|
|
3254
|
+
.rz.fail{color:var(--fail);border-color:var(--fail)}
|
|
3255
|
+
.rz.na{color:var(--cyan);border-color:var(--cyan);background:rgba(108,196,212,.08)}
|
|
3256
|
+
.rz.ne{color:var(--warn);border-color:var(--warn);background:rgba(214,158,46,.08)}
|
|
3257
|
+
.rz.zero{color:var(--dim);border-color:var(--border);background:none}
|
|
3258
|
+
.rz-reason{display:inline-block;font-size:var(--fs-2xs);color:var(--muted);font-style:italic;margin-left:4px}
|
|
3259
|
+
.rz-act{display:inline-flex;flex-direction:column;font-family:var(--mono);font-size:var(--fs-2xs);color:var(--warn);border:1px dashed var(--warn);padding:2px 7px}
|
|
3260
|
+
.rz-act .rz-reason{color:var(--muted);font-family:var(--sans,inherit)}
|
|
3261
|
+
/* WS-3 — one-line plain-language gloss (what it measures · why it matters), muted,
|
|
3262
|
+
sits ABOVE the technical block in BOTH §3 scorecard subcards and §4 finding cards */
|
|
3263
|
+
.plain{font-size:var(--fs-xs);color:var(--muted);line-height:1.5;margin:6px 0 8px;padding:6px 9px;border-left:2px solid var(--cyan);background:rgba(108,196,212,.06);font-style:italic}
|
|
3264
|
+
/* WS-3 follow-up — per-criterion plain-language KEY directly under the §3 heatmap (ALL criteria) */
|
|
3265
|
+
.plain-legend{margin:8px 0 4px}
|
|
3266
|
+
.plain-legend .pl-h{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.04em;color:var(--cyan);margin:0 0 5px}
|
|
3267
|
+
.plain-leg{display:flex;gap:8px;align-items:baseline;margin:0 0 4px;flex-wrap:wrap}
|
|
3268
|
+
.plain-leg .pl-id{font-style:normal;color:var(--fg-strong);font-size:var(--fs-2xs);flex:0 0 auto}
|
|
3269
|
+
.plain-leg .pl-tx{font-style:italic;color:var(--muted);flex:1 1 240px;min-width:0}
|
|
3270
|
+
/* WS-1 — "how the judge reasoned" per-criterion critique block (side-by-side + scorecard drill) */
|
|
3271
|
+
.cvblock{margin:8px 0}
|
|
3272
|
+
.cvblock .cvbody{display:flex;flex-direction:column;gap:6px;padding:7px 9px}
|
|
3273
|
+
.cvrow{border:1px solid var(--border);border-left:3px solid var(--border-strong);background:var(--surf);padding:7px 9px}
|
|
3274
|
+
.cvrow.pass{border-left-color:var(--pass)}.cvrow.fail{border-left-color:var(--fail)}
|
|
3275
|
+
.cvrow.uncertain,.cvrow.indeterminate{border-left-color:var(--warn)}.cvrow.na{border-left-color:var(--border)}
|
|
3276
|
+
.cvh{display:flex;gap:7px;align-items:center;flex-wrap:wrap;margin-bottom:3px}
|
|
3277
|
+
.cvb{font-family:var(--mono);font-size:var(--fs-2xs);text-transform:uppercase;letter-spacing:.03em;padding:1px 6px;border:1px solid var(--border-strong)}
|
|
3278
|
+
.cvb.pass{color:var(--pass);border-color:var(--pass)}.cvb.fail{color:var(--fail);border-color:var(--fail)}
|
|
3279
|
+
.cvb.uncertain,.cvb.indeterminate{color:var(--warn);border-color:var(--warn)}.cvb.na{color:var(--dim)}
|
|
3280
|
+
.cvid{font-family:var(--mono);font-size:var(--fs-xs);color:var(--fg-strong)}
|
|
3281
|
+
.cvconf{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--dim);margin-left:auto}
|
|
3282
|
+
.cvcrit{font-size:var(--fs-sm);color:var(--fg);line-height:1.5}.cvcrit.dim{color:var(--dim)}
|
|
3283
|
+
.cvrefs{display:flex;flex-wrap:wrap;gap:5px;margin-top:5px}
|
|
3284
|
+
.cvref{font-family:var(--mono);font-size:var(--fs-2xs);color:var(--cyan);border:1px solid var(--border);padding:1px 5px;word-break:break-all}
|
|
3285
|
+
`;
|
|
3286
|
+
|
|
3287
|
+
// ── the embedded client JS (ledger filter/search + drill + tabs + toggles) ───
|
|
3288
|
+
|
|
3289
|
+
/**
|
|
3290
|
+
* The embedded client JS — ledger filter/search + click-row drill (side-by-side
|
|
3291
|
+
* when judge_steps exist, else the per-trajectory scorecard) + tab switching +
|
|
3292
|
+
* calibration/alignment toggles + copy-as-markdown. The data (ledger `L`,
|
|
3293
|
+
* criteria `C`, judge-walks `J`) is injected as deterministic JSON.
|
|
3294
|
+
*/
|
|
3295
|
+
function clientScript(ledgerJson: string, critJson: string, walksJson: string, resMetaJson: string): string {
|
|
3296
|
+
return `
|
|
3297
|
+
const L=${ledgerJson},C=${critJson},J=${walksJson},RES=${resMetaJson};let F='all';
|
|
3298
|
+
function resBadge(res){var m=RES[res];if(!m)return '';return '<span class="resbadge '+m.cls+'" title="'+esc(m.routing)+'">'+esc(m.badge)+'</span>';}
|
|
3299
|
+
function resRouting(res){var m=RES[res];if(!m)return '';return '<div class="routing '+m.cls+'"><span class="routing-k">resolution</span><span class="routing-b '+m.cls+'">'+esc(m.badge)+'</span><span class="routing-v">'+esc(m.routing)+'</span></div>';}
|
|
3300
|
+
function esc(s){return String(s==null?'':s).replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');}
|
|
3301
|
+
function wireTabs(){var btns=document.querySelectorAll('nav.tabs button'),panels=document.querySelectorAll('main .panel');
|
|
3302
|
+
btns.forEach(function(b){b.addEventListener('click',function(){var key=b.getAttribute('data-tab');
|
|
3303
|
+
btns.forEach(function(x){x.classList.remove('active');});panels.forEach(function(p){p.classList.remove('active');});
|
|
3304
|
+
b.classList.add('active');var p=document.getElementById(key);if(p)p.classList.add('active');});});}
|
|
3305
|
+
function cnt(c){var p=0,f=0,n=0;for(var k in c){if(c[k]=='pass')p++;else if(c[k]=='fail')f++;else if(c[k]=='uncertain'||c[k]=='indeterminate')n++;}return[p,f,n];}
|
|
3306
|
+
function render(){var qi=document.getElementById('q');if(!qi)return;var q=(qi.value||'').toLowerCase();
|
|
3307
|
+
var rows=L.filter(function(r){return (F=='all'||r.v==F)&&(!q||r.t.toLowerCase().indexOf(q)>=0||(r.r||'').toLowerCase().indexOf(q)>=0);});
|
|
3308
|
+
var cntEl=document.getElementById('cnt');if(cntEl)cntEl.textContent=rows.length+' / '+L.length+' trajectories';
|
|
3309
|
+
var cap=rows.slice(0,150);
|
|
3310
|
+
document.getElementById('lrows').innerHTML=cap.map(function(r){var c=cnt(r.c),p=c[0],f=c[1],n=c[2];
|
|
3311
|
+
return '<tr data-id="'+esc(r.t)+'" class="vrow '+esc(r.v)+'"><td class="tid">'+esc(r.t.slice(0,14))+(r.js?' <span class=haswalk>✦</span>':'')+'</td><td>'+esc(r.r||'all')+'</td><td><span class="mini '+esc(r.v)+'">'+esc(r.v)+'</span></td><td>'+resBadge(r.res)+'</td><td>'+p+'</td><td>'+f+'</td><td>'+n+'</td><td style="color:var(--fail)">'+esc((r.f||[]).join(', '))+'</td></tr>';}).join('');
|
|
3312
|
+
var virt=document.getElementById('virt');if(virt)virt.textContent=rows.length>150?('▾ showing 150 of '+rows.length+' — filter/search to narrow'):'';
|
|
3313
|
+
document.querySelectorAll('table.ledger tbody tr').forEach(function(tr){tr.addEventListener('click',function(){drill(tr.getAttribute('data-id'),tr);});});
|
|
3314
|
+
/* Surface the Agent‖Judge side-by-side BY DEFAULT: on the first render, auto-open the
|
|
3315
|
+
first JUDGED trace (one that has a judge walk) so the two-lane trajectory shows without
|
|
3316
|
+
a click. Once only — never hijacks the drill after the user starts navigating. */
|
|
3317
|
+
if(!window._autoDrilled){var jr=null;for(var i=0;i<cap.length;i++){if(cap[i].js){jr=cap[i];break;}}jr=jr||cap[0];if(jr){var atr=document.querySelector('table.ledger tbody tr[data-id="'+jr.t+'"]');if(atr){window._autoDrilled=true;drill(jr.t,atr);}}}}
|
|
3318
|
+
// WS-1 — the "How the Judge Reasoned" per-criterion block: result badge +
|
|
3319
|
+
// critique-before-verdict + grounding refs, read off the verdict file's verdicts[].
|
|
3320
|
+
// Shared by BOTH the side-by-side drill (walk traces) and the per-trajectory
|
|
3321
|
+
// scorecard drill (no-walk traces) so NO judged trace renders an empty panel.
|
|
3322
|
+
function cvRefs(refs){if(!refs||!refs.length)return '';
|
|
3323
|
+
return '<div class="cvrefs">'+refs.map(function(rf){var s=(typeof rf==='string')?rf:[rf.obs,rf.path,rf.value].filter(Boolean).join(' · ');return s?'<span class="cvref">'+esc(s)+'</span>':'';}).join('')+'</div>';}
|
|
3324
|
+
function critiqueBlock(cvs){if(!cvs||!cvs.length)return '';
|
|
3325
|
+
var order={fail:0,uncertain:1,indeterminate:1,pass:2,na:3};
|
|
3326
|
+
var rows=cvs.slice().sort(function(a,b){return (order[a.result]==null?9:order[a.result])-(order[b.result]==null?9:order[b.result]);}).map(function(v){
|
|
3327
|
+
var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
|
|
3328
|
+
var conf=(v.confidence!=null)?'<span class="cvconf">conf '+esc(v.confidence)+(v.confidenceBand?' · '+esc(v.confidenceBand):'')+'</span>':'';
|
|
3329
|
+
var crit=v.critique?'<div class="cvcrit">'+esc(v.critique)+'</div>':'<div class="cvcrit dim">— no critique recorded for this criterion</div>';
|
|
3330
|
+
return '<div class="cvrow '+esc(res)+'"><div class="cvh"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="cvid">'+esc(v.criterionId||'')+'</span>'+conf+'</div>'+crit+cvRefs(v.refs)+'</div>';}).join('');
|
|
3331
|
+
return '<div class="ctx cvblock"><div class="ctx-h">◇ how the judge reasoned — per-criterion verdict · critique-before-verdict · grounding refs ('+cvs.length+')</div><div class="cvbody">'+rows+'</div></div>';}
|
|
3332
|
+
function sideBySide(d){var ctx=d.context||{};
|
|
3333
|
+
// Gap A — the §2 "input + scenario" cell renders the RAW triggering INPUT (the
|
|
3334
|
+
// thing that fired the agent) ABOVE the judge's scenario LABEL. Long inputs
|
|
3335
|
+
// collapse into a <details> (lean, no JS) so the cell stays compact; short ones
|
|
3336
|
+
// render inline (clamped). ABSENT input ⇒ the cell shows the scenario alone (or
|
|
3337
|
+
// "—"). Font stays at the --fs-2xs (11px) floor + brand mono.
|
|
3338
|
+
var inputCell=function(raw,scen){
|
|
3339
|
+
var sc=scen?'<div class="iscn">scenario · '+esc(scen)+'</div>':'';
|
|
3340
|
+
if(!raw)return (scen?'<div class="ival">'+esc(scen)+'</div>':'—');
|
|
3341
|
+
var long=String(raw).length>180;
|
|
3342
|
+
var body=long
|
|
3343
|
+
? '<details class="iexp"><summary>raw input · '+String(raw).length+' chars</summary><pre class="iraw">'+esc(raw)+'</pre></details>'
|
|
3344
|
+
: '<pre class="iraw clamp">'+esc(raw)+'</pre>';
|
|
3345
|
+
return body+sc;};
|
|
3346
|
+
var refStr=function(rf){if(!rf)return '';if(typeof rf==='string')return rf;return [rf.obs,rf.path,rf.value].filter(Boolean).join(':');};
|
|
3347
|
+
// UI-4 — surface the judge's REASONING (why the verdict + why the exit-states were
|
|
3348
|
+
// concluded) from the EXISTING judge walk: an ordered why-chain band + the decide/bind
|
|
3349
|
+
// text inlined where it explains a conclusion. No emit-contract change — read-only over
|
|
3350
|
+
// the judge_steps already on the verdict file.
|
|
3351
|
+
var KORD={gather:0,context:1,examine:2,detect:3,bind:4,ground:5,critique:6,decide:7,verify:8};
|
|
3352
|
+
var allJs=(d.judgeSteps||[]).slice();
|
|
3353
|
+
var stepText=function(kind){return allJs.filter(function(s){return s.kind===kind;}).map(function(s){return s.text||'';}).filter(Boolean).join(' · ');};
|
|
3354
|
+
var decideWhy=stepText('decide')||stepText('critique');
|
|
3355
|
+
var stateWhy=[stepText('bind'),stepText('gather')].filter(Boolean).join(' · ');
|
|
3356
|
+
var chainSteps=allJs.slice().sort(function(a,b){var x=KORD[a.kind];var y=KORD[b.kind];return (x==null?9:x)-(y==null?9:y);});
|
|
3357
|
+
var whyChain=chainSteps.length?('<div class="ctx whychain"><div class="ctx-h">◇ judge reasoning — full why-chain (gather → bind → ground → decide)</div><div class="wc-body">'+chainSteps.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+(rs?' <span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('')+'</div></div>'):'';
|
|
3358
|
+
var verdictWhy=decideWhy?('<div class="band whyverdict"><div class="bh" style="color:var(--primary-soft)">◇ why this verdict</div><div class="wv-t">'+esc(decideWhy)+'</div></div>'):'';
|
|
3359
|
+
// -- §2 judge lane = per-step EVAL COVERAGE --
|
|
3360
|
+
// The judge lane (.step-r) used to filter judgeSteps by anchor === a.n -- but
|
|
3361
|
+
// judges never emit an anchor, so EVERY step rendered a bare dash. Instead we
|
|
3362
|
+
// map each agent step -> the per-criterion verdicts (d.criterionVerdicts) whose
|
|
3363
|
+
// grounding refs EXAMINED that step: a precise ref.obs === step.obs match plus
|
|
3364
|
+
// the tool-name fallback (ref.path === 'name' && ref.value === step.tool).
|
|
3365
|
+
// Each examining criterion renders one compact entry -- result + CODE/JUDGE tag
|
|
3366
|
+
// + criterionId + the judge reasoning (critique). A step no criterion references
|
|
3367
|
+
// says 'not examined by any eval' (honest), never a bare dash. Any judge step
|
|
3368
|
+
// that DOES carry a real anchor is still honored (future-proof).
|
|
3369
|
+
var cvAll=(d.criterionVerdicts||[]);
|
|
3370
|
+
var refExaminesStep=function(rf,a){
|
|
3371
|
+
if(!rf||typeof rf==='string')return false;
|
|
3372
|
+
if(a.obs&&rf.obs&&String(rf.obs)===String(a.obs))return true;
|
|
3373
|
+
if(rf.path==='name'&&rf.value!=null&&a.tool&&String(rf.value)===String(a.tool))return true;
|
|
3374
|
+
return false;};
|
|
3375
|
+
var covEntry=function(v,a){
|
|
3376
|
+
var res=v.result||'na';var disp=res==='uncertain'?'indeterminate':res;
|
|
3377
|
+
// the router carries TWO vocabularies: matrix-derived ('deterministic') and
|
|
3378
|
+
// mined ('code-based'); 'hybrid' is shared. A '[code-eval ...]'-prefixed critique
|
|
3379
|
+
// is the deterministic-logic fallback when the method is unset.
|
|
3380
|
+
var method=((typeof C!=='undefined'&&C[v.criterionId])||{}).m||'';
|
|
3381
|
+
var isCode=method==='deterministic'||method==='code-based'||method==='hybrid'||String(v.critique||'').trim().indexOf('[code-eval')===0;
|
|
3382
|
+
var tag=method==='hybrid'?'HYBRID':(isCode?'CODE':'JUDGE');
|
|
3383
|
+
var matched=(v.refs||[]).filter(function(rf){return typeof rf!=='string'&&refExaminesStep(rf,a);});
|
|
3384
|
+
var crit=v.critique?esc(v.critique):'<span class="dim">— no critique recorded</span>';
|
|
3385
|
+
return '<div class="jcov '+esc(res)+'"><div class="jcov-h"><span class="cvb '+esc(res)+'">'+esc(disp)+'</span><span class="jm '+(isCode?'code':'judge')+'" title="'+(isCode?'deterministically checked by a code-eval (not LLM-judged)':'reasoned by the LLM judge')+'">'+esc(tag)+'</span><span class="jcid">'+esc(v.criterionId||'')+'</span></div><div class="jcrit">'+crit+'</div>'+cvRefs(matched)+'</div>';};
|
|
3386
|
+
var rowsHtml='';(d.agentSteps||[]).forEach(function(a){
|
|
3387
|
+
// future-proof: a judge step that carries a REAL anchor still renders.
|
|
3388
|
+
var js=(d.judgeSteps||[]).filter(function(s){return s.anchor!=null&&String(s.anchor)===String(a.n)&&s.kind!=='context';});
|
|
3389
|
+
var anchoredHtml=js.map(function(s){var rs=refStr(s.ref);return '<div class="jstep"><span class="k '+esc(s.kind)+'">'+esc(s.kind)+'</span><span class="t">'+esc(s.text||'')+' '+(rs?'<span class=ref>'+esc(rs)+'</span>':'')+'</span></div>';}).join('');
|
|
3390
|
+
var examiners=cvAll.filter(function(v){return (v.refs||[]).some(function(rf){return refExaminesStep(rf,a);});});
|
|
3391
|
+
var covHtml=examiners.map(function(v){return covEntry(v,a);}).join('');
|
|
3392
|
+
var jhtml=anchoredHtml+covHtml;
|
|
3393
|
+
if(!jhtml)jhtml='<div class="jstep noexam"><span class="t">— not examined by any eval</span></div>';
|
|
3394
|
+
rowsHtml+='<div class="step-l"><div class="evb"><div class="top"><span class="tool">'+esc(a.tool||'')+'</span><span class="st '+esc(a.status||'')+'">'+esc(a.status||'')+'</span></div><div class="det">'+esc(a.detail||'')+'</div></div></div><div class="node '+esc(a.status||'')+'"><div class="ln"></div><div class="n">'+esc(a.n)+'</div><div class="ln"></div></div><div class="step-r"><div class="evb r">'+jhtml+'</div></div>';});
|
|
3395
|
+
var h=d.health||{};
|
|
3396
|
+
var sp=d.subjectProfile;
|
|
3397
|
+
var spHtml=sp?'<div class="ctx"><div class="ctx-h">◇ judge · subject profile (M1) · '+esc(sp.provenance||'')+'</div><div class="ctx-g"><div class="ctx-c"><div class="l">identity</div><div class="v">'+esc(sp.identity||'—')+'</div></div><div class="ctx-c"><div class="l">purpose</div><div class="v">'+esc(sp.purpose||'—')+'</div></div><div class="ctx-c"><div class="l">scope · harness</div><div class="v">'+esc((sp.scope||'—'))+' · harness: '+esc(sp.harness||'—')+'</div></div></div></div>':'';
|
|
3398
|
+
var u=d.understanding;
|
|
3399
|
+
var uHtml=u?'<div class="band"><div class="bh" style="color:var(--cyan)">◇ node-0 GATHER — understanding (M2)</div><div class="re-rephrase" style="margin-top:4px">“'+esc(u.rephrase||'')+'”</div></div>':'';
|
|
3400
|
+
var et=d.expectedTrajectory||[];
|
|
3401
|
+
var etHtml=et.length?'<div class="band"><div class="bh" style="color:var(--primary-soft)">◇ node-0.5 EXPECTED-TRAJECTORY (M3) — how it SHOULD have acted</div><ol class="re-ul" style="margin-top:4px">'+et.map(function(s,i){return '<li><b>'+esc(s.step||i+1)+'.</b> '+esc(s.expected||'')+(s.rationale?' <span class=re-rat>— '+esc(s.rationale)+'</span>':'')+'</li>';}).join('')+'</ol></div>':'';
|
|
3402
|
+
return '<div class="drillbox"><div style="display:flex;gap:9px;align-items:center;flex-wrap:wrap;margin-bottom:6px"><b class="mono">'+esc(d.traceId)+'</b><span class="chip">'+esc(d.route||'all')+'</span><span class="verd '+(d.verdict=='FAIL'?'fail':d.verdict=='PASS'?'pass':'inc')+'">'+esc(d.verdict)+'</span></div>'+
|
|
3403
|
+
resRouting(d.res||'judge-walk')+
|
|
3404
|
+
spHtml+
|
|
3405
|
+
verdictWhy+
|
|
3406
|
+
critiqueBlock(d.criterionVerdicts)+
|
|
3407
|
+
'<div class="ctx"><div class="ctx-h">◇ judge · gather context</div><div class="ctx-g"><div class="ctx-c"><div class="l">harness</div><div class="v">'+esc(ctx.harness||'—')+'</div></div><div class="ctx-c"><div class="l">input + scenario</div><div class="v">'+inputCell(d.input,ctx.scenario)+'</div></div><div class="ctx-c"><div class="l">exit states</div><div class="v">'+esc(ctx.exitStates||'—')+(stateWhy?'<div class="why-note"><span class="ref">why concluded:</span> '+esc(stateWhy)+'</div>':'')+'</div></div></div></div>'+
|
|
3408
|
+
whyChain+
|
|
3409
|
+
uHtml+etHtml+
|
|
3410
|
+
'<div class="lanehdr"><div class="a">target agent — what it did</div><div class="x">step</div><div class="j">judge — eval coverage (which criteria examined this step)</div></div>'+
|
|
3411
|
+
'<div class="grid2">'+rowsHtml+'<div class="band loc"><div class="bh">↯ localize (root, not symptom)</div><div style="font-size:var(--fs-sm);margin-top:4px">'+esc(d.localize||'—')+'</div></div></div>'+
|
|
3412
|
+
'<div class="health"><div class="hc"><div class="l">context</div><div class="v good">'+(h.contextGathered?'✓':'—')+'</div></div><div class="hc"><div class="l">grounded</div><div class="v good">'+esc(h.grounded||0)+'</div></div><div class="hc"><div class="l">assumed</div><div class="v '+((h.assumed||0)>0?'warn':'good')+'">'+esc(h.assumed||0)+'</div></div><div class="hc"><div class="l">root vs symptom</div><div class="v good">'+(h.stoppedAtSymptom?'symptom':'✓ root')+'</div></div></div></div>';}
|
|
3413
|
+
function scorecard(r){var order={fail:0,uncertain:1,indeterminate:1,pass:2,na:3};
|
|
3414
|
+
var cells=Object.keys(r.c).sort(function(a,b){return (order[r.c[a]]||9)-(order[r.c[b]]||9);}).map(function(k){var v=r.c[k];var disp=v=='uncertain'?'indeterminate':v;
|
|
3415
|
+
return '<div class="scc '+esc(v)+'"><span class="nm" title="'+esc((C[k]||{}).n||k)+'">'+esc(k)+'</span><span style="margin-left:auto;color:var(--muted)">'+esc(disp)+'</span></div>';}).join('');
|
|
3416
|
+
return '<div class="drillbox"><div style="display:flex;gap:9px;align-items:center;margin-bottom:6px"><b class="mono">'+esc(r.t)+'</b><span class="chip">'+esc(r.r||'all')+'</span><span class="verd '+(r.v=='PASS'?'pass':r.v=='FAIL'?'fail':'inc')+'">'+esc(r.v)+'</span></div>'+
|
|
3417
|
+
resRouting(r.res||'judged-walk-not-captured')+
|
|
3418
|
+
(r.root?'<div class="note" style="margin:0 0 8px"><span class="tag">★ ROOT</span> '+esc(r.root)+'</div>':'')+
|
|
3419
|
+
(r.g?'<div class="mono" style="font-size:var(--fs-xs);color:var(--muted);margin-bottom:4px">grounding: '+esc(r.g)+'</div>':'')+
|
|
3420
|
+
'<div style="font-family:var(--mono);font-size:var(--fs-2xs);color:var(--muted);text-transform:uppercase;margin:6px 0">per-trajectory scorecard — '+Object.keys(r.c).length+' criteria</div>'+
|
|
3421
|
+
'<div class="scgrid">'+cells+'</div>'+
|
|
3422
|
+
// WS-1 — no-walk traces still surface their judge reasoning (per-criterion
|
|
3423
|
+
// critique + refs) here, so a trace WITHOUT a judge_steps walk is never an
|
|
3424
|
+
// empty "How the Judge Reasoned" panel when its verdict file carries verdicts.
|
|
3425
|
+
critiqueBlock(r.cv)+'</div>';}
|
|
3426
|
+
function drill(id,el){document.querySelectorAll('table.ledger tr').forEach(function(t){t.classList.remove('sel');});if(el)el.classList.add('sel');
|
|
3427
|
+
var box=document.getElementById('drill');if(!box)return;
|
|
3428
|
+
var lrow=L.filter(function(x){return x.t==id;})[0];
|
|
3429
|
+
if(J[id]){var d=J[id];
|
|
3430
|
+
// WS-1 — graft the per-criterion verdicts (carried on the ledger row for EVERY
|
|
3431
|
+
// trace) onto the walk object so the side-by-side drill also shows the per-
|
|
3432
|
+
// criterion critique + refs, not just the step-anchored judge walk.
|
|
3433
|
+
if(lrow&&(!d.criterionVerdicts||!d.criterionVerdicts.length))d.criterionVerdicts=lrow.cv||[];
|
|
3434
|
+
if(lrow&&!d.res)d.res=lrow.res;
|
|
3435
|
+
box.innerHTML=sideBySide(d);}
|
|
3436
|
+
else if(lrow){box.innerHTML=scorecard(lrow);}
|
|
3437
|
+
box.scrollIntoView({behavior:'smooth',block:'nearest'});}
|
|
3438
|
+
function wireFilters(){document.querySelectorAll('.lfilter b').forEach(function(b){b.addEventListener('click',function(){F=b.getAttribute('data-flt');document.querySelectorAll('.lfilter b').forEach(function(x){x.classList.remove('on');});b.classList.add('on');render();});});
|
|
3439
|
+
var q=document.getElementById('q');if(q)q.addEventListener('input',render);}
|
|
3440
|
+
function wireToggles(){
|
|
3441
|
+
// UI-8 — calibrate tags are MUTUALLY EXCLUSIVE within a group (radio semantics):
|
|
3442
|
+
// selecting one clears the others in the same [data-calgroup]; clicking the active
|
|
3443
|
+
// one again clears it. The agree/revise/refute alignment row stays free-toggle.
|
|
3444
|
+
document.querySelectorAll('.calgroup').forEach(function(g){var btns=[].slice.call(g.querySelectorAll('b'));
|
|
3445
|
+
btns.forEach(function(b){b.addEventListener('click',function(){var wasOn=b.classList.contains('on');
|
|
3446
|
+
btns.forEach(function(x){x.classList.remove('on');});if(!wasOn)b.classList.add('on');});});});
|
|
3447
|
+
document.querySelectorAll('.areview b').forEach(function(b){b.addEventListener('click',function(){b.classList.toggle('on');});});
|
|
3448
|
+
// WS-3 — §5 self-eval HITL picks (5.4 keep/revise/retire · 5.8 agree/disagree):
|
|
3449
|
+
// mutually-exclusive within a [data-vgroup]; click the active one again to clear.
|
|
3450
|
+
var seGroups={};document.querySelectorAll('.se-pick').forEach(function(p){var g=p.getAttribute('data-vgroup');(seGroups[g]=seGroups[g]||[]).push(p);});
|
|
3451
|
+
Object.keys(seGroups).forEach(function(g){var picks=seGroups[g];picks.forEach(function(p){p.addEventListener('click',function(){var wasOn=p.classList.contains('on');picks.forEach(function(x){x.classList.remove('on');});if(!wasOn)p.classList.add('on');});});});
|
|
3452
|
+
document.querySelectorAll('.lnk[data-drill]').forEach(function(l){l.addEventListener('click',function(){var t2=document.querySelector('nav.tabs button[data-tab="t2"]');if(t2)t2.click();setTimeout(function(){drill(l.getAttribute('data-drill'));},60);});});
|
|
3453
|
+
// WS-4 — wire EVERY copy-as-markdown button (the §5 full-handover button, each
|
|
3454
|
+
// per-routed-item button, and any other): each copies its OWN data-md payload and
|
|
3455
|
+
// flashes a "copied ✓" confirmation. Was querySelector (first-only) → querySelectorAll.
|
|
3456
|
+
// Restricted clipboard contexts (headless / some file://) REJECT writeText — the
|
|
3457
|
+
// rejection AND any synchronous throw are CAUGHT so a denied clipboard NEVER raises an
|
|
3458
|
+
// uncaught page error; it just shows a "copy failed" hint and restores. ZERO errors.
|
|
3459
|
+
document.querySelectorAll('.copy-md').forEach(function(cp){cp.addEventListener('click',function(){
|
|
3460
|
+
var md=cp.getAttribute('data-md')||'';
|
|
3461
|
+
var prev=cp.textContent;
|
|
3462
|
+
var done=function(ok){cp.textContent=ok?'copied ✓':'copy failed';cp.classList.add(ok?'copied':'copyfail');
|
|
3463
|
+
setTimeout(function(){cp.textContent=prev;cp.classList.remove('copied');cp.classList.remove('copyfail');},1200);};
|
|
3464
|
+
try{
|
|
3465
|
+
if(navigator.clipboard&&navigator.clipboard.writeText){
|
|
3466
|
+
navigator.clipboard.writeText(md).then(function(){done(true);},function(){done(false);});
|
|
3467
|
+
}else{done(false);}
|
|
3468
|
+
}catch(e){done(false);}
|
|
3469
|
+
});});}
|
|
3470
|
+
wireTabs();wireFilters();wireToggles();render();
|
|
3471
|
+
`;
|
|
3472
|
+
}
|
|
3473
|
+
|
|
3474
|
+
// ── the full 5-tab report ────────────────────────────────────────────────────
|
|
3475
|
+
|
|
3476
|
+
/**
|
|
3477
|
+
* Render the 5-tab HTML eval-report (the operator-APPROVED component spec). Pure
|
|
3478
|
+
* string assembly; the only I/O is reading the bundled brand assets. The injected
|
|
3479
|
+
* `generatedAt` is the only non-deterministic input → masked-byte-identical.
|
|
3480
|
+
*/
|
|
3481
|
+
export function renderEvalReport(input: EvalReportInput): string {
|
|
3482
|
+
const theme = readFileSync(join(BRAND_DIR, "theme.css"), "utf8");
|
|
3483
|
+
const wordmark = readFileSync(join(BRAND_DIR, "wordmark.html"), "utf8");
|
|
3484
|
+
|
|
3485
|
+
// PRD R-4 — audience gate. CLIENT is the DEFAULT (leak-safe): the §5 Self-Eval [INTERNAL]
|
|
3486
|
+
// tab + nav button are NOT emitted into the HTML at all (NODE-STRIP, not CSS-hidden).
|
|
3487
|
+
const internal = (input.audience ?? "client") === "internal";
|
|
3488
|
+
const selfEvalNav = internal
|
|
3489
|
+
? ` <button class="tab-btn internal" data-tab="t5">⑤ Self-Eval [INTERNAL]</button>\n`
|
|
3490
|
+
: "";
|
|
3491
|
+
const selfEvalPanel = internal
|
|
3492
|
+
? ` <section class="panel" id="t5">${selfEvalTab(input)}</section>\n`
|
|
3493
|
+
: "";
|
|
3494
|
+
|
|
3495
|
+
const headerTitle = "evaluator · Eval Report";
|
|
3496
|
+
const headerMeta =
|
|
3497
|
+
`<span class="mk">subject</span> <span class="mv">${esc(input.subject.name)}</span>` +
|
|
3498
|
+
`<span class="sep">·</span><span class="mk">generated</span> <span class="mv">${esc(input.generatedAt)}</span>`;
|
|
3499
|
+
const header = wordmark.replaceAll("{{HEADER_TITLE}}", esc(headerTitle)).replaceAll("{{HEADER_META}}", headerMeta);
|
|
3500
|
+
|
|
3501
|
+
// deterministic embedded data for the §2 ledger + drill
|
|
3502
|
+
const ledger = input.ledger ?? [];
|
|
3503
|
+
const rootByTrace = new Map((input.topFindings ?? []).filter((tf) => tf.exampleTraceId).map((tf) => [tf.exampleTraceId as string, tf.root ?? ""]));
|
|
3504
|
+
const compactLedger = ledger.map((r) => ({
|
|
3505
|
+
t: r.trajectoryId,
|
|
3506
|
+
r: r.route ?? "all",
|
|
3507
|
+
v: r.verdict,
|
|
3508
|
+
c: r.perCriterion,
|
|
3509
|
+
f: r.failingCriteria,
|
|
3510
|
+
root: rootByTrace.get(r.trajectoryId) ?? "",
|
|
3511
|
+
g: r.grounding ?? "",
|
|
3512
|
+
js: r.judgeSteps && r.judgeSteps.length > 0 ? 1 : 0,
|
|
3513
|
+
// WS-2 — how this trajectory resolved (judge-walk · judged-walk-not-captured ·
|
|
3514
|
+
// truncated) → the ledger resolution badge + the drill routing line.
|
|
3515
|
+
res: r.resolution ?? "",
|
|
3516
|
+
// WS-1 — the per-criterion JUDGE verdicts (result · critique · refs) for THIS
|
|
3517
|
+
// trace. Carried on EVERY ledger row so the §2 drill renders "How the Judge
|
|
3518
|
+
// Reasoned" for every judged trace — the walk traces graft it onto the side-by-
|
|
3519
|
+
// side; the no-walk traces render it under the per-trajectory scorecard. Empty []
|
|
3520
|
+
// only when the verdict file genuinely carried no per-criterion verdict.
|
|
3521
|
+
cv: r.criterionVerdicts ?? [],
|
|
3522
|
+
}));
|
|
3523
|
+
// `m` = the code-vs-judge router (`checkMethod`) so the §2 judge lane can tag a
|
|
3524
|
+
// step's eval-coverage entry CODE (deterministic) / HYBRID / JUDGE (llm).
|
|
3525
|
+
const critMap: Record<string, { n: string; s: string; m: string }> = {};
|
|
3526
|
+
for (const c of input.criteria) critMap[c.id] = { n: c.statement, s: c.severity, m: c.checkMethod ?? "" };
|
|
3527
|
+
const walks: Record<string, unknown> = {};
|
|
3528
|
+
for (const r of ledger) {
|
|
3529
|
+
if (r.judgeSteps && r.judgeSteps.length > 0) {
|
|
3530
|
+
walks[r.trajectoryId] = {
|
|
3531
|
+
traceId: r.trajectoryId,
|
|
3532
|
+
route: r.route ?? "all",
|
|
3533
|
+
verdict: r.verdict,
|
|
3534
|
+
// Gap A — the raw triggering INPUT (ground-truth from the trace), rendered
|
|
3535
|
+
// ABOVE the judge's scenario LABEL in the §2 drill. "" ⇒ drill shows "—".
|
|
3536
|
+
input: r.input ?? "",
|
|
3537
|
+
context: r.context ?? {},
|
|
3538
|
+
agentSteps: r.agentSteps ?? [],
|
|
3539
|
+
judgeSteps: r.judgeSteps ?? [],
|
|
3540
|
+
localize: r.localize ?? "",
|
|
3541
|
+
health: r.health ?? {},
|
|
3542
|
+
// §9.4.4 v2.2 — node-0 understanding · node-0.5 expected-trajectory · M1 profile.
|
|
3543
|
+
understanding: r.understanding ?? null,
|
|
3544
|
+
expectedTrajectory: r.expectedTrajectory ?? [],
|
|
3545
|
+
subjectProfile: r.subjectProfile ?? input.subjectProfile ?? null,
|
|
3546
|
+
};
|
|
3547
|
+
}
|
|
3548
|
+
}
|
|
3549
|
+
const ledgerJson = JSON.stringify(compactLedger);
|
|
3550
|
+
const critJson = JSON.stringify(critMap);
|
|
3551
|
+
const walksJson = JSON.stringify(walks);
|
|
3552
|
+
// WS-2 — the resolution badge label + routing explanation per class (server-derived,
|
|
3553
|
+
// injected so the client row badge + drill routing line read ONE source of truth).
|
|
3554
|
+
const resMetaJson = JSON.stringify({
|
|
3555
|
+
"judge-walk": resolutionMeta("judge-walk"),
|
|
3556
|
+
"judged-walk-not-captured": resolutionMeta("judged-walk-not-captured"),
|
|
3557
|
+
truncated: resolutionMeta("truncated"),
|
|
3558
|
+
});
|
|
3559
|
+
|
|
3560
|
+
return `<!DOCTYPE html>
|
|
3561
|
+
<html lang="en" data-theme="dark">
|
|
3562
|
+
<head>
|
|
3563
|
+
<meta charset="UTF-8">
|
|
3564
|
+
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
3565
|
+
<title>${esc(headerTitle)} — ${esc(input.subject.name)}</title>
|
|
3566
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
3567
|
+
<link href="https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
|
3568
|
+
<style>
|
|
3569
|
+
${theme}
|
|
3570
|
+
${REPORT_CSS}
|
|
3571
|
+
</style>
|
|
3572
|
+
</head>
|
|
3573
|
+
<body>
|
|
3574
|
+
${header}
|
|
3575
|
+
<nav class="tabs">
|
|
3576
|
+
<button class="tab-btn active" data-tab="t1">① Overview</button>
|
|
3577
|
+
<button class="tab-btn" data-tab="t2">② Trajectory · Judge Behaviour</button>
|
|
3578
|
+
<button class="tab-btn" data-tab="t3">③ Eval Scorecard</button>
|
|
3579
|
+
<button class="tab-btn" data-tab="t4">④ Findings</button>
|
|
3580
|
+
${selfEvalNav}</nav>
|
|
3581
|
+
<main>
|
|
3582
|
+
<section class="panel active" id="t1">${overviewTab(input)}</section>
|
|
3583
|
+
<section class="panel" id="t2">${trajectoryTab(input)}</section>
|
|
3584
|
+
<section class="panel" id="t3">${scorecardTab(input)}</section>
|
|
3585
|
+
<section class="panel" id="t4">${findingsTab(input)}</section>
|
|
3586
|
+
${selfEvalPanel}</main>
|
|
3587
|
+
<script data-pii="ledger">
|
|
3588
|
+
${clientScript(ledgerJson, critJson, walksJson, resMetaJson)}
|
|
3589
|
+
</script>
|
|
3590
|
+
</body>
|
|
3591
|
+
</html>
|
|
3592
|
+
`;
|
|
3593
|
+
}
|
|
3594
|
+
|
|
3595
|
+
// ── WS-1 file-based re-render entry (models discover's writeDiscoverReportFromFiles) ──
|
|
3596
|
+
|
|
3597
|
+
/** Injected I/O for the file-based eval-report entry (keeps the function pure of fs). */
|
|
3598
|
+
export interface EvalReportFilesIO {
|
|
3599
|
+
readFile: (p: string) => string;
|
|
3600
|
+
writeFile: (p: string, s: string) => void;
|
|
3601
|
+
readDir: (p: string) => string[];
|
|
3602
|
+
}
|
|
3603
|
+
|
|
3604
|
+
/** The data-completeness probe the harness reports after a re-render. */
|
|
3605
|
+
export interface EvalReportCompleteness {
|
|
3606
|
+
tracesInVerdicts: number;
|
|
3607
|
+
tracesRenderedWithJudgeSteps: number;
|
|
3608
|
+
/** WS-1 — the honest completeness metric: traces whose drill surfaces ≥1 per-
|
|
3609
|
+
* criterion judge verdict (critique-before-verdict). Should == every JUDGED trace. */
|
|
3610
|
+
tracesRenderedWithCritique: number;
|
|
3611
|
+
/** traces whose verdict file carried renderable judge data (verdicts[] or a walk)
|
|
3612
|
+
* but the drill would render an EMPTY "How the Judge Reasoned" panel. Goal: 0. */
|
|
3613
|
+
emptyUIrows: number;
|
|
3614
|
+
/** WS-6 — char length of the reconstructed subject system prompt (0 ⇒ UNAVAILABLE). */
|
|
3615
|
+
systemPromptChars: number;
|
|
3616
|
+
/** WS-6 — true when the entity hero renders a reconstructed system prompt. */
|
|
3617
|
+
systemPromptReconstructed: boolean;
|
|
3618
|
+
/** WS-4 — number of true-FAIL criteria routed into the §5 EV-051 diagnostics handover. */
|
|
3619
|
+
routedFailures: number;
|
|
3620
|
+
}
|
|
3621
|
+
|
|
3622
|
+
/**
|
|
3623
|
+
* WS-1 — re-render the §2 trajectory report DIRECTLY from on-disk artifacts (the
|
|
3624
|
+
* RICH per-trajectory `verdicts/*.json` + the folded `AGG-RESULT.json` + the
|
|
3625
|
+
* criteria/suite), binding the per-criterion judge reasoning (critique + refs) and
|
|
3626
|
+
* the judge_steps walk (where present) to EVERY evaluated trace. This is the eval
|
|
3627
|
+
* analogue of `writeDiscoverReportFromFiles`: a pure shipped-function call over the
|
|
3628
|
+
* file set, with NO strict-schema gate that would drop the rich verdict files (the
|
|
3629
|
+
* verdict files that carry an object `exitStates` / omit `subjectProfile.tools` —
|
|
3630
|
+
* exactly the ones with the walk — survive here, where production's strict
|
|
3631
|
+
* `parseMatrixVerdictFile` would have rejected them).
|
|
3632
|
+
*
|
|
3633
|
+
* Returns the report path + the data-completeness probe. PURE except the injected I/O.
|
|
3634
|
+
*/
|
|
3635
|
+
export function writeEvalReportFromFiles(
|
|
3636
|
+
params: {
|
|
3637
|
+
verdictsDir: string;
|
|
3638
|
+
aggPath: string;
|
|
3639
|
+
criteriaPath: string;
|
|
3640
|
+
outPath: string;
|
|
3641
|
+
subjectName: string;
|
|
3642
|
+
subjectSource?: string;
|
|
3643
|
+
generatedAt: string;
|
|
3644
|
+
/** WS-6 — OPTIONAL trace batch (or a single representative trace) used ONLY to
|
|
3645
|
+
* reconstruct the subject SYSTEM PROMPT for the entity hero. The harness streams
|
|
3646
|
+
* one trace off the raw ndjson and passes it here; the system prompt is identical
|
|
3647
|
+
* across the batch so one trace suffices. ABSENT ⇒ system prompt stays UNAVAILABLE. */
|
|
3648
|
+
traces?: EvalTrace[];
|
|
3649
|
+
/** PRD R-4 — render audience (default "client", leak-safe). "internal" shows §5. */
|
|
3650
|
+
audience?: "client" | "internal";
|
|
3651
|
+
},
|
|
3652
|
+
io: EvalReportFilesIO,
|
|
3653
|
+
): { report: string; completeness: EvalReportCompleteness } {
|
|
3654
|
+
// 1) read the RICH verdict files TOLERANTLY (raw JSON.parse — never the strict
|
|
3655
|
+
// schema gate, which rejects exactly the walk-bearing files). NO trace dropped.
|
|
3656
|
+
const verdictFiles = io
|
|
3657
|
+
.readDir(params.verdictsDir)
|
|
3658
|
+
.filter((f) => f.endsWith(".json"))
|
|
3659
|
+
.map((f) => {
|
|
3660
|
+
try {
|
|
3661
|
+
return JSON.parse(io.readFile(join(params.verdictsDir, f))) as MatrixVerdictFile;
|
|
3662
|
+
} catch {
|
|
3663
|
+
return null;
|
|
3664
|
+
}
|
|
3665
|
+
})
|
|
3666
|
+
.filter((v): v is MatrixVerdictFile => v !== null && typeof (v as { trajectoryId?: unknown }).trajectoryId === "string");
|
|
3667
|
+
|
|
3668
|
+
// 2) the folded scorecard (AGG-RESULT.maskedScorecard = gate + variance) + the
|
|
3669
|
+
// per-criterion folded verdicts (summary.perCriterion), with a representative
|
|
3670
|
+
// critique grafted from a matching per-trajectory verdict (for the gating rolls).
|
|
3671
|
+
const agg = JSON.parse(io.readFile(params.aggPath)) as {
|
|
3672
|
+
summary?: { perCriterion?: Array<{ criterionId: string; folded: string; confidence?: number }> };
|
|
3673
|
+
maskedScorecard?: Scorecard;
|
|
3674
|
+
};
|
|
3675
|
+
const scorecard = (agg.maskedScorecard ?? { gate: { total: 0, passCount: 0, failedCriteria: [], gatedBy: [], indeterminateBy: [], passed: true, runVerdict: "pass" } }) as Scorecard;
|
|
3676
|
+
const repCritique = new Map<string, { critique: string; refs?: unknown[]; traceId: string }>();
|
|
3677
|
+
for (const f of verdictFiles) {
|
|
3678
|
+
for (const v of (f as { verdicts?: Array<{ criterionId: string; result: string; critique?: string; refs?: unknown[] }> }).verdicts ?? []) {
|
|
3679
|
+
if (v.critique && !repCritique.has(`${v.criterionId}:${v.result}`)) {
|
|
3680
|
+
repCritique.set(`${v.criterionId}:${v.result}`, { critique: v.critique, refs: v.refs, traceId: (f as { trajectoryId: string }).trajectoryId });
|
|
3681
|
+
}
|
|
3682
|
+
}
|
|
3683
|
+
}
|
|
3684
|
+
const verdicts: CriterionVerdict[] = (agg.summary?.perCriterion ?? []).map((pc) => {
|
|
3685
|
+
const rep = repCritique.get(`${pc.criterionId}:${pc.folded}`);
|
|
3686
|
+
return {
|
|
3687
|
+
criterionId: pc.criterionId,
|
|
3688
|
+
traceId: rep?.traceId ?? "(suite)",
|
|
3689
|
+
result: pc.folded as CriterionVerdict["result"],
|
|
3690
|
+
confidence: pc.confidence ?? 0,
|
|
3691
|
+
critique: rep?.critique ?? "",
|
|
3692
|
+
...(rep?.refs ? { refs: rep.refs as CriterionVerdict["refs"] } : {}),
|
|
3693
|
+
};
|
|
3694
|
+
});
|
|
3695
|
+
|
|
3696
|
+
// 3) the criteria (discover/suite shape) → ReportCriterion[].
|
|
3697
|
+
const rawCrit = JSON.parse(io.readFile(params.criteriaPath)) as unknown;
|
|
3698
|
+
const critArr: Array<Record<string, unknown>> = Array.isArray(rawCrit)
|
|
3699
|
+
? (rawCrit as Array<Record<string, unknown>>)
|
|
3700
|
+
: ((rawCrit as { entries?: Array<Record<string, unknown>> }).entries ?? []);
|
|
3701
|
+
const criteria: ReportCriterion[] = critArr.map((c) => {
|
|
3702
|
+
const meta = (c.metadata ?? {}) as Record<string, unknown>;
|
|
3703
|
+
const severity = String((c.severity as string) ?? (meta.severity as string) ?? "MED");
|
|
3704
|
+
return {
|
|
3705
|
+
id: String(c.id ?? c.criterionId ?? ""),
|
|
3706
|
+
statement: String(c.statement ?? ""),
|
|
3707
|
+
severity,
|
|
3708
|
+
gating: GATING_SEVERITIES.has(severity),
|
|
3709
|
+
...(meta.dimension !== undefined ? { dimension: String(meta.dimension) } : {}),
|
|
3710
|
+
...(Array.isArray(c.judgeInputs) ? { judgeInputs: c.judgeInputs as string[] } : {}),
|
|
3711
|
+
checkMethod: String((c.judgeKind as string) ?? (meta.check_method as string) ?? "llm-judge"),
|
|
3712
|
+
provenance: { kind: "defined" as const, label: "defined eval-matrix criterion" },
|
|
3713
|
+
};
|
|
3714
|
+
});
|
|
3715
|
+
|
|
3716
|
+
// 3b) WS-4 — reconstruct the EV-051 diagnostics HANDOVER from the folded gate, using
|
|
3717
|
+
// the SHIPPED `routeFailures` (no hand-rolled routing). Each failed criterion →
|
|
3718
|
+
// a FailureRef enriched with its folded result + a representative critique/trace
|
|
3719
|
+
// (from the verdict files). `routeFailures` then partitions: only TRUE FAILs
|
|
3720
|
+
// (result === "fail") route to diagnostics — uncertain/indeterminate go to the
|
|
3721
|
+
// calibration loop and are excluded — so the §5 handover shows exactly the routed
|
|
3722
|
+
// count, matching the run's `summary.routedFailures`. NEVER invents a failure.
|
|
3723
|
+
const foldedById = new Map((agg.summary?.perCriterion ?? []).map((pc) => [pc.criterionId, pc.folded]));
|
|
3724
|
+
const failedCriteria = (scorecard.gate?.failedCriteria ?? []) as Array<{ criterionId: string; severity: string }>;
|
|
3725
|
+
const failures: FailureRef[] = failedCriteria.map((fc) => {
|
|
3726
|
+
const folded = foldedById.get(fc.criterionId) ?? "fail";
|
|
3727
|
+
const rep = repCritique.get(`${fc.criterionId}:${folded}`) ?? repCritique.get(`${fc.criterionId}:fail`);
|
|
3728
|
+
return {
|
|
3729
|
+
criterionId: fc.criterionId,
|
|
3730
|
+
severity: fc.severity,
|
|
3731
|
+
flag: CriterionFlag.EvalWorthy,
|
|
3732
|
+
traceId: rep?.traceId ?? "(suite)",
|
|
3733
|
+
result: folded as FailureRef["result"],
|
|
3734
|
+
critique: rep?.critique ?? "(no critique recorded)",
|
|
3735
|
+
};
|
|
3736
|
+
});
|
|
3737
|
+
// routeFailures internally filters to true FAILs (partitionRouting); pass them all.
|
|
3738
|
+
const diagnoseFailures = failures.filter((f) => f.result === OutcomeVerdict.Fail);
|
|
3739
|
+
const handover: HandoverBundle | null =
|
|
3740
|
+
diagnoseFailures.length > 0
|
|
3741
|
+
? routeFailures({
|
|
3742
|
+
subject: { kind: "agent", name: params.subjectName, path: `subjects/${params.subjectName}` },
|
|
3743
|
+
failures: diagnoseFailures,
|
|
3744
|
+
artifacts: [],
|
|
3745
|
+
producedBy: "evaluator",
|
|
3746
|
+
producedAt: params.generatedAt,
|
|
3747
|
+
})
|
|
3748
|
+
: null;
|
|
3749
|
+
|
|
3750
|
+
// 4) build the rich input + render.
|
|
3751
|
+
const input = buildEvalReportInput({
|
|
3752
|
+
subject: { name: params.subjectName, ...(params.subjectSource ? { source: params.subjectSource } : {}) },
|
|
3753
|
+
scorecard,
|
|
3754
|
+
verdicts,
|
|
3755
|
+
criteria,
|
|
3756
|
+
matrixVerdictFiles: verdictFiles,
|
|
3757
|
+
handover,
|
|
3758
|
+
generatedAt: params.generatedAt,
|
|
3759
|
+
// WS-6 — forward the trace(s) so the system prompt is reconstructed for the hero.
|
|
3760
|
+
...(params.traces !== undefined ? { traces: params.traces } : {}),
|
|
3761
|
+
});
|
|
3762
|
+
// PRD R-4 — set audience on the built input (buildEvalReportInput does not forward it).
|
|
3763
|
+
if (params.audience !== undefined) input.audience = params.audience;
|
|
3764
|
+
io.writeFile(params.outPath, renderEvalReport(input));
|
|
3765
|
+
|
|
3766
|
+
// 5) the data-completeness probe over the BOUND ledger (what the UI will render).
|
|
3767
|
+
const ledger = input.ledger ?? [];
|
|
3768
|
+
const completeness: EvalReportCompleteness = {
|
|
3769
|
+
tracesInVerdicts: verdictFiles.length,
|
|
3770
|
+
tracesRenderedWithJudgeSteps: ledger.filter((r) => (r.judgeSteps?.length ?? 0) > 0).length,
|
|
3771
|
+
tracesRenderedWithCritique: ledger.filter((r) => (r.criterionVerdicts?.length ?? 0) > 0).length,
|
|
3772
|
+
// an empty UI row = the verdict file carried judge data (a walk OR ≥1 per-criterion
|
|
3773
|
+
// verdict) yet the drill would bind NOTHING. With per-criterion critique bound to
|
|
3774
|
+
// every judged row, this is 0 unless a verdict file is genuinely empty.
|
|
3775
|
+
emptyUIrows: ledger.filter(
|
|
3776
|
+
(r) => (r.judgeSteps?.length ?? 0) === 0 && (r.criterionVerdicts?.length ?? 0) === 0,
|
|
3777
|
+
).length,
|
|
3778
|
+
systemPromptChars: (input.subjectProfile?.systemPrompt ?? "").length,
|
|
3779
|
+
systemPromptReconstructed: (input.subjectProfile?.inferredFields ?? []).includes("systemPrompt"),
|
|
3780
|
+
routedFailures: input.handover?.acceptance.criteria.length ?? 0,
|
|
3781
|
+
};
|
|
3782
|
+
return { report: params.outPath, completeness };
|
|
3783
|
+
}
|
|
3784
|
+
|
|
3785
|
+
// ── auto-open helper (built; fired by the orchestrator, NOT here) ────────────
|
|
3786
|
+
|
|
3787
|
+
/**
|
|
3788
|
+
* The cross-platform command to open a rendered report. PURE — returns the
|
|
3789
|
+
* command string; it NEVER spawns a process (the orchestrator fires it
|
|
3790
|
+
* post-render; a test must never auto-open). macOS `open` · Linux `xdg-open` ·
|
|
3791
|
+
* Windows `start`.
|
|
3792
|
+
*/
|
|
3793
|
+
export function autoOpenCommand(platform: string, filePath: string): string {
|
|
3794
|
+
if (platform === "darwin") return `open ${filePath}`;
|
|
3795
|
+
if (platform === "win32") return `cmd /c start "" ${filePath}`;
|
|
3796
|
+
return `xdg-open ${filePath}`;
|
|
3797
|
+
}
|
|
3798
|
+
|
|
3799
|
+
// ── CLI entrypoint ──────────────────────────────────────────────────────────
|
|
3800
|
+
//
|
|
3801
|
+
// bun scripts/render-eval-report.ts <input.json> [out.html] [runId]
|
|
3802
|
+
// Reads an EvalReportInput (the rich, already-built shape), prints the D-1
|
|
3803
|
+
// terminal cards, writes the 5-tab HTML report, and prints the cross-platform
|
|
3804
|
+
// auto-open command (does NOT fire it — the orchestrator opens post-render).
|
|
3805
|
+
|
|
3806
|
+
declare const Bun: { argv: string[] } | undefined;
|
|
3807
|
+
|
|
3808
|
+
async function main(): Promise<void> {
|
|
3809
|
+
const argv = typeof Bun !== "undefined" ? Bun.argv.slice(2) : process.argv.slice(2);
|
|
3810
|
+
const [inputPath, outArg, runIdArg] = argv;
|
|
3811
|
+
if (!inputPath) {
|
|
3812
|
+
console.error("usage: render-eval-report.ts <input.json> [out.html] [runId]");
|
|
3813
|
+
process.exit(2);
|
|
3814
|
+
return;
|
|
3815
|
+
}
|
|
3816
|
+
const { readFileSync: rf, writeFileSync, mkdirSync } = await import("node:fs");
|
|
3817
|
+
const { reportDir } = await import("./artifact-paths.ts");
|
|
3818
|
+
const { join: pjoin } = await import("node:path");
|
|
3819
|
+
const input = JSON.parse(rf(inputPath, "utf8")) as EvalReportInput;
|
|
3820
|
+
// P8: a bare run DEFAULTS the report under the localized dot-root
|
|
3821
|
+
// `.mutagent/evaluator/reports/<runId>/evaluation-report.html` (never /tmp or .memory).
|
|
3822
|
+
const runId = runIdArg ?? input.subject.name;
|
|
3823
|
+
const outPath = outArg ?? pjoin(reportDir(runId), "evaluation-report.html");
|
|
3824
|
+
mkdirSync(pjoin(outPath, ".."), { recursive: true });
|
|
3825
|
+
console.info(renderEvalCards(input));
|
|
3826
|
+
writeFileSync(outPath, renderEvalReport(input));
|
|
3827
|
+
const plat = typeof process !== "undefined" ? process.platform : "linux";
|
|
3828
|
+
console.info(`\nreport written: ${outPath}`);
|
|
3829
|
+
console.info(`auto-open (orchestrator fires this): ${autoOpenCommand(plat, outPath)}`);
|
|
3830
|
+
process.exit(0);
|
|
3831
|
+
}
|
|
3832
|
+
|
|
3833
|
+
const isMain =
|
|
3834
|
+
typeof import.meta !== "undefined" &&
|
|
3835
|
+
(import.meta as unknown as { main?: boolean }).main === true;
|
|
3836
|
+
if (isMain) {
|
|
3837
|
+
void main();
|
|
3838
|
+
}
|