observability-toolkit-hooks 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG.md +144 -0
- package/LICENSE +95 -0
- package/README.md +201 -2
- package/dist/handlers/compact.js +38 -0
- package/dist/handlers/mcp-status.js +66 -0
- package/dist/handlers/notification.js +45 -0
- package/dist/handlers/post-tool-agent.js +157 -0
- package/dist/handlers/post-tool-changelog-sync.js +148 -0
- package/dist/handlers/post-tool-code-structure.js +116 -0
- package/dist/handlers/post-tool-edit-ledger.js +15 -0
- package/dist/handlers/post-tool-failure.js +83 -0
- package/dist/handlers/post-tool-git.js +79 -0
- package/dist/handlers/post-tool-mcp.js +105 -0
- package/dist/handlers/post-tool-outcome.js +48 -0
- package/dist/handlers/post-tool-task-state.js +74 -0
- package/dist/handlers/post-tool.js +132 -0
- package/dist/handlers/pre-tool-agent.js +192 -0
- package/dist/handlers/pre-tool.js +218 -0
- package/dist/handlers/session-end.js +39 -0
- package/dist/handlers/session-start.js +262 -0
- package/dist/handlers/stop-error-reminder.js +195 -0
- package/dist/handlers/stop-quality-evaluation.js +58 -0
- package/dist/handlers/stop-session-summary.js +62 -0
- package/dist/handlers/stop-token-metrics.js +83 -0
- package/dist/handlers/stop.js +25 -0
- package/dist/handlers/subagent.js +105 -0
- package/dist/handlers/tool-event-router.js +39 -0
- package/dist/handlers/user-prompt.js +218 -0
- package/dist/hook-runner.js +10 -0
- package/dist/hook-runner.js.map +7 -0
- package/dist/lib/agent/agent-context.js +123 -0
- package/dist/lib/agent/agent-evaluations.js +155 -0
- package/dist/lib/agent/agent-file-budget.js +65 -0
- package/dist/lib/agent/agent-sources.js +165 -0
- package/dist/lib/agent/synth-spans.js +51 -0
- package/dist/lib/core/bounded-label.js +6 -0
- package/dist/lib/core/category-rules.js +11 -0
- package/dist/lib/core/child-result.js +19 -0
- package/dist/lib/core/config-dir-recovery.js +22 -0
- package/dist/lib/core/constants.js +171 -0
- package/dist/lib/core/detached-runner.js +55 -0
- package/dist/lib/core/file-utils.js +353 -0
- package/dist/lib/core/format-utils.js +51 -0
- package/dist/lib/core/http-utils.js +74 -0
- package/dist/lib/core/http1-fetch.js +14 -0
- package/dist/lib/core/identity-key-resolver.js +149 -0
- package/dist/lib/core/jsonl-tail.js +207 -0
- package/dist/lib/core/load-envrc.js +72 -0
- package/dist/lib/core/parsers.js +43 -0
- package/dist/lib/core/regex.js +27 -0
- package/dist/lib/core/settings-env.js +31 -0
- package/dist/lib/core/shell-command.js +122 -0
- package/dist/lib/core/units.js +97 -0
- package/dist/lib/git/git-cache.js +158 -0
- package/dist/lib/git/git-commit.js +189 -0
- package/dist/lib/git/git-exec.js +30 -0
- package/dist/lib/mcp/mcp-completeness.js +40 -0
- package/dist/lib/mcp/mcp-config.js +118 -0
- package/dist/lib/mcp/mcp-failure-latch.js +22 -0
- package/dist/lib/mcp/mcp-identity-launcher.js +43 -0
- package/dist/lib/otel/attributes.js +322 -0
- package/dist/lib/otel/evaluation-attrs.js +51 -0
- package/dist/lib/otel/file-exporters.js +169 -0
- package/dist/lib/otel/host-resource.js +61 -0
- package/dist/lib/otel/otel-headers-helper.js +26 -0
- package/dist/lib/otel/otel-monitor.js +181 -0
- package/dist/lib/otel/otel.js +235 -0
- package/dist/lib/otel/otlp-headers.js +101 -0
- package/dist/lib/otel/otlp-post.js +122 -0
- package/dist/lib/otel/otlp-serialize.js +142 -0
- package/dist/lib/otel/otlp-signal.js +2 -0
- package/dist/lib/otel/remote-spans.js +142 -0
- package/dist/lib/otel/service-identity.js +10 -0
- package/dist/lib/otel/telemetry-alerts.js +43 -0
- package/dist/lib/otel/token-metrics.js +122 -0
- package/dist/lib/otel/trace-context.js +81 -0
- package/dist/lib/quality/judge-meta-eval.js +70 -0
- package/dist/lib/quality/judge-prompts.js +104 -0
- package/dist/lib/quality/judge-providers.js +193 -0
- package/dist/lib/quality/judge-transport.js +93 -0
- package/dist/lib/quality/llm-judge.js +170 -0
- package/dist/lib/quality/quality-budget.js +99 -0
- package/dist/lib/quality/quality-degradation.js +156 -0
- package/dist/lib/quality/quality-eval-handoff.js +74 -0
- package/dist/lib/quality/quality-evaluator.js +178 -0
- package/dist/lib/quality/quality-sampler.js +43 -0
- package/dist/lib/quality/quality-signals.js +150 -0
- package/dist/lib/runtime/categorizers.js +101 -0
- package/dist/lib/runtime/hook-input.js +16 -0
- package/dist/lib/runtime/hook-output.js +50 -0
- package/dist/lib/runtime/output-analyzer.js +160 -0
- package/dist/lib/runtime/run-handlers.js +32 -0
- package/dist/lib/runtime/tool-input.js +111 -0
- package/dist/lib/runtime/tool-output.js +44 -0
- package/dist/lib/runtime/unknown-tool.js +32 -0
- package/dist/lib/session/edit-ledger.js +29 -0
- package/dist/lib/session/forked-skill.js +39 -0
- package/dist/lib/session/handoffs.js +77 -0
- package/dist/lib/session/session-state.js +169 -0
- package/dist/lib/session/skill-rules.js +196 -0
- package/dist/lib/session/task-tracker.js +83 -0
- package/dist/lib/shipper/shipper-credentials.js +38 -0
- package/dist/lib/shipper/shipper-cursor.js +89 -0
- package/dist/lib/shipper/shipper-status.js +174 -0
- package/dist/lib/shipper/span-shipper.js +272 -0
- package/dist/lib/shipper/telemetry-shipper.js +39 -0
- package/dist/lib/transcript/agent-transcript-metrics.js +34 -0
- package/dist/lib/transcript/context-breakdown.js +134 -0
- package/dist/lib/transcript/context-history.js +69 -0
- package/dist/lib/transcript/context-tracker.js +226 -0
- package/dist/lib/transcript/message-count.js +49 -0
- package/dist/lib/transcript/model-ids.js +36 -0
- package/dist/lib/transcript/model-pricing.js +153 -0
- package/dist/lib/transcript/token-metrics-checkpoint.js +152 -0
- package/dist/lib/transcript/tool-call-durations.js +64 -0
- package/dist/lib/transcript/transcript-entry.js +67 -0
- package/dist/lib/transcript/transcript-fold.js +130 -0
- package/dist/lib/transcript/transcript-layout.js +39 -0
- package/dist/lib/transcript/transcript-parser.js +117 -0
- package/dist/runner/agent-context-4XTCLMLZ.js +23 -0
- package/dist/runner/agent-context-4XTCLMLZ.js.map +7 -0
- package/dist/runner/agent-file-budget-Q33ESBBE.js +68 -0
- package/dist/runner/agent-file-budget-Q33ESBBE.js.map +7 -0
- package/dist/runner/agent-transcript-metrics-W5CZZFMI.js +39 -0
- package/dist/runner/agent-transcript-metrics-W5CZZFMI.js.map +7 -0
- package/dist/runner/chunk-2NTNAJZE.js +49 -0
- package/dist/runner/chunk-2NTNAJZE.js.map +7 -0
- package/dist/runner/chunk-4KWBQTTG.js +78 -0
- package/dist/runner/chunk-4KWBQTTG.js.map +7 -0
- package/dist/runner/chunk-52RY6LQT.js +699 -0
- package/dist/runner/chunk-52RY6LQT.js.map +7 -0
- package/dist/runner/chunk-7OM5CYES.js +8695 -0
- package/dist/runner/chunk-7OM5CYES.js.map +7 -0
- package/dist/runner/chunk-7PEOKGYR.js +1568 -0
- package/dist/runner/chunk-7PEOKGYR.js.map +7 -0
- package/dist/runner/chunk-AGSGKELQ.js +126 -0
- package/dist/runner/chunk-AGSGKELQ.js.map +7 -0
- package/dist/runner/chunk-AVM7PJFW.js +12 -0
- package/dist/runner/chunk-AVM7PJFW.js.map +7 -0
- package/dist/runner/chunk-C3WHUMU5.js +178 -0
- package/dist/runner/chunk-C3WHUMU5.js.map +7 -0
- package/dist/runner/chunk-DGB4NSSD.js +157 -0
- package/dist/runner/chunk-DGB4NSSD.js.map +7 -0
- package/dist/runner/chunk-DNSTWC26.js +151 -0
- package/dist/runner/chunk-DNSTWC26.js.map +7 -0
- package/dist/runner/chunk-FDJJ3TYU.js +449 -0
- package/dist/runner/chunk-FDJJ3TYU.js.map +7 -0
- package/dist/runner/chunk-FGKYDQHZ.js +137 -0
- package/dist/runner/chunk-FGKYDQHZ.js.map +7 -0
- package/dist/runner/chunk-GG2I4TGI.js +582 -0
- package/dist/runner/chunk-GG2I4TGI.js.map +7 -0
- package/dist/runner/chunk-GNNKUMRF.js +60 -0
- package/dist/runner/chunk-GNNKUMRF.js.map +7 -0
- package/dist/runner/chunk-I7TD2QVR.js +118 -0
- package/dist/runner/chunk-I7TD2QVR.js.map +7 -0
- package/dist/runner/chunk-IJ4SLUBV.js +78 -0
- package/dist/runner/chunk-IJ4SLUBV.js.map +7 -0
- package/dist/runner/chunk-IY7AP7NA.js +32 -0
- package/dist/runner/chunk-IY7AP7NA.js.map +7 -0
- package/dist/runner/chunk-J3F3Z3EQ.js +177 -0
- package/dist/runner/chunk-J3F3Z3EQ.js.map +7 -0
- package/dist/runner/chunk-JLEO5OHU.js +104 -0
- package/dist/runner/chunk-JLEO5OHU.js.map +7 -0
- package/dist/runner/chunk-JRNNLOPR.js +168 -0
- package/dist/runner/chunk-JRNNLOPR.js.map +7 -0
- package/dist/runner/chunk-KGSA5JYP.js +82 -0
- package/dist/runner/chunk-KGSA5JYP.js.map +7 -0
- package/dist/runner/chunk-LBCZAYFR.js +125 -0
- package/dist/runner/chunk-LBCZAYFR.js.map +7 -0
- package/dist/runner/chunk-LHWRXY2G.js +41 -0
- package/dist/runner/chunk-LHWRXY2G.js.map +7 -0
- package/dist/runner/chunk-LQ2LGUYJ.js +41 -0
- package/dist/runner/chunk-LQ2LGUYJ.js.map +7 -0
- package/dist/runner/chunk-LYY6RCIE.js +158 -0
- package/dist/runner/chunk-LYY6RCIE.js.map +7 -0
- package/dist/runner/chunk-M4R4QIQH.js +41 -0
- package/dist/runner/chunk-M4R4QIQH.js.map +7 -0
- package/dist/runner/chunk-MC75VXMX.js +207 -0
- package/dist/runner/chunk-MC75VXMX.js.map +7 -0
- package/dist/runner/chunk-NKZU4RRG.js +104 -0
- package/dist/runner/chunk-NKZU4RRG.js.map +7 -0
- package/dist/runner/chunk-PJHWHJ3K.js +56 -0
- package/dist/runner/chunk-PJHWHJ3K.js.map +7 -0
- package/dist/runner/chunk-RIGVYJ2O.js +114 -0
- package/dist/runner/chunk-RIGVYJ2O.js.map +7 -0
- package/dist/runner/chunk-RWRGC7ZK.js +93 -0
- package/dist/runner/chunk-RWRGC7ZK.js.map +7 -0
- package/dist/runner/chunk-SRMAJQMV.js +54 -0
- package/dist/runner/chunk-SRMAJQMV.js.map +7 -0
- package/dist/runner/chunk-TRMFD36J.js +47 -0
- package/dist/runner/chunk-TRMFD36J.js.map +7 -0
- package/dist/runner/chunk-U4IOZYQ3.js +21 -0
- package/dist/runner/chunk-U4IOZYQ3.js.map +7 -0
- package/dist/runner/chunk-UEXDE3WP.js +83 -0
- package/dist/runner/chunk-UEXDE3WP.js.map +7 -0
- package/dist/runner/chunk-UW3GQZSM.js +50 -0
- package/dist/runner/chunk-UW3GQZSM.js.map +7 -0
- package/dist/runner/chunk-V2GSP6VR.js +46 -0
- package/dist/runner/chunk-V2GSP6VR.js.map +7 -0
- package/dist/runner/chunk-VOV4Y5VU.js +81 -0
- package/dist/runner/chunk-VOV4Y5VU.js.map +7 -0
- package/dist/runner/chunk-VT5T6OEI.js +65 -0
- package/dist/runner/chunk-VT5T6OEI.js.map +7 -0
- package/dist/runner/chunk-VUYVFWIN.js +12 -0
- package/dist/runner/chunk-VUYVFWIN.js.map +7 -0
- package/dist/runner/chunk-VVGOPPEN.js +36 -0
- package/dist/runner/chunk-VVGOPPEN.js.map +7 -0
- package/dist/runner/chunk-VXFSLM6P.js +14 -0
- package/dist/runner/chunk-VXFSLM6P.js.map +7 -0
- package/dist/runner/chunk-W3EK7JUT.js +26125 -0
- package/dist/runner/chunk-W3EK7JUT.js.map +7 -0
- package/dist/runner/chunk-WJHNCOUF.js +52 -0
- package/dist/runner/chunk-WJHNCOUF.js.map +7 -0
- package/dist/runner/chunk-WU7ANGCW.js +269 -0
- package/dist/runner/chunk-WU7ANGCW.js.map +7 -0
- package/dist/runner/chunk-X7XLMSZX.js +44 -0
- package/dist/runner/chunk-X7XLMSZX.js.map +7 -0
- package/dist/runner/chunk-XSO3B4QN.js +124 -0
- package/dist/runner/chunk-XSO3B4QN.js.map +7 -0
- package/dist/runner/chunk-YGFN3LN2.js +174 -0
- package/dist/runner/chunk-YGFN3LN2.js.map +7 -0
- package/dist/runner/chunk-YHYDBLVB.js +33 -0
- package/dist/runner/chunk-YHYDBLVB.js.map +7 -0
- package/dist/runner/compact-36NP3PKT.js +60 -0
- package/dist/runner/compact-36NP3PKT.js.map +7 -0
- package/dist/runner/getMachineId-bsd-AGENNHN6.js +42 -0
- package/dist/runner/getMachineId-bsd-AGENNHN6.js.map +7 -0
- package/dist/runner/getMachineId-bsd-I5WZGCL4.js +33 -0
- package/dist/runner/getMachineId-bsd-I5WZGCL4.js.map +7 -0
- package/dist/runner/getMachineId-bsd-ZXK225P7.js +33 -0
- package/dist/runner/getMachineId-bsd-ZXK225P7.js.map +7 -0
- package/dist/runner/getMachineId-darwin-47GDLEXL.js +42 -0
- package/dist/runner/getMachineId-darwin-47GDLEXL.js.map +7 -0
- package/dist/runner/getMachineId-darwin-5CRLMDD6.js +33 -0
- package/dist/runner/getMachineId-darwin-5CRLMDD6.js.map +7 -0
- package/dist/runner/getMachineId-darwin-ADG4I32N.js +33 -0
- package/dist/runner/getMachineId-darwin-ADG4I32N.js.map +7 -0
- package/dist/runner/getMachineId-linux-5KVS3T2M.js +27 -0
- package/dist/runner/getMachineId-linux-5KVS3T2M.js.map +7 -0
- package/dist/runner/getMachineId-linux-GDFB5GSO.js +34 -0
- package/dist/runner/getMachineId-linux-GDFB5GSO.js.map +7 -0
- package/dist/runner/getMachineId-linux-NQMWCRO3.js +27 -0
- package/dist/runner/getMachineId-linux-NQMWCRO3.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-5PFIGMVX.js +18 -0
- package/dist/runner/getMachineId-unsupported-5PFIGMVX.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-DW27K3ZX.js +18 -0
- package/dist/runner/getMachineId-unsupported-DW27K3ZX.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-XV3FHTHD.js +25 -0
- package/dist/runner/getMachineId-unsupported-XV3FHTHD.js.map +7 -0
- package/dist/runner/getMachineId-win-5JVMW3EI.js +44 -0
- package/dist/runner/getMachineId-win-5JVMW3EI.js.map +7 -0
- package/dist/runner/getMachineId-win-SNESQF7R.js +35 -0
- package/dist/runner/getMachineId-win-SNESQF7R.js.map +7 -0
- package/dist/runner/getMachineId-win-UJYHSBM2.js +35 -0
- package/dist/runner/getMachineId-win-UJYHSBM2.js.map +7 -0
- package/dist/runner/git-commit-BA53DOJQ.js +30 -0
- package/dist/runner/git-commit-BA53DOJQ.js.map +7 -0
- package/dist/runner/hook-runner-JHH6PGKE.js +220 -0
- package/dist/runner/hook-runner-JHH6PGKE.js.map +7 -0
- package/dist/runner/judge-transport-R4TF5LEH.js +91 -0
- package/dist/runner/judge-transport-R4TF5LEH.js.map +7 -0
- package/dist/runner/mcp-status-QEUH2KGX.js +89 -0
- package/dist/runner/mcp-status-QEUH2KGX.js.map +7 -0
- package/dist/runner/notification-FKMBCSXF.js +69 -0
- package/dist/runner/notification-FKMBCSXF.js.map +7 -0
- package/dist/runner/post-tool-3MRHTBKH.js +192 -0
- package/dist/runner/post-tool-3MRHTBKH.js.map +7 -0
- package/dist/runner/post-tool-agent-BB34MNHI.js +370 -0
- package/dist/runner/post-tool-agent-BB34MNHI.js.map +7 -0
- package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js +147 -0
- package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js.map +7 -0
- package/dist/runner/post-tool-code-structure-LE7J2SC2.js +130 -0
- package/dist/runner/post-tool-code-structure-LE7J2SC2.js.map +7 -0
- package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js +30 -0
- package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js.map +7 -0
- package/dist/runner/post-tool-failure-XTH2I2Z7.js +111 -0
- package/dist/runner/post-tool-failure-XTH2I2Z7.js.map +7 -0
- package/dist/runner/post-tool-git-E46HZGZO.js +94 -0
- package/dist/runner/post-tool-git-E46HZGZO.js.map +7 -0
- package/dist/runner/post-tool-mcp-AMCDY5GH.js +144 -0
- package/dist/runner/post-tool-mcp-AMCDY5GH.js.map +7 -0
- package/dist/runner/post-tool-task-state-TLDOI33H.js +100 -0
- package/dist/runner/post-tool-task-state-TLDOI33H.js.map +7 -0
- package/dist/runner/pre-tool-SQE2FZQO.js +245 -0
- package/dist/runner/pre-tool-SQE2FZQO.js.map +7 -0
- package/dist/runner/pre-tool-agent-OZUPVFUI.js +218 -0
- package/dist/runner/pre-tool-agent-OZUPVFUI.js.map +7 -0
- package/dist/runner/quality-evaluator-XYNDKDRG.js +901 -0
- package/dist/runner/quality-evaluator-XYNDKDRG.js.map +7 -0
- package/dist/runner/session-end-3MJTDLLA.js +57 -0
- package/dist/runner/session-end-3MJTDLLA.js.map +7 -0
- package/dist/runner/session-start-5ZMCLKBS.js +352 -0
- package/dist/runner/session-start-5ZMCLKBS.js.map +7 -0
- package/dist/runner/shipper-status-MADX6SPG.js +37 -0
- package/dist/runner/shipper-status-MADX6SPG.js.map +7 -0
- package/dist/runner/span-shipper-4HQDHC47.js +3408 -0
- package/dist/runner/span-shipper-4HQDHC47.js.map +7 -0
- package/dist/runner/stop-RTKSIYJV.js +596 -0
- package/dist/runner/stop-RTKSIYJV.js.map +7 -0
- package/dist/runner/subagent-CBCG4OLZ.js +116 -0
- package/dist/runner/subagent-CBCG4OLZ.js.map +7 -0
- package/dist/runner/token-metrics-AD2RAFGM.js +31 -0
- package/dist/runner/token-metrics-AD2RAFGM.js.map +7 -0
- package/dist/runner/user-prompt-T5NZ2UJD.js +376 -0
- package/dist/runner/user-prompt-T5NZ2UJD.js.map +7 -0
- package/package.json +70 -5
|
@@ -0,0 +1,901 @@
|
|
|
1
|
+
import { createRequire as __hookRunnerCreateRequire } from 'node:module';
|
|
2
|
+
const require = __hookRunnerCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
COST_PER_METRIC_CENTS,
|
|
5
|
+
QUALITY_ATTRS,
|
|
6
|
+
QUALITY_EVALUATION_HOOK_TYPE,
|
|
7
|
+
QUALITY_EVAL_SKIP_REASONS,
|
|
8
|
+
QUALITY_EVAL_TYPE_LLM,
|
|
9
|
+
getBudgetUsage,
|
|
10
|
+
parseEvaluationArgs,
|
|
11
|
+
releaseBudget,
|
|
12
|
+
trueUpBudget,
|
|
13
|
+
tryReserveBudget
|
|
14
|
+
} from "./chunk-J3F3Z3EQ.js";
|
|
15
|
+
import {
|
|
16
|
+
spawnTelemetryShipper
|
|
17
|
+
} from "./chunk-UEXDE3WP.js";
|
|
18
|
+
import "./chunk-JRNNLOPR.js";
|
|
19
|
+
import {
|
|
20
|
+
recordInferenceCost,
|
|
21
|
+
recordTokenUsage
|
|
22
|
+
} from "./chunk-FGKYDQHZ.js";
|
|
23
|
+
import "./chunk-VT5T6OEI.js";
|
|
24
|
+
import {
|
|
25
|
+
EVALUATION_ATTRS,
|
|
26
|
+
EVALUATION_COHORT,
|
|
27
|
+
EVALUATOR_KIND,
|
|
28
|
+
QUALITY_METRIC_NAMES,
|
|
29
|
+
appendEvaluation,
|
|
30
|
+
emitEvaluationLatency
|
|
31
|
+
} from "./chunk-MC75VXMX.js";
|
|
32
|
+
import "./chunk-LYY6RCIE.js";
|
|
33
|
+
import {
|
|
34
|
+
SAMPLING_CONFIG
|
|
35
|
+
} from "./chunk-PJHWHJ3K.js";
|
|
36
|
+
import {
|
|
37
|
+
usageCostUsd
|
|
38
|
+
} from "./chunk-XSO3B4QN.js";
|
|
39
|
+
import {
|
|
40
|
+
providerOfModel
|
|
41
|
+
} from "./chunk-X7XLMSZX.js";
|
|
42
|
+
import {
|
|
43
|
+
classifyEntry,
|
|
44
|
+
isFailedRequest,
|
|
45
|
+
mayBeConversational,
|
|
46
|
+
toBlocks
|
|
47
|
+
} from "./chunk-4KWBQTTG.js";
|
|
48
|
+
import "./chunk-YGFN3LN2.js";
|
|
49
|
+
import {
|
|
50
|
+
HTTP_STATUS,
|
|
51
|
+
bearerHeaders,
|
|
52
|
+
errorCode,
|
|
53
|
+
isHttpTimeoutError,
|
|
54
|
+
jsonHeaders
|
|
55
|
+
} from "./chunk-IJ4SLUBV.js";
|
|
56
|
+
import {
|
|
57
|
+
instrumentHook,
|
|
58
|
+
setSessionId
|
|
59
|
+
} from "./chunk-WU7ANGCW.js";
|
|
60
|
+
import {
|
|
61
|
+
ERROR_TYPE_ATTR,
|
|
62
|
+
GENAI_AGENT_ATTRIBUTES,
|
|
63
|
+
GENAI_CLIENT_INFERENCE_DURATION_METRIC,
|
|
64
|
+
GENAI_CLIENT_OPERATION_SPAN,
|
|
65
|
+
GENAI_DURATION_BUCKETS_S,
|
|
66
|
+
GENAI_OPERATION,
|
|
67
|
+
GENAI_PROVIDER_NAMES,
|
|
68
|
+
GENAI_REQUEST_MODEL_ATTR,
|
|
69
|
+
HOOK_NAMES,
|
|
70
|
+
HOOK_TRIGGER_ATTR,
|
|
71
|
+
HOOK_TYPE_ATTR,
|
|
72
|
+
SESSION_ATTRIBUTES,
|
|
73
|
+
addEvent,
|
|
74
|
+
recordCounter,
|
|
75
|
+
recordMetric,
|
|
76
|
+
shutdownTelemetry,
|
|
77
|
+
withSpan
|
|
78
|
+
} from "./chunk-7OM5CYES.js";
|
|
79
|
+
import "./chunk-GG2I4TGI.js";
|
|
80
|
+
import "./chunk-SRMAJQMV.js";
|
|
81
|
+
import "./chunk-YHYDBLVB.js";
|
|
82
|
+
import {
|
|
83
|
+
qualityStateDir
|
|
84
|
+
} from "./chunk-DNSTWC26.js";
|
|
85
|
+
import {
|
|
86
|
+
HOOK_EVENT_NAMES,
|
|
87
|
+
errorTypeName,
|
|
88
|
+
isJsonObject,
|
|
89
|
+
loadJsonObject,
|
|
90
|
+
parseJsonLines,
|
|
91
|
+
readFileSlice,
|
|
92
|
+
truncate,
|
|
93
|
+
writeJsonAtomic
|
|
94
|
+
} from "./chunk-52RY6LQT.js";
|
|
95
|
+
import {
|
|
96
|
+
DURATION_MS,
|
|
97
|
+
TIME_MS,
|
|
98
|
+
UNITS,
|
|
99
|
+
isByteCount
|
|
100
|
+
} from "./chunk-RWRGC7ZK.js";
|
|
101
|
+
import "./chunk-7PEOKGYR.js";
|
|
102
|
+
import "./chunk-GNNKUMRF.js";
|
|
103
|
+
|
|
104
|
+
// lib/transcript/transcript-parser.ts
|
|
105
|
+
import { createHash } from "node:crypto";
|
|
106
|
+
var MAX_TEXT_LENGTH = 8e3;
|
|
107
|
+
var MAX_TOOL_RESULTS_PER_TURN = 20;
|
|
108
|
+
var MAX_TOOL_RESULT_LENGTH = 500;
|
|
109
|
+
function extractTextContent(content) {
|
|
110
|
+
if (typeof content === "string") return content;
|
|
111
|
+
return toBlocks(content).filter((block) => typeof block.text === "string").map((block) => block.text).join("\n");
|
|
112
|
+
}
|
|
113
|
+
function extractToolResults(content, limit) {
|
|
114
|
+
return toBlocks(content).filter((block) => block.type === "tool_result").slice(0, limit).map((block) => truncate(extractTextContent(block.content), MAX_TOOL_RESULT_LENGTH));
|
|
115
|
+
}
|
|
116
|
+
function newDraft(userText) {
|
|
117
|
+
return { userText, assistantText: "", toolResults: [] };
|
|
118
|
+
}
|
|
119
|
+
function appendAssistantText(draft, text) {
|
|
120
|
+
if (!text || draft.assistantText.length > MAX_TEXT_LENGTH) return;
|
|
121
|
+
draft.assistantText = draft.assistantText ? `${draft.assistantText}
|
|
122
|
+
${text}` : text;
|
|
123
|
+
}
|
|
124
|
+
function toTurn(draft) {
|
|
125
|
+
if (!draft.userText || !draft.assistantText) return null;
|
|
126
|
+
return {
|
|
127
|
+
...draft,
|
|
128
|
+
userText: truncate(draft.userText, MAX_TEXT_LENGTH),
|
|
129
|
+
assistantText: truncate(draft.assistantText, MAX_TEXT_LENGTH)
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
function readTranscript(transcriptPath, maxBytes) {
|
|
133
|
+
try {
|
|
134
|
+
return readFileSlice(transcriptPath, maxBytes);
|
|
135
|
+
} catch {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
function extractTurns(transcriptPath, maxBytes) {
|
|
140
|
+
if (!isByteCount(maxBytes)) {
|
|
141
|
+
throw new RangeError(`extractTurns: maxBytes must be a non-negative integer, got ${String(maxBytes)}`);
|
|
142
|
+
}
|
|
143
|
+
const content = readTranscript(transcriptPath, maxBytes);
|
|
144
|
+
if (content === null) return [];
|
|
145
|
+
const turns = [];
|
|
146
|
+
let draft = newDraft("");
|
|
147
|
+
const emit = () => {
|
|
148
|
+
const turn = toTurn(draft);
|
|
149
|
+
if (turn) turns.push(turn);
|
|
150
|
+
};
|
|
151
|
+
for (const entry of parseJsonLines(content, mayBeConversational)) {
|
|
152
|
+
const rawContent = entry.message?.content;
|
|
153
|
+
const kind = classifyEntry(entry);
|
|
154
|
+
if (kind === "prompt") {
|
|
155
|
+
const remainingToolSlots = MAX_TOOL_RESULTS_PER_TURN - draft.toolResults.length;
|
|
156
|
+
draft.toolResults.push(...extractToolResults(rawContent, remainingToolSlots));
|
|
157
|
+
const text = extractTextContent(rawContent);
|
|
158
|
+
if (text.trim()) {
|
|
159
|
+
emit();
|
|
160
|
+
draft = newDraft(text);
|
|
161
|
+
}
|
|
162
|
+
} else if (isFailedRequest(entry)) {
|
|
163
|
+
draft.cutOffBy = "api_error";
|
|
164
|
+
} else if (kind === "response") {
|
|
165
|
+
appendAssistantText(draft, extractTextContent(rawContent));
|
|
166
|
+
if (entry.message?.id) draft.responseId = entry.message.id;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
emit();
|
|
170
|
+
return turns;
|
|
171
|
+
}
|
|
172
|
+
function sampleTurns(turns, maxTurns) {
|
|
173
|
+
const eligible = turns.flatMap((turn, index) => turn.cutOffBy ? [] : [{ ...turn, index }]);
|
|
174
|
+
if (eligible.length <= maxTurns) return eligible;
|
|
175
|
+
for (let i = 0; i < maxTurns; i++) {
|
|
176
|
+
const j = i + Math.floor(Math.random() * (eligible.length - i));
|
|
177
|
+
const [picked, displaced] = [eligible[j], eligible[i]];
|
|
178
|
+
if (picked !== void 0 && displaced !== void 0) [eligible[i], eligible[j]] = [picked, displaced];
|
|
179
|
+
}
|
|
180
|
+
return eligible.slice(0, maxTurns).sort((a, b) => a.index - b.index);
|
|
181
|
+
}
|
|
182
|
+
var TURN_USER_TEXT_HASH_ALGORITHM = "sha256";
|
|
183
|
+
function hashTurnUserText(userText) {
|
|
184
|
+
return createHash(TURN_USER_TEXT_HASH_ALGORITHM).update(userText).digest("hex");
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// lib/quality/judge-providers.ts
|
|
188
|
+
var JUDGE_MAX_TOKENS = 384;
|
|
189
|
+
var JUDGE_TIMEOUT_MS = DURATION_MS.FIFTEEN_SECONDS;
|
|
190
|
+
var REASONING_MODEL_TIMEOUT_MS = TIME_MS.MINUTE;
|
|
191
|
+
var XAI_TICKS_PER_USD = 1e10;
|
|
192
|
+
var ANTHROPIC_MESSAGES_URL = "https://api.anthropic.com/v1/messages";
|
|
193
|
+
var ANTHROPIC_VERSION = "2023-06-01";
|
|
194
|
+
var XAI_CHAT_COMPLETIONS_URL = "https://api.x.ai/v1/chat/completions";
|
|
195
|
+
var OPENAI_CHAT_COMPLETIONS_URL = "https://api.openai.com/v1/chat/completions";
|
|
196
|
+
var OPENAI_KEY_PREFIX = "sk-";
|
|
197
|
+
var RESPONSE_SHAPES = {
|
|
198
|
+
ANTHROPIC: "Anthropic",
|
|
199
|
+
OPENAI_COMPATIBLE: "OpenAI-compatible"
|
|
200
|
+
};
|
|
201
|
+
var JudgeResponseShapeError = class extends Error {
|
|
202
|
+
constructor(shape) {
|
|
203
|
+
super(`judge response does not match the ${shape} shape`);
|
|
204
|
+
this.name = "JudgeResponseShapeError";
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
function isNumber(value) {
|
|
208
|
+
return typeof value === "number";
|
|
209
|
+
}
|
|
210
|
+
function isString(value) {
|
|
211
|
+
return typeof value === "string";
|
|
212
|
+
}
|
|
213
|
+
function isAbsentOr(value, check) {
|
|
214
|
+
return value === void 0 || value === null || check(value);
|
|
215
|
+
}
|
|
216
|
+
function isAnthropicContent(value) {
|
|
217
|
+
if (!Array.isArray(value)) return false;
|
|
218
|
+
const first = value[0];
|
|
219
|
+
return isAbsentOr(first, (block) => isJsonObject(block) && isAbsentOr(block.text, isString));
|
|
220
|
+
}
|
|
221
|
+
function isAnthropicUsage(value) {
|
|
222
|
+
return isJsonObject(value) && isNumber(value.input_tokens) && isNumber(value.output_tokens);
|
|
223
|
+
}
|
|
224
|
+
function isAnthropicResponse(data) {
|
|
225
|
+
return isJsonObject(data) && isAbsentOr(data.content, isAnthropicContent) && isAbsentOr(data.usage, isAnthropicUsage);
|
|
226
|
+
}
|
|
227
|
+
function asAnthropicResponse(data) {
|
|
228
|
+
if (!isAnthropicResponse(data)) throw new JudgeResponseShapeError(RESPONSE_SHAPES.ANTHROPIC);
|
|
229
|
+
return data;
|
|
230
|
+
}
|
|
231
|
+
function isOpenAiMessage(value) {
|
|
232
|
+
return isJsonObject(value) && isAbsentOr(value.content, isString);
|
|
233
|
+
}
|
|
234
|
+
function isOpenAiChoices(value) {
|
|
235
|
+
if (!Array.isArray(value)) return false;
|
|
236
|
+
const first = value[0];
|
|
237
|
+
return isAbsentOr(first, (choice) => isJsonObject(choice) && isAbsentOr(choice.message, isOpenAiMessage));
|
|
238
|
+
}
|
|
239
|
+
function isOpenAiUsage(value) {
|
|
240
|
+
if (!isJsonObject(value)) return false;
|
|
241
|
+
const isDetails = (details) => isJsonObject(details) && isAbsentOr(details.reasoning_tokens, isNumber);
|
|
242
|
+
return isNumber(value.prompt_tokens) && isNumber(value.completion_tokens) && isAbsentOr(value.completion_tokens_details, isDetails);
|
|
243
|
+
}
|
|
244
|
+
function isOpenAiCompatibleResponse(data) {
|
|
245
|
+
return isJsonObject(data) && isAbsentOr(data.choices, isOpenAiChoices) && isAbsentOr(data.usage, isOpenAiUsage) && isAbsentOr(data.cost_in_usd_ticks, isNumber);
|
|
246
|
+
}
|
|
247
|
+
function asOpenAiCompatibleResponse(data) {
|
|
248
|
+
if (!isOpenAiCompatibleResponse(data)) throw new JudgeResponseShapeError(RESPONSE_SHAPES.OPENAI_COMPATIBLE);
|
|
249
|
+
return data;
|
|
250
|
+
}
|
|
251
|
+
function parseOpenAiCompatible(payload) {
|
|
252
|
+
return {
|
|
253
|
+
text: payload.choices?.[0]?.message?.content ?? "",
|
|
254
|
+
inputTokens: payload.usage?.prompt_tokens,
|
|
255
|
+
outputTokens: payload.usage?.completion_tokens,
|
|
256
|
+
// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 0 reasoning tokens maps to undefined
|
|
257
|
+
reasoningTokens: payload.usage?.completion_tokens_details?.reasoning_tokens || void 0
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
function promptBody(model, prompt) {
|
|
261
|
+
return { model, temperature: 0, messages: [{ role: "user", content: prompt }] };
|
|
262
|
+
}
|
|
263
|
+
function maxTokensBody(model, prompt) {
|
|
264
|
+
return { ...promptBody(model, prompt), max_tokens: JUDGE_MAX_TOKENS };
|
|
265
|
+
}
|
|
266
|
+
function maxCompletionTokensBody(model, prompt) {
|
|
267
|
+
return { ...promptBody(model, prompt), max_completion_tokens: JUDGE_MAX_TOKENS, reasoning_effort: "none" };
|
|
268
|
+
}
|
|
269
|
+
var PROVIDER_SPECS = {
|
|
270
|
+
[GENAI_PROVIDER_NAMES.ANTHROPIC]: {
|
|
271
|
+
url: ANTHROPIC_MESSAGES_URL,
|
|
272
|
+
timeoutMs: JUDGE_TIMEOUT_MS,
|
|
273
|
+
// Not ANTHROPIC_API_KEY first: Claude Code reads it and drops subscription auth when set.
|
|
274
|
+
// The backup covers a revoked judge key (JUDGE-KEY-401).
|
|
275
|
+
keyEnvVars: ["LLM_JUDGE_ANTHROPIC_KEY", "ANTHROPIC_API_KEY"],
|
|
276
|
+
body: maxTokensBody,
|
|
277
|
+
headers: (apiKey) => jsonHeaders({ "x-api-key": apiKey, "anthropic-version": ANTHROPIC_VERSION }),
|
|
278
|
+
parse: (data) => {
|
|
279
|
+
const payload = asAnthropicResponse(data);
|
|
280
|
+
return {
|
|
281
|
+
text: payload.content?.[0]?.text ?? "",
|
|
282
|
+
inputTokens: payload.usage?.input_tokens,
|
|
283
|
+
outputTokens: payload.usage?.output_tokens
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
[GENAI_PROVIDER_NAMES.XAI]: {
|
|
288
|
+
url: XAI_CHAT_COMPLETIONS_URL,
|
|
289
|
+
timeoutMs: REASONING_MODEL_TIMEOUT_MS,
|
|
290
|
+
keyEnvVars: ["LLM_JUDGE_XAI_KEY", "XAI_API_KEY"],
|
|
291
|
+
body: maxTokensBody,
|
|
292
|
+
headers: (apiKey) => jsonHeaders(bearerHeaders(apiKey)),
|
|
293
|
+
parse: (data) => {
|
|
294
|
+
const payload = asOpenAiCompatibleResponse(data);
|
|
295
|
+
const base = parseOpenAiCompatible(payload);
|
|
296
|
+
const ticks = payload.cost_in_usd_ticks ?? void 0;
|
|
297
|
+
return {
|
|
298
|
+
...base,
|
|
299
|
+
// xAI's `completion_tokens` EXCLUDES reasoning (~18× under-report alone, Gate 2b).
|
|
300
|
+
outputTokens: base.outputTokens === void 0 ? void 0 : base.outputTokens + (base.reasoningTokens ?? 0),
|
|
301
|
+
actualCostUsd: ticks === void 0 ? void 0 : ticks / XAI_TICKS_PER_USD
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
},
|
|
305
|
+
[GENAI_PROVIDER_NAMES.OPENAI]: {
|
|
306
|
+
url: OPENAI_CHAT_COMPLETIONS_URL,
|
|
307
|
+
timeoutMs: REASONING_MODEL_TIMEOUT_MS,
|
|
308
|
+
// OPEN_AI_KEY first: OPENAI_API_KEY is a revoked key in Doppler.
|
|
309
|
+
keyEnvVars: ["LLM_JUDGE_OPENAI_KEY", "OPEN_AI_KEY", "OPENAI_API_KEY"],
|
|
310
|
+
keyPrefix: OPENAI_KEY_PREFIX,
|
|
311
|
+
body: maxCompletionTokensBody,
|
|
312
|
+
headers: (apiKey) => jsonHeaders(bearerHeaders(apiKey)),
|
|
313
|
+
// OpenAI's `completion_tokens` already INCLUDES reasoning; no per-response cost.
|
|
314
|
+
parse: (data) => parseOpenAiCompatible(asOpenAiCompatibleResponse(data))
|
|
315
|
+
}
|
|
316
|
+
};
|
|
317
|
+
var JUDGE_PROVIDERS = new Set(Object.values(GENAI_PROVIDER_NAMES));
|
|
318
|
+
function isJudgeProvider(provider) {
|
|
319
|
+
return provider !== void 0 && JUDGE_PROVIDERS.has(provider);
|
|
320
|
+
}
|
|
321
|
+
function providerForModel(model) {
|
|
322
|
+
const provider = providerOfModel(model);
|
|
323
|
+
return isJudgeProvider(provider) ? provider : void 0;
|
|
324
|
+
}
|
|
325
|
+
function* keySources(spec) {
|
|
326
|
+
for (const name of spec.keyEnvVars) yield process.env[name];
|
|
327
|
+
}
|
|
328
|
+
function* judgeApiKeys(provider) {
|
|
329
|
+
const spec = PROVIDER_SPECS[provider];
|
|
330
|
+
const seen = /* @__PURE__ */ new Set();
|
|
331
|
+
for (const value of keySources(spec)) {
|
|
332
|
+
if (!value || seen.has(value)) continue;
|
|
333
|
+
if (spec.keyPrefix && !value.startsWith(spec.keyPrefix)) continue;
|
|
334
|
+
seen.add(value);
|
|
335
|
+
yield value;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
function firstJudgeApiKey(provider) {
|
|
339
|
+
const first = judgeApiKeys(provider).next();
|
|
340
|
+
return first.done ? void 0 : first.value;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// lib/quality/judge-prompts.ts
|
|
344
|
+
var JUDGE_METRIC_NAMES = {
|
|
345
|
+
RELEVANCE: "relevance",
|
|
346
|
+
COHERENCE: "coherence"
|
|
347
|
+
};
|
|
348
|
+
var MAX_USER_TEXT_LEN = 500;
|
|
349
|
+
var MAX_ASSISTANT_TEXT_LEN = 2e3;
|
|
350
|
+
var MAX_REASON_LEN = 400;
|
|
351
|
+
var MAX_TOOL_RESULTS_IN_PROMPT = 3;
|
|
352
|
+
var JUDGE_SCORE_PATTERN = /Score:\s*(\d+(?:\.\d+)?)/i;
|
|
353
|
+
var JUDGE_EXPLANATION_PATTERN = /Explanation:\s*(.+)/i;
|
|
354
|
+
var SCORE_PARSE_FAILED = "llm_judge_score_parse_failed";
|
|
355
|
+
function parseJudgeVerdict(text) {
|
|
356
|
+
const scoreText = JUDGE_SCORE_PATTERN.exec(text)?.[1];
|
|
357
|
+
if (scoreText === void 0) throw new Error(SCORE_PARSE_FAILED);
|
|
358
|
+
const score = Math.max(0, Math.min(1, parseFloat(scoreText)));
|
|
359
|
+
const reason = JUDGE_EXPLANATION_PATTERN.exec(text)?.[1]?.trim() || text.trim();
|
|
360
|
+
return { score, reason };
|
|
361
|
+
}
|
|
362
|
+
function buildJudgePrompt(metricName, turn) {
|
|
363
|
+
return metricName === JUDGE_METRIC_NAMES.RELEVANCE ? buildRelevancePrompt(turn) : buildCoherencePrompt(turn);
|
|
364
|
+
}
|
|
365
|
+
function buildExplanationQualityPrompt(evaluationName, score, reason, userText) {
|
|
366
|
+
const safeUser = truncate(userText, MAX_USER_TEXT_LEN);
|
|
367
|
+
const safeReason = truncate(reason, MAX_REASON_LEN);
|
|
368
|
+
return `You are evaluating the quality of a judge's explanation for an evaluation score.
|
|
369
|
+
|
|
370
|
+
The judge evaluated "${evaluationName}" and gave a score of ${score.toFixed(2)}.
|
|
371
|
+
|
|
372
|
+
Original user request:
|
|
373
|
+
${safeUser}
|
|
374
|
+
|
|
375
|
+
Judge's explanation:
|
|
376
|
+
${safeReason}
|
|
377
|
+
|
|
378
|
+
Rate the explanation quality on a scale from 0.0 to 1.0 based on:
|
|
379
|
+
1. Specificity: Does it reference concrete details from the evaluated output?
|
|
380
|
+
2. Evidence citation: Does it quote or paraphrase specific passages as evidence?
|
|
381
|
+
3. Actionability: Does it suggest what could be improved?
|
|
382
|
+
|
|
383
|
+
- 1.0: Highly specific, cites evidence, actionable
|
|
384
|
+
- 0.7: Specific with some evidence, partially actionable
|
|
385
|
+
- 0.5: General but relevant reasoning
|
|
386
|
+
- 0.3: Vague or generic explanation
|
|
387
|
+
- 0.0: No meaningful reasoning provided
|
|
388
|
+
|
|
389
|
+
A concise but specific explanation can score highly.
|
|
390
|
+
|
|
391
|
+
Respond in exactly this format:
|
|
392
|
+
Score: X.XX
|
|
393
|
+
Explanation: [1-2 sentences about the quality of the explanation]`;
|
|
394
|
+
}
|
|
395
|
+
function buildRelevancePrompt(turn) {
|
|
396
|
+
const safeUser = truncate(turn.userText, MAX_USER_TEXT_LEN);
|
|
397
|
+
const safeAssistant = truncate(turn.assistantText, MAX_ASSISTANT_TEXT_LEN);
|
|
398
|
+
const toolContext = turn.toolResults.length > 0 ? `
|
|
399
|
+
Tool results used:
|
|
400
|
+
${turn.toolResults.slice(0, MAX_TOOL_RESULTS_IN_PROMPT).join("\n---\n")}` : "";
|
|
401
|
+
return `You are evaluating the relevance of an AI assistant's response to a user's request.
|
|
402
|
+
|
|
403
|
+
User request:
|
|
404
|
+
${safeUser}
|
|
405
|
+
|
|
406
|
+
Assistant response:
|
|
407
|
+
${safeAssistant}${toolContext}
|
|
408
|
+
|
|
409
|
+
Rate the relevance of the response on a scale from 0.0 to 1.0:
|
|
410
|
+
- 1.0: Perfectly relevant, directly addresses the user's request
|
|
411
|
+
- 0.7: Mostly relevant with minor tangents
|
|
412
|
+
- 0.5: Partially relevant
|
|
413
|
+
- 0.3: Mostly irrelevant
|
|
414
|
+
- 0.0: Completely irrelevant
|
|
415
|
+
|
|
416
|
+
Respond in exactly this format:
|
|
417
|
+
Score: X.XX
|
|
418
|
+
Explanation: [1-2 sentences citing specific evidence from the response]`;
|
|
419
|
+
}
|
|
420
|
+
function buildCoherencePrompt(turn) {
|
|
421
|
+
const safeAssistant = truncate(turn.assistantText, MAX_ASSISTANT_TEXT_LEN);
|
|
422
|
+
return `You are evaluating the coherence of an AI assistant's response.
|
|
423
|
+
|
|
424
|
+
Assistant response:
|
|
425
|
+
${safeAssistant}
|
|
426
|
+
|
|
427
|
+
Rate the coherence on a scale from 0.0 to 1.0:
|
|
428
|
+
- 1.0: Well-structured, clear, logically organized
|
|
429
|
+
- 0.7: Mostly coherent with minor issues
|
|
430
|
+
- 0.5: Somewhat coherent but hard to follow
|
|
431
|
+
- 0.3: Mostly incoherent
|
|
432
|
+
- 0.0: Completely incoherent
|
|
433
|
+
|
|
434
|
+
Respond in exactly this format:
|
|
435
|
+
Score: X.XX
|
|
436
|
+
Explanation: [1-2 sentences citing specific evidence from the response]`;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// lib/quality/llm-judge.ts
|
|
440
|
+
var DEFAULT_JUDGE_MODEL = "claude-haiku-4-5-20251001";
|
|
441
|
+
var JUDGE_MODEL = process.env.QUALITY_JUDGE_MODEL || DEFAULT_JUDGE_MODEL;
|
|
442
|
+
var CANARY_SCORE_MIN = 0.2;
|
|
443
|
+
var CANARY_SCORE_SPAN = 0.3;
|
|
444
|
+
var CANARY_REASON = "canary";
|
|
445
|
+
var CENTS_PER_USD = 100;
|
|
446
|
+
var JUDGE_RETRY_METRIC = "integritystudio.llm_judge.retries";
|
|
447
|
+
var RETRY_REASON_ATTR = "integritystudio.llm_judge.retry_reason";
|
|
448
|
+
var REASONING_OUTPUT_TOKENS_ATTR = "gen_ai.usage.reasoning.output_tokens";
|
|
449
|
+
var MissingJudgeCredentialError = class extends Error {
|
|
450
|
+
constructor(provider) {
|
|
451
|
+
super(`no api key for judge provider ${provider}`);
|
|
452
|
+
this.provider = provider;
|
|
453
|
+
this.name = "MissingJudgeCredentialError";
|
|
454
|
+
}
|
|
455
|
+
provider;
|
|
456
|
+
};
|
|
457
|
+
var UnknownJudgeModelError = class extends Error {
|
|
458
|
+
constructor(model) {
|
|
459
|
+
super(`no judge provider claims model ${model}`);
|
|
460
|
+
this.model = model;
|
|
461
|
+
this.name = "UnknownJudgeModelError";
|
|
462
|
+
}
|
|
463
|
+
model;
|
|
464
|
+
};
|
|
465
|
+
var JudgeHttpError = class extends Error {
|
|
466
|
+
constructor(provider, status) {
|
|
467
|
+
super(`${provider} API error: ${status}`);
|
|
468
|
+
this.status = status;
|
|
469
|
+
this.name = "JudgeHttpError";
|
|
470
|
+
}
|
|
471
|
+
status;
|
|
472
|
+
};
|
|
473
|
+
function recordRetry(provider, reason) {
|
|
474
|
+
recordMetric(JUDGE_RETRY_METRIC, 1, {
|
|
475
|
+
[GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider,
|
|
476
|
+
[RETRY_REASON_ATTR]: reason
|
|
477
|
+
}, { unit: UNITS.RETRY });
|
|
478
|
+
}
|
|
479
|
+
function judgeErrorType(error) {
|
|
480
|
+
if (error instanceof JudgeHttpError) return `http_error_${error.status}`;
|
|
481
|
+
if (!(error instanceof Error)) return "unknown";
|
|
482
|
+
if (isHttpTimeoutError(error)) return "timeout";
|
|
483
|
+
return errorCode(error) ?? errorTypeName(error);
|
|
484
|
+
}
|
|
485
|
+
function elapsedSeconds(startMs) {
|
|
486
|
+
return (Date.now() - startMs) / TIME_MS.SECOND;
|
|
487
|
+
}
|
|
488
|
+
function judgeCostUsd(model, usage) {
|
|
489
|
+
if (usage.actualCostUsd !== void 0) return usage.actualCostUsd;
|
|
490
|
+
if (usage.inputTokens === void 0 || usage.outputTokens === void 0) return void 0;
|
|
491
|
+
return usageCostUsd(model, { input: usage.inputTokens, output: usage.outputTokens });
|
|
492
|
+
}
|
|
493
|
+
function recordJudgeUsage(span, model, provider, usage) {
|
|
494
|
+
const { inputTokens, outputTokens, reasoningTokens } = usage;
|
|
495
|
+
const costUsd = judgeCostUsd(model, usage);
|
|
496
|
+
const labels = {
|
|
497
|
+
model,
|
|
498
|
+
operation: GENAI_OPERATION.EVALUATE,
|
|
499
|
+
attributes: { [GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider }
|
|
500
|
+
};
|
|
501
|
+
if (inputTokens !== void 0 && outputTokens !== void 0) {
|
|
502
|
+
recordTokenUsage({ ...labels, inputTokens, outputTokens, reasoningTokens });
|
|
503
|
+
}
|
|
504
|
+
if (costUsd !== void 0) recordInferenceCost(labels, costUsd);
|
|
505
|
+
if (reasoningTokens !== void 0) span.setAttribute(REASONING_OUTPUT_TOKENS_ATTR, reasoningTokens);
|
|
506
|
+
return costUsd;
|
|
507
|
+
}
|
|
508
|
+
async function postWithKeyFallback(post, primaryKey, backupKeys) {
|
|
509
|
+
let response = await post(primaryKey);
|
|
510
|
+
while (response.status === HTTP_STATUS.UNAUTHORIZED) {
|
|
511
|
+
const backup = backupKeys.next();
|
|
512
|
+
if (backup.done) break;
|
|
513
|
+
await response.discard();
|
|
514
|
+
response = await post(backup.value);
|
|
515
|
+
}
|
|
516
|
+
return response;
|
|
517
|
+
}
|
|
518
|
+
async function callJudge(prompt, canary, model) {
|
|
519
|
+
if (canary) return { score: CANARY_SCORE_MIN + Math.random() * CANARY_SCORE_SPAN, reason: CANARY_REASON };
|
|
520
|
+
const resolvedModel = model ?? JUDGE_MODEL;
|
|
521
|
+
const provider = providerForModel(resolvedModel);
|
|
522
|
+
if (!provider) throw new UnknownJudgeModelError(resolvedModel);
|
|
523
|
+
const spec = PROVIDER_SPECS[provider];
|
|
524
|
+
const keys = judgeApiKeys(provider);
|
|
525
|
+
const primary = keys.next();
|
|
526
|
+
if (primary.done) throw new MissingJudgeCredentialError(provider);
|
|
527
|
+
const genAiAttributes = {
|
|
528
|
+
[GENAI_AGENT_ATTRIBUTES.OPERATION_NAME]: GENAI_OPERATION.CHAT,
|
|
529
|
+
[GENAI_REQUEST_MODEL_ATTR]: resolvedModel,
|
|
530
|
+
[GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider
|
|
531
|
+
};
|
|
532
|
+
return withSpan(
|
|
533
|
+
GENAI_CLIENT_OPERATION_SPAN,
|
|
534
|
+
genAiAttributes,
|
|
535
|
+
async (span) => {
|
|
536
|
+
const { judgePost } = await import("./judge-transport-R4TF5LEH.js");
|
|
537
|
+
const startMs = Date.now();
|
|
538
|
+
let errorType;
|
|
539
|
+
try {
|
|
540
|
+
const body = JSON.stringify(spec.body(resolvedModel, prompt));
|
|
541
|
+
const post = (apiKey) => judgePost({
|
|
542
|
+
url: spec.url,
|
|
543
|
+
headers: spec.headers(apiKey),
|
|
544
|
+
body,
|
|
545
|
+
timeoutMs: spec.timeoutMs,
|
|
546
|
+
onRetry: (reason2) => recordRetry(provider, reason2)
|
|
547
|
+
});
|
|
548
|
+
const response = await postWithKeyFallback(post, primary.value, keys);
|
|
549
|
+
if (!response.ok) {
|
|
550
|
+
await response.discard();
|
|
551
|
+
throw new JudgeHttpError(provider, response.status);
|
|
552
|
+
}
|
|
553
|
+
const usage = spec.parse(await response.json());
|
|
554
|
+
const callCostUsd = recordJudgeUsage(span, resolvedModel, provider, usage);
|
|
555
|
+
const { score, reason } = parseJudgeVerdict(usage.text);
|
|
556
|
+
const { traceId, spanId } = span.spanContext();
|
|
557
|
+
const actualCostCents = callCostUsd === void 0 ? void 0 : callCostUsd * CENTS_PER_USD;
|
|
558
|
+
return { score, reason, actualCostCents, spanContext: { traceId, spanId } };
|
|
559
|
+
} catch (error) {
|
|
560
|
+
errorType = judgeErrorType(error);
|
|
561
|
+
span.setAttribute(ERROR_TYPE_ATTR, errorType);
|
|
562
|
+
throw error;
|
|
563
|
+
} finally {
|
|
564
|
+
const inferenceAttributes = errorType === void 0 ? genAiAttributes : { ...genAiAttributes, [ERROR_TYPE_ATTR]: errorType };
|
|
565
|
+
recordMetric(GENAI_CLIENT_INFERENCE_DURATION_METRIC, elapsedSeconds(startMs), inferenceAttributes, {
|
|
566
|
+
unit: UNITS.SECOND,
|
|
567
|
+
boundaries: GENAI_DURATION_BUCKETS_S.TOOL
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
async function callLlmJudge(metricName, turn, canary) {
|
|
574
|
+
return callJudge(buildJudgePrompt(metricName, turn), canary);
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// lib/quality/judge-meta-eval.ts
|
|
578
|
+
var META_EVAL_JUDGE_MODEL = process.env.META_EVAL_JUDGE_MODEL || JUDGE_MODEL;
|
|
579
|
+
var META_EVAL_SAMPLE_RATE = 0.1;
|
|
580
|
+
function metaEvalSampleRate() {
|
|
581
|
+
const raw = process.env.META_EVAL_SAMPLE_RATE;
|
|
582
|
+
const rate = raw === void 0 || raw.trim() === "" ? NaN : Number(raw);
|
|
583
|
+
return Number.isFinite(rate) && rate >= 0 && rate <= 1 ? rate : META_EVAL_SAMPLE_RATE;
|
|
584
|
+
}
|
|
585
|
+
var META_EVAL_METRIC_NAME = "explanation_quality";
|
|
586
|
+
function shouldMetaEvaluate(evaluationName, canary) {
|
|
587
|
+
if (canary) return false;
|
|
588
|
+
if (evaluationName === META_EVAL_METRIC_NAME) return false;
|
|
589
|
+
return Math.random() < metaEvalSampleRate();
|
|
590
|
+
}
|
|
591
|
+
var META_EVAL_JUDGE_MODEL_POOL = (process.env.META_EVAL_JUDGE_MODELS ?? "").split(",").map((model) => model.trim()).filter(Boolean);
|
|
592
|
+
var META_EVAL_UNAVAILABLE_REASONS = {
|
|
593
|
+
UNKNOWN_PROVIDER: "unknown_provider",
|
|
594
|
+
MISSING_CREDENTIAL: "missing_credential"
|
|
595
|
+
};
|
|
596
|
+
function recordArmUnavailable(model, reason, extra = {}) {
|
|
597
|
+
recordMetric(
|
|
598
|
+
QUALITY_METRIC_NAMES.META_EVAL_ARM_UNAVAILABLE,
|
|
599
|
+
1,
|
|
600
|
+
{ [GENAI_REQUEST_MODEL_ATTR]: model, [QUALITY_ATTRS.META_EVAL_UNAVAILABLE_REASON]: reason, ...extra },
|
|
601
|
+
{ unit: UNITS.FAILURE }
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
function resolveUsableMetaEvalModels() {
|
|
605
|
+
const configured = META_EVAL_JUDGE_MODEL_POOL.length > 0 ? META_EVAL_JUDGE_MODEL_POOL : [META_EVAL_JUDGE_MODEL];
|
|
606
|
+
const usable = [];
|
|
607
|
+
for (const model of configured) {
|
|
608
|
+
const provider = providerForModel(model);
|
|
609
|
+
if (!provider) {
|
|
610
|
+
recordArmUnavailable(model, META_EVAL_UNAVAILABLE_REASONS.UNKNOWN_PROVIDER);
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
if (firstJudgeApiKey(provider) === void 0) {
|
|
614
|
+
recordArmUnavailable(model, META_EVAL_UNAVAILABLE_REASONS.MISSING_CREDENTIAL, { [GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider });
|
|
615
|
+
continue;
|
|
616
|
+
}
|
|
617
|
+
usable.push(model);
|
|
618
|
+
}
|
|
619
|
+
return usable;
|
|
620
|
+
}
|
|
621
|
+
var usableMetaEvalModels;
|
|
622
|
+
function pickMetaEvalJudgeModel() {
|
|
623
|
+
usableMetaEvalModels ??= resolveUsableMetaEvalModels();
|
|
624
|
+
if (usableMetaEvalModels.length === 0) return void 0;
|
|
625
|
+
const index = Math.floor(Math.random() * usableMetaEvalModels.length);
|
|
626
|
+
return usableMetaEvalModels[index];
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
// lib/quality/quality-degradation.ts
|
|
630
|
+
import { join } from "node:path";
|
|
631
|
+
var POST_JUDGE_RECENT_SCORES_LIMIT = 20;
|
|
632
|
+
var POST_JUDGE_BREACH_CONFIRMATION = 2;
|
|
633
|
+
var POST_JUDGE_EWMA_LAMBDA = 0.1;
|
|
634
|
+
var POST_JUDGE_DRIFT_MAD_MULTIPLE = 2;
|
|
635
|
+
var POST_JUDGE_MIN_SCORES = 5;
|
|
636
|
+
var POST_JUDGE_BASELINE_FRACTION = 0.7;
|
|
637
|
+
var POST_JUDGE_BASELINE_MIN_POINTS = 3;
|
|
638
|
+
var POST_JUDGE_RANGE_FALLBACK_RATIO = 0.1;
|
|
639
|
+
var POST_JUDGE_MAD_CONSISTENCY_FACTOR = 1.4826;
|
|
640
|
+
var DEGRADATION_STATE_FILENAME = "degradation.json";
|
|
641
|
+
function degradationStatePath() {
|
|
642
|
+
return join(qualityStateDir(), DEGRADATION_STATE_FILENAME);
|
|
643
|
+
}
|
|
644
|
+
var DEGRADATION_CHECK_METRIC = "integritystudio.quality.degradation_check";
|
|
645
|
+
var DEGRADATION_CONFIRMED_EVENT = "integritystudio.quality.degradation_confirmed";
|
|
646
|
+
function loadDegradationHooksState() {
|
|
647
|
+
const state = { consecutiveBreaches: {}, recentScores: {}, lastUpdated: "" };
|
|
648
|
+
const raw = loadJsonObject(degradationStatePath());
|
|
649
|
+
if (!raw) return state;
|
|
650
|
+
if (isJsonObject(raw.recentScores)) {
|
|
651
|
+
for (const [metric, scores] of Object.entries(raw.recentScores)) {
|
|
652
|
+
if (Array.isArray(scores)) {
|
|
653
|
+
state.recentScores[metric] = scores.filter((s) => typeof s === "number");
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
if (isJsonObject(raw.consecutiveBreaches)) {
|
|
658
|
+
for (const [metric, count] of Object.entries(raw.consecutiveBreaches)) {
|
|
659
|
+
if (typeof count === "number") state.consecutiveBreaches[metric] = count;
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
return state;
|
|
663
|
+
}
|
|
664
|
+
function saveDegradationHooksState(state) {
|
|
665
|
+
writeJsonAtomic(degradationStatePath(), state, { silent: true });
|
|
666
|
+
}
|
|
667
|
+
var ascending = (a, b) => a - b;
|
|
668
|
+
function median(sortedValues) {
|
|
669
|
+
const mid = Math.floor(sortedValues.length / 2);
|
|
670
|
+
const upper = sortedValues[mid] ?? Number.NaN;
|
|
671
|
+
if (sortedValues.length % 2 !== 0) return upper;
|
|
672
|
+
return ((sortedValues[mid - 1] ?? Number.NaN) + upper) / 2;
|
|
673
|
+
}
|
|
674
|
+
function scaledMAD(values) {
|
|
675
|
+
const center = median([...values].sort(ascending));
|
|
676
|
+
const deviations = values.map((v) => Math.abs(v - center)).sort(ascending);
|
|
677
|
+
return median(deviations) * POST_JUDGE_MAD_CONSISTENCY_FACTOR;
|
|
678
|
+
}
|
|
679
|
+
function ewma(values) {
|
|
680
|
+
const [first = Number.NaN, ...rest] = values;
|
|
681
|
+
let smoothed = first;
|
|
682
|
+
for (const value of rest) {
|
|
683
|
+
smoothed = POST_JUDGE_EWMA_LAMBDA * value + (1 - POST_JUDGE_EWMA_LAMBDA) * smoothed;
|
|
684
|
+
}
|
|
685
|
+
return smoothed;
|
|
686
|
+
}
|
|
687
|
+
function detectDrift(values) {
|
|
688
|
+
if (values.length < POST_JUDGE_MIN_SCORES) return false;
|
|
689
|
+
const baselineEnd = Math.max(
|
|
690
|
+
POST_JUDGE_BASELINE_MIN_POINTS,
|
|
691
|
+
Math.floor(values.length * POST_JUDGE_BASELINE_FRACTION)
|
|
692
|
+
);
|
|
693
|
+
const baseline = values.slice(0, baselineEnd);
|
|
694
|
+
const baselineMean = baseline.reduce((sum, v) => sum + v, 0) / baseline.length;
|
|
695
|
+
const deviation = Math.abs(ewma(values) - baselineMean);
|
|
696
|
+
const mad = scaledMAD(baseline);
|
|
697
|
+
if (mad === 0) {
|
|
698
|
+
const range = Math.max(...values) - Math.min(...values);
|
|
699
|
+
return range > 0 && deviation > POST_JUDGE_RANGE_FALLBACK_RATIO * range;
|
|
700
|
+
}
|
|
701
|
+
return deviation > POST_JUDGE_DRIFT_MAD_MULTIPLE * mad;
|
|
702
|
+
}
|
|
703
|
+
function applyScore(state, { metricName, score }) {
|
|
704
|
+
const scores = [...state.recentScores[metricName] ?? [], score].slice(-POST_JUDGE_RECENT_SCORES_LIMIT);
|
|
705
|
+
state.recentScores[metricName] = scores;
|
|
706
|
+
if (scores.length < POST_JUDGE_MIN_SCORES) return null;
|
|
707
|
+
const drifting = detectDrift(scores);
|
|
708
|
+
const consecutiveBreaches = drifting ? (state.consecutiveBreaches[metricName] ?? 0) + 1 : 0;
|
|
709
|
+
state.consecutiveBreaches[metricName] = consecutiveBreaches;
|
|
710
|
+
if (!drifting) return { status: "healthy", consecutiveBreaches };
|
|
711
|
+
const confirmed = consecutiveBreaches >= POST_JUDGE_BREACH_CONFIRMATION;
|
|
712
|
+
return { status: confirmed ? "critical" : "warning", consecutiveBreaches };
|
|
713
|
+
}
|
|
714
|
+
function emitDrift(metricName, { status, consecutiveBreaches }, sessionId) {
|
|
715
|
+
const confirmed = status === "critical";
|
|
716
|
+
recordCounter(DEGRADATION_CHECK_METRIC, 1, {
|
|
717
|
+
[EVALUATION_ATTRS.NAME]: metricName,
|
|
718
|
+
[QUALITY_ATTRS.STATUS]: status,
|
|
719
|
+
[QUALITY_ATTRS.CONFIRMED]: confirmed,
|
|
720
|
+
[SESSION_ATTRIBUTES.SESSION_ID]: sessionId
|
|
721
|
+
}, { unit: UNITS.CHECK });
|
|
722
|
+
if (confirmed) {
|
|
723
|
+
addEvent(DEGRADATION_CONFIRMED_EVENT, {
|
|
724
|
+
[QUALITY_ATTRS.METRIC_NAME]: metricName,
|
|
725
|
+
[QUALITY_ATTRS.STATUS]: status,
|
|
726
|
+
[QUALITY_ATTRS.CONSECUTIVE_BREACHES]: consecutiveBreaches,
|
|
727
|
+
[QUALITY_ATTRS.DRIFT_DETECTED]: true,
|
|
728
|
+
[QUALITY_ATTRS.CONFIRMED]: true,
|
|
729
|
+
[SESSION_ATTRIBUTES.SESSION_ID]: sessionId
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
function checkPostJudgeDegradationBatch(entries, sessionId) {
|
|
734
|
+
const nothing = entries.map(() => null);
|
|
735
|
+
if (!entries.some(({ score }) => Number.isFinite(score))) return nothing;
|
|
736
|
+
try {
|
|
737
|
+
const state = loadDegradationHooksState();
|
|
738
|
+
const results = entries.map((entry) => Number.isFinite(entry.score) ? applyScore(state, entry) : null);
|
|
739
|
+
state.lastUpdated = (/* @__PURE__ */ new Date()).toISOString();
|
|
740
|
+
saveDegradationHooksState(state);
|
|
741
|
+
entries.forEach(({ metricName }, index) => {
|
|
742
|
+
const result = results[index];
|
|
743
|
+
if (result && result.status !== "healthy") emitDrift(metricName, result, sessionId);
|
|
744
|
+
});
|
|
745
|
+
return results;
|
|
746
|
+
} catch {
|
|
747
|
+
return nothing;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
// lib/quality/quality-evaluator.ts
|
|
752
|
+
var BASE_JUDGE_METRICS = [JUDGE_METRIC_NAMES.RELEVANCE, JUDGE_METRIC_NAMES.COHERENCE];
|
|
753
|
+
var BASE_JUDGE_METRIC_KEYS = {
|
|
754
|
+
[JUDGE_METRIC_NAMES.RELEVANCE]: QUALITY_METRIC_NAMES.LLM_RELEVANCE,
|
|
755
|
+
[JUDGE_METRIC_NAMES.COHERENCE]: QUALITY_METRIC_NAMES.LLM_COHERENCE
|
|
756
|
+
};
|
|
757
|
+
var PRODUCER = "hook:stop-quality-evaluation";
|
|
758
|
+
async function runEvaluationHandoff(args) {
|
|
759
|
+
const record = parseEvaluationArgs(args);
|
|
760
|
+
if (!record) return false;
|
|
761
|
+
setSessionId(record.sessionId);
|
|
762
|
+
try {
|
|
763
|
+
const allTurns = extractTurns(record.transcriptPath, record.transcriptBytes);
|
|
764
|
+
await evaluateTurns({
|
|
765
|
+
sessionId: record.sessionId,
|
|
766
|
+
canary: record.canary,
|
|
767
|
+
turns: sampleTurns(allTurns, SAMPLING_CONFIG.MAX_TURNS_PER_SESSION),
|
|
768
|
+
turnsTotal: allTurns.length
|
|
769
|
+
});
|
|
770
|
+
return true;
|
|
771
|
+
} finally {
|
|
772
|
+
await shutdownTelemetry();
|
|
773
|
+
spawnTelemetryShipper();
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
async function evaluateTurns(session) {
|
|
777
|
+
const { sessionId, canary, turns } = session;
|
|
778
|
+
await instrumentHook(HOOK_NAMES.QUALITY_EVALUATION, async (ctx) => {
|
|
779
|
+
const startMs = Date.now();
|
|
780
|
+
ctx.addAttributes({
|
|
781
|
+
[SESSION_ATTRIBUTES.SESSION_ID]: sessionId,
|
|
782
|
+
[QUALITY_ATTRS.EVAL_TYPE]: QUALITY_EVAL_TYPE_LLM,
|
|
783
|
+
[QUALITY_ATTRS.CANARY]: canary,
|
|
784
|
+
[QUALITY_ATTRS.TURNS_SAMPLED]: turns.length,
|
|
785
|
+
[QUALITY_ATTRS.TURNS_TOTAL]: session.turnsTotal
|
|
786
|
+
});
|
|
787
|
+
try {
|
|
788
|
+
if (turns.length === 0) {
|
|
789
|
+
ctx.addAttribute(QUALITY_ATTRS.EVAL_SKIPPED_REASON, QUALITY_EVAL_SKIP_REASONS.NO_TURNS);
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
const calls = turns.flatMap((turn) => BASE_JUDGE_METRICS.map((metricName) => ({ turn, metricName })));
|
|
793
|
+
const results = await Promise.all(calls.map(
|
|
794
|
+
({ turn, metricName }) => reservedJudgeCall(
|
|
795
|
+
() => callLlmJudge(metricName, turn, canary),
|
|
796
|
+
metricName,
|
|
797
|
+
sessionId
|
|
798
|
+
)
|
|
799
|
+
));
|
|
800
|
+
const pendingMetaEvals = [];
|
|
801
|
+
const driftScores = [];
|
|
802
|
+
for (const [index, { turn, metricName }] of calls.entries()) {
|
|
803
|
+
const result = results[index];
|
|
804
|
+
if (!result) continue;
|
|
805
|
+
recordMetric(BASE_JUDGE_METRIC_KEYS[metricName], result.score, judgeScoreAttrs(session));
|
|
806
|
+
appendJudgeEvaluation(session, { name: metricName, result, turnRef: turnRefOf(turn) });
|
|
807
|
+
driftScores.push({ metricName: BASE_JUDGE_METRIC_KEYS[metricName], score: result.score });
|
|
808
|
+
pendingMetaEvals.push(() => maybeMetaEvaluate(session, metricName, result, turn));
|
|
809
|
+
}
|
|
810
|
+
if (!canary) checkPostJudgeDegradationBatch(driftScores, sessionId);
|
|
811
|
+
const metaEvalsRan = await Promise.all(pendingMetaEvals.map((run) => run()));
|
|
812
|
+
ctx.addAttribute(QUALITY_ATTRS.META_EVALS_RUN, metaEvalsRan.filter(Boolean).length);
|
|
813
|
+
} finally {
|
|
814
|
+
const durationMs = Date.now() - startMs;
|
|
815
|
+
emitEvaluationLatency(durationMs, HOOK_NAMES.QUALITY_EVALUATION);
|
|
816
|
+
ctx.addAttribute(QUALITY_ATTRS.EVAL_DURATION_MS, durationMs);
|
|
817
|
+
const budget = getBudgetUsage();
|
|
818
|
+
ctx.addAttribute(QUALITY_ATTRS.BUDGET_USED_CENTS, budget.estimatedCostCents);
|
|
819
|
+
ctx.addAttribute(QUALITY_ATTRS.BUDGET_REMAINING_CENTS, budget.remaining);
|
|
820
|
+
}
|
|
821
|
+
}, { [HOOK_TRIGGER_ATTR]: HOOK_EVENT_NAMES.stop, [HOOK_TYPE_ATTR]: QUALITY_EVALUATION_HOOK_TYPE });
|
|
822
|
+
}
|
|
823
|
+
async function reservedJudgeCall(call, evaluationName, sessionId, onResult) {
|
|
824
|
+
if (!tryReserveBudget(COST_PER_METRIC_CENTS)) return null;
|
|
825
|
+
try {
|
|
826
|
+
const result = await call();
|
|
827
|
+
if (result.actualCostCents !== void 0) {
|
|
828
|
+
trueUpBudget(result.actualCostCents, COST_PER_METRIC_CENTS);
|
|
829
|
+
}
|
|
830
|
+
onResult?.(result);
|
|
831
|
+
return result;
|
|
832
|
+
} catch {
|
|
833
|
+
releaseBudget(COST_PER_METRIC_CENTS);
|
|
834
|
+
recordMetric(QUALITY_METRIC_NAMES.LLM_JUDGE_FAILURES, 1, { [SESSION_ATTRIBUTES.SESSION_ID]: sessionId, [EVALUATION_ATTRS.NAME]: evaluationName }, { unit: UNITS.FAILURE });
|
|
835
|
+
return null;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
function cohortFor(canary) {
|
|
839
|
+
return canary ? EVALUATION_COHORT.CANARY : EVALUATION_COHORT.NORMAL;
|
|
840
|
+
}
|
|
841
|
+
function turnRefOf(turn) {
|
|
842
|
+
return {
|
|
843
|
+
index: turn.index,
|
|
844
|
+
userTextHash: hashTurnUserText(turn.userText),
|
|
845
|
+
...turn.responseId && { responseId: turn.responseId }
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
function judgeScoreAttrs({ sessionId, canary }) {
|
|
849
|
+
return {
|
|
850
|
+
[EVALUATION_ATTRS.EVALUATOR_KIND]: EVALUATOR_KIND.LLM,
|
|
851
|
+
[EVALUATION_ATTRS.COHORT]: cohortFor(canary),
|
|
852
|
+
[SESSION_ATTRIBUTES.SESSION_ID]: sessionId
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
function appendJudgeEvaluation({ sessionId, canary }, { name, result, turnRef, extraAttributes }) {
|
|
856
|
+
appendEvaluation({
|
|
857
|
+
name,
|
|
858
|
+
score: result.score,
|
|
859
|
+
kind: EVALUATOR_KIND.LLM,
|
|
860
|
+
cohort: cohortFor(canary),
|
|
861
|
+
producer: PRODUCER,
|
|
862
|
+
sessionId,
|
|
863
|
+
judgeSpan: result.spanContext,
|
|
864
|
+
turn: turnRef,
|
|
865
|
+
explanation: result.reason,
|
|
866
|
+
extraAttributes
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
async function maybeMetaEvaluate(session, evaluationName, result, turn) {
|
|
870
|
+
const { sessionId, canary } = session;
|
|
871
|
+
if (!shouldMetaEvaluate(evaluationName, canary)) return false;
|
|
872
|
+
const metaModel = pickMetaEvalJudgeModel();
|
|
873
|
+
if (!metaModel) return false;
|
|
874
|
+
const scored = await reservedJudgeCall(
|
|
875
|
+
() => callJudge(
|
|
876
|
+
buildExplanationQualityPrompt(evaluationName, result.score, result.reason, turn.userText),
|
|
877
|
+
false,
|
|
878
|
+
metaModel
|
|
879
|
+
),
|
|
880
|
+
META_EVAL_METRIC_NAME,
|
|
881
|
+
sessionId,
|
|
882
|
+
(metaResult) => {
|
|
883
|
+
const original = { [EVALUATION_ATTRS.META_ORIGINAL_EVALUATION]: evaluationName };
|
|
884
|
+
recordMetric(QUALITY_METRIC_NAMES.LLM_EXPLANATION_QUALITY, metaResult.score, {
|
|
885
|
+
...judgeScoreAttrs(session),
|
|
886
|
+
...original
|
|
887
|
+
});
|
|
888
|
+
appendJudgeEvaluation(session, {
|
|
889
|
+
name: META_EVAL_METRIC_NAME,
|
|
890
|
+
result: metaResult,
|
|
891
|
+
turnRef: turnRefOf(turn),
|
|
892
|
+
extraAttributes: original
|
|
893
|
+
});
|
|
894
|
+
}
|
|
895
|
+
);
|
|
896
|
+
return scored !== null;
|
|
897
|
+
}
|
|
898
|
+
export {
|
|
899
|
+
runEvaluationHandoff
|
|
900
|
+
};
|
|
901
|
+
//# sourceMappingURL=quality-evaluator-XYNDKDRG.js.map
|