observability-toolkit-hooks 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG.md +144 -0
- package/LICENSE +95 -0
- package/README.md +201 -2
- package/dist/handlers/compact.js +38 -0
- package/dist/handlers/mcp-status.js +66 -0
- package/dist/handlers/notification.js +45 -0
- package/dist/handlers/post-tool-agent.js +157 -0
- package/dist/handlers/post-tool-changelog-sync.js +148 -0
- package/dist/handlers/post-tool-code-structure.js +116 -0
- package/dist/handlers/post-tool-edit-ledger.js +15 -0
- package/dist/handlers/post-tool-failure.js +83 -0
- package/dist/handlers/post-tool-git.js +79 -0
- package/dist/handlers/post-tool-mcp.js +105 -0
- package/dist/handlers/post-tool-outcome.js +48 -0
- package/dist/handlers/post-tool-task-state.js +74 -0
- package/dist/handlers/post-tool.js +132 -0
- package/dist/handlers/pre-tool-agent.js +192 -0
- package/dist/handlers/pre-tool.js +218 -0
- package/dist/handlers/session-end.js +39 -0
- package/dist/handlers/session-start.js +262 -0
- package/dist/handlers/stop-error-reminder.js +195 -0
- package/dist/handlers/stop-quality-evaluation.js +58 -0
- package/dist/handlers/stop-session-summary.js +62 -0
- package/dist/handlers/stop-token-metrics.js +83 -0
- package/dist/handlers/stop.js +25 -0
- package/dist/handlers/subagent.js +105 -0
- package/dist/handlers/tool-event-router.js +39 -0
- package/dist/handlers/user-prompt.js +218 -0
- package/dist/hook-runner.js +10 -0
- package/dist/hook-runner.js.map +7 -0
- package/dist/lib/agent/agent-context.js +123 -0
- package/dist/lib/agent/agent-evaluations.js +155 -0
- package/dist/lib/agent/agent-file-budget.js +65 -0
- package/dist/lib/agent/agent-sources.js +165 -0
- package/dist/lib/agent/synth-spans.js +51 -0
- package/dist/lib/core/bounded-label.js +6 -0
- package/dist/lib/core/category-rules.js +11 -0
- package/dist/lib/core/child-result.js +19 -0
- package/dist/lib/core/config-dir-recovery.js +22 -0
- package/dist/lib/core/constants.js +171 -0
- package/dist/lib/core/detached-runner.js +55 -0
- package/dist/lib/core/file-utils.js +353 -0
- package/dist/lib/core/format-utils.js +51 -0
- package/dist/lib/core/http-utils.js +74 -0
- package/dist/lib/core/http1-fetch.js +14 -0
- package/dist/lib/core/identity-key-resolver.js +149 -0
- package/dist/lib/core/jsonl-tail.js +207 -0
- package/dist/lib/core/load-envrc.js +72 -0
- package/dist/lib/core/parsers.js +43 -0
- package/dist/lib/core/regex.js +27 -0
- package/dist/lib/core/settings-env.js +31 -0
- package/dist/lib/core/shell-command.js +122 -0
- package/dist/lib/core/units.js +97 -0
- package/dist/lib/git/git-cache.js +158 -0
- package/dist/lib/git/git-commit.js +189 -0
- package/dist/lib/git/git-exec.js +30 -0
- package/dist/lib/mcp/mcp-completeness.js +40 -0
- package/dist/lib/mcp/mcp-config.js +118 -0
- package/dist/lib/mcp/mcp-failure-latch.js +22 -0
- package/dist/lib/mcp/mcp-identity-launcher.js +43 -0
- package/dist/lib/otel/attributes.js +322 -0
- package/dist/lib/otel/evaluation-attrs.js +51 -0
- package/dist/lib/otel/file-exporters.js +169 -0
- package/dist/lib/otel/host-resource.js +61 -0
- package/dist/lib/otel/otel-headers-helper.js +26 -0
- package/dist/lib/otel/otel-monitor.js +181 -0
- package/dist/lib/otel/otel.js +235 -0
- package/dist/lib/otel/otlp-headers.js +101 -0
- package/dist/lib/otel/otlp-post.js +122 -0
- package/dist/lib/otel/otlp-serialize.js +142 -0
- package/dist/lib/otel/otlp-signal.js +2 -0
- package/dist/lib/otel/remote-spans.js +142 -0
- package/dist/lib/otel/service-identity.js +10 -0
- package/dist/lib/otel/telemetry-alerts.js +43 -0
- package/dist/lib/otel/token-metrics.js +122 -0
- package/dist/lib/otel/trace-context.js +81 -0
- package/dist/lib/quality/judge-meta-eval.js +70 -0
- package/dist/lib/quality/judge-prompts.js +104 -0
- package/dist/lib/quality/judge-providers.js +193 -0
- package/dist/lib/quality/judge-transport.js +93 -0
- package/dist/lib/quality/llm-judge.js +170 -0
- package/dist/lib/quality/quality-budget.js +99 -0
- package/dist/lib/quality/quality-degradation.js +156 -0
- package/dist/lib/quality/quality-eval-handoff.js +74 -0
- package/dist/lib/quality/quality-evaluator.js +178 -0
- package/dist/lib/quality/quality-sampler.js +43 -0
- package/dist/lib/quality/quality-signals.js +150 -0
- package/dist/lib/runtime/categorizers.js +101 -0
- package/dist/lib/runtime/hook-input.js +16 -0
- package/dist/lib/runtime/hook-output.js +50 -0
- package/dist/lib/runtime/output-analyzer.js +160 -0
- package/dist/lib/runtime/run-handlers.js +32 -0
- package/dist/lib/runtime/tool-input.js +111 -0
- package/dist/lib/runtime/tool-output.js +44 -0
- package/dist/lib/runtime/unknown-tool.js +32 -0
- package/dist/lib/session/edit-ledger.js +29 -0
- package/dist/lib/session/forked-skill.js +39 -0
- package/dist/lib/session/handoffs.js +77 -0
- package/dist/lib/session/session-state.js +169 -0
- package/dist/lib/session/skill-rules.js +196 -0
- package/dist/lib/session/task-tracker.js +83 -0
- package/dist/lib/shipper/shipper-credentials.js +38 -0
- package/dist/lib/shipper/shipper-cursor.js +89 -0
- package/dist/lib/shipper/shipper-status.js +174 -0
- package/dist/lib/shipper/span-shipper.js +272 -0
- package/dist/lib/shipper/telemetry-shipper.js +39 -0
- package/dist/lib/transcript/agent-transcript-metrics.js +34 -0
- package/dist/lib/transcript/context-breakdown.js +134 -0
- package/dist/lib/transcript/context-history.js +69 -0
- package/dist/lib/transcript/context-tracker.js +226 -0
- package/dist/lib/transcript/message-count.js +49 -0
- package/dist/lib/transcript/model-ids.js +36 -0
- package/dist/lib/transcript/model-pricing.js +153 -0
- package/dist/lib/transcript/token-metrics-checkpoint.js +152 -0
- package/dist/lib/transcript/tool-call-durations.js +64 -0
- package/dist/lib/transcript/transcript-entry.js +67 -0
- package/dist/lib/transcript/transcript-fold.js +130 -0
- package/dist/lib/transcript/transcript-layout.js +39 -0
- package/dist/lib/transcript/transcript-parser.js +117 -0
- package/dist/runner/agent-context-4XTCLMLZ.js +23 -0
- package/dist/runner/agent-context-4XTCLMLZ.js.map +7 -0
- package/dist/runner/agent-file-budget-Q33ESBBE.js +68 -0
- package/dist/runner/agent-file-budget-Q33ESBBE.js.map +7 -0
- package/dist/runner/agent-transcript-metrics-W5CZZFMI.js +39 -0
- package/dist/runner/agent-transcript-metrics-W5CZZFMI.js.map +7 -0
- package/dist/runner/chunk-2NTNAJZE.js +49 -0
- package/dist/runner/chunk-2NTNAJZE.js.map +7 -0
- package/dist/runner/chunk-4KWBQTTG.js +78 -0
- package/dist/runner/chunk-4KWBQTTG.js.map +7 -0
- package/dist/runner/chunk-52RY6LQT.js +699 -0
- package/dist/runner/chunk-52RY6LQT.js.map +7 -0
- package/dist/runner/chunk-7OM5CYES.js +8695 -0
- package/dist/runner/chunk-7OM5CYES.js.map +7 -0
- package/dist/runner/chunk-7PEOKGYR.js +1568 -0
- package/dist/runner/chunk-7PEOKGYR.js.map +7 -0
- package/dist/runner/chunk-AGSGKELQ.js +126 -0
- package/dist/runner/chunk-AGSGKELQ.js.map +7 -0
- package/dist/runner/chunk-AVM7PJFW.js +12 -0
- package/dist/runner/chunk-AVM7PJFW.js.map +7 -0
- package/dist/runner/chunk-C3WHUMU5.js +178 -0
- package/dist/runner/chunk-C3WHUMU5.js.map +7 -0
- package/dist/runner/chunk-DGB4NSSD.js +157 -0
- package/dist/runner/chunk-DGB4NSSD.js.map +7 -0
- package/dist/runner/chunk-DNSTWC26.js +151 -0
- package/dist/runner/chunk-DNSTWC26.js.map +7 -0
- package/dist/runner/chunk-FDJJ3TYU.js +449 -0
- package/dist/runner/chunk-FDJJ3TYU.js.map +7 -0
- package/dist/runner/chunk-FGKYDQHZ.js +137 -0
- package/dist/runner/chunk-FGKYDQHZ.js.map +7 -0
- package/dist/runner/chunk-GG2I4TGI.js +582 -0
- package/dist/runner/chunk-GG2I4TGI.js.map +7 -0
- package/dist/runner/chunk-GNNKUMRF.js +60 -0
- package/dist/runner/chunk-GNNKUMRF.js.map +7 -0
- package/dist/runner/chunk-I7TD2QVR.js +118 -0
- package/dist/runner/chunk-I7TD2QVR.js.map +7 -0
- package/dist/runner/chunk-IJ4SLUBV.js +78 -0
- package/dist/runner/chunk-IJ4SLUBV.js.map +7 -0
- package/dist/runner/chunk-IY7AP7NA.js +32 -0
- package/dist/runner/chunk-IY7AP7NA.js.map +7 -0
- package/dist/runner/chunk-J3F3Z3EQ.js +177 -0
- package/dist/runner/chunk-J3F3Z3EQ.js.map +7 -0
- package/dist/runner/chunk-JLEO5OHU.js +104 -0
- package/dist/runner/chunk-JLEO5OHU.js.map +7 -0
- package/dist/runner/chunk-JRNNLOPR.js +168 -0
- package/dist/runner/chunk-JRNNLOPR.js.map +7 -0
- package/dist/runner/chunk-KGSA5JYP.js +82 -0
- package/dist/runner/chunk-KGSA5JYP.js.map +7 -0
- package/dist/runner/chunk-LBCZAYFR.js +125 -0
- package/dist/runner/chunk-LBCZAYFR.js.map +7 -0
- package/dist/runner/chunk-LHWRXY2G.js +41 -0
- package/dist/runner/chunk-LHWRXY2G.js.map +7 -0
- package/dist/runner/chunk-LQ2LGUYJ.js +41 -0
- package/dist/runner/chunk-LQ2LGUYJ.js.map +7 -0
- package/dist/runner/chunk-LYY6RCIE.js +158 -0
- package/dist/runner/chunk-LYY6RCIE.js.map +7 -0
- package/dist/runner/chunk-M4R4QIQH.js +41 -0
- package/dist/runner/chunk-M4R4QIQH.js.map +7 -0
- package/dist/runner/chunk-MC75VXMX.js +207 -0
- package/dist/runner/chunk-MC75VXMX.js.map +7 -0
- package/dist/runner/chunk-NKZU4RRG.js +104 -0
- package/dist/runner/chunk-NKZU4RRG.js.map +7 -0
- package/dist/runner/chunk-PJHWHJ3K.js +56 -0
- package/dist/runner/chunk-PJHWHJ3K.js.map +7 -0
- package/dist/runner/chunk-RIGVYJ2O.js +114 -0
- package/dist/runner/chunk-RIGVYJ2O.js.map +7 -0
- package/dist/runner/chunk-RWRGC7ZK.js +93 -0
- package/dist/runner/chunk-RWRGC7ZK.js.map +7 -0
- package/dist/runner/chunk-SRMAJQMV.js +54 -0
- package/dist/runner/chunk-SRMAJQMV.js.map +7 -0
- package/dist/runner/chunk-TRMFD36J.js +47 -0
- package/dist/runner/chunk-TRMFD36J.js.map +7 -0
- package/dist/runner/chunk-U4IOZYQ3.js +21 -0
- package/dist/runner/chunk-U4IOZYQ3.js.map +7 -0
- package/dist/runner/chunk-UEXDE3WP.js +83 -0
- package/dist/runner/chunk-UEXDE3WP.js.map +7 -0
- package/dist/runner/chunk-UW3GQZSM.js +50 -0
- package/dist/runner/chunk-UW3GQZSM.js.map +7 -0
- package/dist/runner/chunk-V2GSP6VR.js +46 -0
- package/dist/runner/chunk-V2GSP6VR.js.map +7 -0
- package/dist/runner/chunk-VOV4Y5VU.js +81 -0
- package/dist/runner/chunk-VOV4Y5VU.js.map +7 -0
- package/dist/runner/chunk-VT5T6OEI.js +65 -0
- package/dist/runner/chunk-VT5T6OEI.js.map +7 -0
- package/dist/runner/chunk-VUYVFWIN.js +12 -0
- package/dist/runner/chunk-VUYVFWIN.js.map +7 -0
- package/dist/runner/chunk-VVGOPPEN.js +36 -0
- package/dist/runner/chunk-VVGOPPEN.js.map +7 -0
- package/dist/runner/chunk-VXFSLM6P.js +14 -0
- package/dist/runner/chunk-VXFSLM6P.js.map +7 -0
- package/dist/runner/chunk-W3EK7JUT.js +26125 -0
- package/dist/runner/chunk-W3EK7JUT.js.map +7 -0
- package/dist/runner/chunk-WJHNCOUF.js +52 -0
- package/dist/runner/chunk-WJHNCOUF.js.map +7 -0
- package/dist/runner/chunk-WU7ANGCW.js +269 -0
- package/dist/runner/chunk-WU7ANGCW.js.map +7 -0
- package/dist/runner/chunk-X7XLMSZX.js +44 -0
- package/dist/runner/chunk-X7XLMSZX.js.map +7 -0
- package/dist/runner/chunk-XSO3B4QN.js +124 -0
- package/dist/runner/chunk-XSO3B4QN.js.map +7 -0
- package/dist/runner/chunk-YGFN3LN2.js +174 -0
- package/dist/runner/chunk-YGFN3LN2.js.map +7 -0
- package/dist/runner/chunk-YHYDBLVB.js +33 -0
- package/dist/runner/chunk-YHYDBLVB.js.map +7 -0
- package/dist/runner/compact-36NP3PKT.js +60 -0
- package/dist/runner/compact-36NP3PKT.js.map +7 -0
- package/dist/runner/getMachineId-bsd-AGENNHN6.js +42 -0
- package/dist/runner/getMachineId-bsd-AGENNHN6.js.map +7 -0
- package/dist/runner/getMachineId-bsd-I5WZGCL4.js +33 -0
- package/dist/runner/getMachineId-bsd-I5WZGCL4.js.map +7 -0
- package/dist/runner/getMachineId-bsd-ZXK225P7.js +33 -0
- package/dist/runner/getMachineId-bsd-ZXK225P7.js.map +7 -0
- package/dist/runner/getMachineId-darwin-47GDLEXL.js +42 -0
- package/dist/runner/getMachineId-darwin-47GDLEXL.js.map +7 -0
- package/dist/runner/getMachineId-darwin-5CRLMDD6.js +33 -0
- package/dist/runner/getMachineId-darwin-5CRLMDD6.js.map +7 -0
- package/dist/runner/getMachineId-darwin-ADG4I32N.js +33 -0
- package/dist/runner/getMachineId-darwin-ADG4I32N.js.map +7 -0
- package/dist/runner/getMachineId-linux-5KVS3T2M.js +27 -0
- package/dist/runner/getMachineId-linux-5KVS3T2M.js.map +7 -0
- package/dist/runner/getMachineId-linux-GDFB5GSO.js +34 -0
- package/dist/runner/getMachineId-linux-GDFB5GSO.js.map +7 -0
- package/dist/runner/getMachineId-linux-NQMWCRO3.js +27 -0
- package/dist/runner/getMachineId-linux-NQMWCRO3.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-5PFIGMVX.js +18 -0
- package/dist/runner/getMachineId-unsupported-5PFIGMVX.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-DW27K3ZX.js +18 -0
- package/dist/runner/getMachineId-unsupported-DW27K3ZX.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-XV3FHTHD.js +25 -0
- package/dist/runner/getMachineId-unsupported-XV3FHTHD.js.map +7 -0
- package/dist/runner/getMachineId-win-5JVMW3EI.js +44 -0
- package/dist/runner/getMachineId-win-5JVMW3EI.js.map +7 -0
- package/dist/runner/getMachineId-win-SNESQF7R.js +35 -0
- package/dist/runner/getMachineId-win-SNESQF7R.js.map +7 -0
- package/dist/runner/getMachineId-win-UJYHSBM2.js +35 -0
- package/dist/runner/getMachineId-win-UJYHSBM2.js.map +7 -0
- package/dist/runner/git-commit-BA53DOJQ.js +30 -0
- package/dist/runner/git-commit-BA53DOJQ.js.map +7 -0
- package/dist/runner/hook-runner-JHH6PGKE.js +220 -0
- package/dist/runner/hook-runner-JHH6PGKE.js.map +7 -0
- package/dist/runner/judge-transport-R4TF5LEH.js +91 -0
- package/dist/runner/judge-transport-R4TF5LEH.js.map +7 -0
- package/dist/runner/mcp-status-QEUH2KGX.js +89 -0
- package/dist/runner/mcp-status-QEUH2KGX.js.map +7 -0
- package/dist/runner/notification-FKMBCSXF.js +69 -0
- package/dist/runner/notification-FKMBCSXF.js.map +7 -0
- package/dist/runner/post-tool-3MRHTBKH.js +192 -0
- package/dist/runner/post-tool-3MRHTBKH.js.map +7 -0
- package/dist/runner/post-tool-agent-BB34MNHI.js +370 -0
- package/dist/runner/post-tool-agent-BB34MNHI.js.map +7 -0
- package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js +147 -0
- package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js.map +7 -0
- package/dist/runner/post-tool-code-structure-LE7J2SC2.js +130 -0
- package/dist/runner/post-tool-code-structure-LE7J2SC2.js.map +7 -0
- package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js +30 -0
- package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js.map +7 -0
- package/dist/runner/post-tool-failure-XTH2I2Z7.js +111 -0
- package/dist/runner/post-tool-failure-XTH2I2Z7.js.map +7 -0
- package/dist/runner/post-tool-git-E46HZGZO.js +94 -0
- package/dist/runner/post-tool-git-E46HZGZO.js.map +7 -0
- package/dist/runner/post-tool-mcp-AMCDY5GH.js +144 -0
- package/dist/runner/post-tool-mcp-AMCDY5GH.js.map +7 -0
- package/dist/runner/post-tool-task-state-TLDOI33H.js +100 -0
- package/dist/runner/post-tool-task-state-TLDOI33H.js.map +7 -0
- package/dist/runner/pre-tool-SQE2FZQO.js +245 -0
- package/dist/runner/pre-tool-SQE2FZQO.js.map +7 -0
- package/dist/runner/pre-tool-agent-OZUPVFUI.js +218 -0
- package/dist/runner/pre-tool-agent-OZUPVFUI.js.map +7 -0
- package/dist/runner/quality-evaluator-XYNDKDRG.js +901 -0
- package/dist/runner/quality-evaluator-XYNDKDRG.js.map +7 -0
- package/dist/runner/session-end-3MJTDLLA.js +57 -0
- package/dist/runner/session-end-3MJTDLLA.js.map +7 -0
- package/dist/runner/session-start-5ZMCLKBS.js +352 -0
- package/dist/runner/session-start-5ZMCLKBS.js.map +7 -0
- package/dist/runner/shipper-status-MADX6SPG.js +37 -0
- package/dist/runner/shipper-status-MADX6SPG.js.map +7 -0
- package/dist/runner/span-shipper-4HQDHC47.js +3408 -0
- package/dist/runner/span-shipper-4HQDHC47.js.map +7 -0
- package/dist/runner/stop-RTKSIYJV.js +596 -0
- package/dist/runner/stop-RTKSIYJV.js.map +7 -0
- package/dist/runner/subagent-CBCG4OLZ.js +116 -0
- package/dist/runner/subagent-CBCG4OLZ.js.map +7 -0
- package/dist/runner/token-metrics-AD2RAFGM.js +31 -0
- package/dist/runner/token-metrics-AD2RAFGM.js.map +7 -0
- package/dist/runner/user-prompt-T5NZ2UJD.js +376 -0
- package/dist/runner/user-prompt-T5NZ2UJD.js.map +7 -0
- package/package.json +70 -5
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telemetry names read outside the file that writes them: semconv and `integritystudio.*` attribute keys,
|
|
3
|
+
* span names, metric names and histogram buckets. A key one handler writes and nothing else reads may stay a
|
|
4
|
+
* literal there; move it here once a second file names it.
|
|
5
|
+
*/
|
|
6
|
+
// ---- Semantic-convention keys ----
|
|
7
|
+
/*
|
|
8
|
+
* GenAI semconv source for every `gen_ai.*` name in the hooks (the one place this is dated):
|
|
9
|
+
* https://github.com/open-telemetry/semantic-conventions-genai, release v0.1.0 (published 2026-11-21 UTC,
|
|
10
|
+
* built on semantic-conventions v1.45.0). Doc paths and both bucket advisories checked against main
|
|
11
|
+
* 2026-11-21 UTC. Every `gen_ai.*` convention is Development stability.
|
|
12
|
+
* opentelemetry.io and the main semconv repo are stale stubs for `gen_ai.*`.
|
|
13
|
+
* Attribute registry: docs/registry/attributes/gen-ai.md. Each definition below cites its own doc.
|
|
14
|
+
*/
|
|
15
|
+
/** OTel `error.type` (core registry, stable). Set only on failure; absence means success. */
|
|
16
|
+
export const ERROR_TYPE_ATTR = 'error.type';
|
|
17
|
+
/** `error.type` for a failure no pattern classifies. */
|
|
18
|
+
export const UNKNOWN_ERROR_TYPE = 'unknown';
|
|
19
|
+
/** OTel `file.path` (core registry). */
|
|
20
|
+
export const FILE_PATH_ATTR = 'file.path';
|
|
21
|
+
export const GENAI_TOOL_NAME_ATTR = 'gen_ai.tool.name';
|
|
22
|
+
export const GENAI_TOOL_CALL_ID_ATTR = 'gen_ai.tool.call.id';
|
|
23
|
+
/** GenAI agent-span keys: semantic-conventions-genai docs/gen-ai/gen-ai-agent-spans.md (source dated above). */
|
|
24
|
+
export const GENAI_AGENT_ATTRIBUTES = {
|
|
25
|
+
AGENT_ID: 'gen_ai.agent.id',
|
|
26
|
+
AGENT_NAME: 'gen_ai.agent.name',
|
|
27
|
+
OPERATION_NAME: 'gen_ai.operation.name',
|
|
28
|
+
/** Required on `create_agent` spans. */
|
|
29
|
+
PROVIDER_NAME: 'gen_ai.provider.name',
|
|
30
|
+
/** Not `gen_ai.request.model`, which SHOULD NOT be set with dynamic model selection. */
|
|
31
|
+
RESPONSE_MODEL: 'gen_ai.response.model',
|
|
32
|
+
};
|
|
33
|
+
/** Client calls only; never on agent spans, which use `AGENT_ATTRS.REQUESTED_MODEL`. */
|
|
34
|
+
export const GENAI_REQUEST_MODEL_ATTR = 'gen_ai.request.model';
|
|
35
|
+
export const GENAI_OPERATION = {
|
|
36
|
+
INVOKE_AGENT: 'invoke_agent',
|
|
37
|
+
CREATE_AGENT: 'create_agent',
|
|
38
|
+
/** The judge's `gen_ai.client.operation` span. */
|
|
39
|
+
CHAT: 'chat',
|
|
40
|
+
/** The judge's token and cost counters. */
|
|
41
|
+
EVALUATE: 'evaluate',
|
|
42
|
+
};
|
|
43
|
+
/** `gen_ai.provider.name` values from docs/registry/attributes/gen-ai.md (xAI is `x_ai`; source dated above). */
|
|
44
|
+
export const GENAI_PROVIDER_NAMES = {
|
|
45
|
+
ANTHROPIC: 'anthropic',
|
|
46
|
+
OPENAI: 'openai',
|
|
47
|
+
XAI: 'x_ai',
|
|
48
|
+
};
|
|
49
|
+
/** CONVERSATION_ID comes only from a genuine thread id, never a path; PROJECT_SLUG groups conversations and must not shadow it. */
|
|
50
|
+
export const SESSION_ATTRIBUTES = {
|
|
51
|
+
SESSION_ID: 'session.id',
|
|
52
|
+
/** Standardized GenAI thread id (Development stability). */
|
|
53
|
+
CONVERSATION_ID: 'gen_ai.conversation.id',
|
|
54
|
+
PROJECT_SLUG: 'integritystudio.project.slug',
|
|
55
|
+
PROJECT_NAME: 'integritystudio.project.name',
|
|
56
|
+
PREVIOUS_ID: 'session.previous_id',
|
|
57
|
+
};
|
|
58
|
+
/** Registered `vcs.*` keys (core semconv, release candidate). */
|
|
59
|
+
export const VCS_ATTRIBUTES = {
|
|
60
|
+
REF_HEAD_NAME: 'vcs.ref.head.name',
|
|
61
|
+
/** The repository alone; SHOULD NOT include the owner. */
|
|
62
|
+
REPOSITORY_NAME: 'vcs.repository.name',
|
|
63
|
+
OWNER_NAME: 'vcs.owner.name',
|
|
64
|
+
};
|
|
65
|
+
/** Registered `process.*` keys (core semconv, release candidate). */
|
|
66
|
+
export const PROCESS_ATTRIBUTES = {
|
|
67
|
+
PID: 'process.pid',
|
|
68
|
+
RUNTIME_VERSION: 'process.runtime.version',
|
|
69
|
+
WORKING_DIRECTORY: 'process.working_directory',
|
|
70
|
+
};
|
|
71
|
+
// ---- Vendor keys (integritystudio.*) ----
|
|
72
|
+
/** Hook span keys; here rather than otel-monitor.ts so modules can name them without importing the monitor. */
|
|
73
|
+
export const HOOK_NAME_ATTR = 'integritystudio.hook.name';
|
|
74
|
+
export const HOOK_TYPE_ATTR = 'integritystudio.hook.type';
|
|
75
|
+
export const HOOK_STATUS_ATTR = 'integritystudio.hook.status';
|
|
76
|
+
/** Seeded on handler spans only. */
|
|
77
|
+
export const HOOK_TRIGGER_ATTR = 'integritystudio.hook.trigger';
|
|
78
|
+
export const HOOK_INSTRUMENTATION_ATTR = 'integritystudio.hook.instrumentation';
|
|
79
|
+
export const HOOK_INSTRUMENTATION_INTERNAL = 'internal';
|
|
80
|
+
export const UNKNOWN_HOOK_TYPE = 'unknown';
|
|
81
|
+
/** Every `instrumentHook` name; report readers select spans by them, and the span name is `hookSpanName(name)`. */
|
|
82
|
+
export const HOOK_NAMES = {
|
|
83
|
+
SESSION_START: 'session-start',
|
|
84
|
+
SESSION_END: 'session-end',
|
|
85
|
+
SKILL_ACTIVATION_PROMPT: 'skill-activation-prompt',
|
|
86
|
+
NOTIFICATION: 'notification',
|
|
87
|
+
PRE_COMPACT: 'pre-compact',
|
|
88
|
+
POST_COMPACT: 'post-compact',
|
|
89
|
+
BUILTIN_PRE_TOOL: 'builtin-pre-tool',
|
|
90
|
+
MCP_PRE_TOOL: 'mcp-pre-tool',
|
|
91
|
+
PLUGIN_PRE_TOOL: 'plugin-pre-tool',
|
|
92
|
+
BUILTIN_POST_TOOL: 'builtin-post-tool',
|
|
93
|
+
MCP_POST_TOOL: 'mcp-post-tool',
|
|
94
|
+
PLUGIN_POST_TOOL: 'plugin-post-tool',
|
|
95
|
+
POST_TOOL_FAILURE: 'post-tool-failure',
|
|
96
|
+
UNKNOWN_TOOL: 'unknown-tool',
|
|
97
|
+
TASK_STATE: 'builtin.task-state',
|
|
98
|
+
CODE_STRUCTURE: 'code-structure',
|
|
99
|
+
CHANGELOG_VERSION_BUMP: 'changelog-version-bump',
|
|
100
|
+
POST_COMMIT_REVIEW: 'post-commit-review',
|
|
101
|
+
AGENT_PREPARE: 'agent.operation.prepare',
|
|
102
|
+
AGENT_FINALIZE: 'agent.operation.finalize',
|
|
103
|
+
SUBAGENT_START: 'subagent-start',
|
|
104
|
+
SUBAGENT_STOP: 'subagent-stop',
|
|
105
|
+
TOKEN_METRICS: 'token-metrics-extraction',
|
|
106
|
+
MCP_SERVER_FAILURE: 'mcp-server-failure',
|
|
107
|
+
MCP_SESSION_FAILURE_SUMMARY: 'mcp-session-failure-summary',
|
|
108
|
+
TELEMETRY_ALERT_EVALUATION: 'telemetry-alert-evaluation',
|
|
109
|
+
ERROR_HANDLING_REMINDER: 'error-handling-reminder',
|
|
110
|
+
QUALITY_EVALUATION_QUEUE: 'quality-evaluation-queue',
|
|
111
|
+
QUALITY_EVALUATION: 'quality-evaluation-t2',
|
|
112
|
+
};
|
|
113
|
+
export const HOOK_SPAN_PREFIX = 'hook:';
|
|
114
|
+
export function hookSpanName(hookName) {
|
|
115
|
+
return `${HOOK_SPAN_PREFIX}${hookName}`;
|
|
116
|
+
}
|
|
117
|
+
export const TOOL_ATTRS = {
|
|
118
|
+
CATEGORY: 'integritystudio.tool.category',
|
|
119
|
+
/** Where the tool's work ran (a Bash command's leading `cd`, else the event `cwd`), not the hook process's cwd. */
|
|
120
|
+
WORKING_DIRECTORY: 'integritystudio.tool.working_directory',
|
|
121
|
+
SUCCESS: 'integritystudio.tool.success',
|
|
122
|
+
OUTPUT_SIZE: 'integritystudio.tool.output_size',
|
|
123
|
+
HAS_ERROR: 'integritystudio.tool.has_error',
|
|
124
|
+
ERROR_TYPE: 'integritystudio.tool.error_type',
|
|
125
|
+
ERROR_CATEGORY: 'integritystudio.tool.error_category',
|
|
126
|
+
RESULT_COUNT: 'integritystudio.tool.result_count',
|
|
127
|
+
ROUTER_CATEGORY: 'integritystudio.tool.router_category',
|
|
128
|
+
IS_INTERRUPT: 'integritystudio.tool.is_interrupt',
|
|
129
|
+
ERROR: 'integritystudio.tool.error',
|
|
130
|
+
/** Claude Code's own timing of a failed call; the only measured tool duration. */
|
|
131
|
+
DURATION_MS: 'integritystudio.tool.duration_ms',
|
|
132
|
+
/** A wait the `execute_tool.duration` sample is known to contain; absent means unobservable, e.g. permission prompts. */
|
|
133
|
+
WAIT_KIND: 'integritystudio.tool.wait_kind',
|
|
134
|
+
};
|
|
135
|
+
/** The MCP keys the pre- and post-tool spans, logs and metrics join on. */
|
|
136
|
+
export const MCP_ATTRS = {
|
|
137
|
+
SERVER: 'integritystudio.mcp.server',
|
|
138
|
+
TOOL: 'integritystudio.mcp.tool',
|
|
139
|
+
FULL_NAME: 'integritystudio.mcp.full_name',
|
|
140
|
+
SUCCESS: 'integritystudio.mcp.success',
|
|
141
|
+
/** Where the server is configured; a metric label as well as a span key. */
|
|
142
|
+
SCOPE: 'integritystudio.mcp.scope',
|
|
143
|
+
OUTPUT_SIZE: 'integritystudio.mcp.output_size',
|
|
144
|
+
HAS_ERROR: 'integritystudio.mcp.has_error',
|
|
145
|
+
ERROR_TYPE: 'integritystudio.mcp.error_type',
|
|
146
|
+
ERROR_CATEGORY: 'integritystudio.mcp.error_category',
|
|
147
|
+
/** The output looks like it came from a different project than the event `cwd`. */
|
|
148
|
+
CONTEXT_MISMATCH: 'integritystudio.mcp.context_mismatch',
|
|
149
|
+
SUGGESTION: 'integritystudio.mcp.suggestion',
|
|
150
|
+
};
|
|
151
|
+
/** Span attribute and metric name alike. */
|
|
152
|
+
export const CODE_STRUCTURE_SCORE = 'integritystudio.code.structure.score';
|
|
153
|
+
export const TASK_ATTRS = {
|
|
154
|
+
ID: 'integritystudio.task.id',
|
|
155
|
+
STATUS: 'integritystudio.task.status',
|
|
156
|
+
};
|
|
157
|
+
export const ALERT_ATTRS = {
|
|
158
|
+
NAME: 'integritystudio.alert.name',
|
|
159
|
+
MESSAGE: 'integritystudio.alert.message',
|
|
160
|
+
SEVERITY: 'integritystudio.alert.severity',
|
|
161
|
+
};
|
|
162
|
+
/** Session token totals on the Stop hook's token-metrics span. */
|
|
163
|
+
export const TOKEN_ATTRS = {
|
|
164
|
+
FOUND: 'integritystudio.tokens.found',
|
|
165
|
+
UNCHANGED: 'integritystudio.tokens.unchanged',
|
|
166
|
+
INPUT: 'integritystudio.tokens.input',
|
|
167
|
+
OUTPUT: 'integritystudio.tokens.output',
|
|
168
|
+
CACHE_READ: 'integritystudio.tokens.cache_read',
|
|
169
|
+
CACHE_CREATION: 'integritystudio.tokens.cache_creation',
|
|
170
|
+
MESSAGES: 'integritystudio.tokens.messages',
|
|
171
|
+
MODEL: 'integritystudio.tokens.model',
|
|
172
|
+
};
|
|
173
|
+
/** The plugin keys invocations and completions join on. */
|
|
174
|
+
export const PLUGIN_ATTRS = {
|
|
175
|
+
NAME: 'integritystudio.plugin.name',
|
|
176
|
+
NAMESPACE: 'integritystudio.plugin.namespace',
|
|
177
|
+
};
|
|
178
|
+
export const TOOL_WAIT_KIND = {
|
|
179
|
+
INTERACTIVE: 'interactive',
|
|
180
|
+
/** Result in a later turn than the call; observable only through the Stop checkpoint. */
|
|
181
|
+
CROSS_TURN: 'cross_turn',
|
|
182
|
+
};
|
|
183
|
+
export const AGENT_ATTRS = {
|
|
184
|
+
TYPE: 'integritystudio.agent.type',
|
|
185
|
+
CATEGORY: 'integritystudio.agent.category',
|
|
186
|
+
SOURCE_TYPE: 'integritystudio.agent.source_type',
|
|
187
|
+
/** The harness's per-run `agent_id`; `gen_ai.agent.id` is the stable identity (`composeAgentId`). */
|
|
188
|
+
INSTANCE_ID: 'integritystudio.agent.instance_id',
|
|
189
|
+
IS_RESUME: 'integritystudio.agent.is_resume',
|
|
190
|
+
IS_BACKGROUND: 'integritystudio.agent.is_background',
|
|
191
|
+
OUTPUT_SIZE: 'integritystudio.agent.output_size',
|
|
192
|
+
HAS_ERROR: 'integritystudio.agent.has_error',
|
|
193
|
+
HAS_RATE_LIMIT: 'integritystudio.agent.has_rate_limit',
|
|
194
|
+
OUTPUT_MENTIONS_ERROR: 'integritystudio.agent.output_mentions_error',
|
|
195
|
+
OUTPUT_HAS_STRUCTURE: 'integritystudio.agent.output.has_structure',
|
|
196
|
+
OUTPUT_HAS_CODE: 'integritystudio.agent.output.has_code',
|
|
197
|
+
OUTPUT_TRUNCATED: 'integritystudio.agent.output.truncated',
|
|
198
|
+
OUTPUT_EMPTY: 'integritystudio.agent.output.empty',
|
|
199
|
+
OUTPUT_HAS_ACTIONS: 'integritystudio.agent.output.has_actions',
|
|
200
|
+
TRANSCRIPT_PATH: 'integritystudio.agent.transcript_path',
|
|
201
|
+
HAS_TRANSCRIPT: 'integritystudio.agent.has_transcript',
|
|
202
|
+
STOP_HOOK_ACTIVE: 'integritystudio.agent.stop_hook_active',
|
|
203
|
+
LAST_MESSAGE_LENGTH: 'integritystudio.agent.last_message_length',
|
|
204
|
+
/** Shared by the pre/post hook spans and the synthetic span, so Q4 pairs by exact id. */
|
|
205
|
+
INVOCATION_ID: 'integritystudio.agent.invocation_id',
|
|
206
|
+
/** Per-call model override, in place of `gen_ai.request.model`. */
|
|
207
|
+
REQUESTED_MODEL: 'integritystudio.agent.requested_model',
|
|
208
|
+
/** Set only when the agent ships inside a skill. */
|
|
209
|
+
PARENT_SKILL: 'integritystudio.agent.parent_skill',
|
|
210
|
+
TASK_DESCRIPTION: 'integritystudio.agent.task_description',
|
|
211
|
+
};
|
|
212
|
+
/** Labels the cross-trace link from a synthetic `invoke_agent` span back to its dispatching span. */
|
|
213
|
+
export const AGENT_LINK_ATTRIBUTES = {
|
|
214
|
+
LINK_KIND: 'integritystudio.link.kind',
|
|
215
|
+
};
|
|
216
|
+
export const AGENT_LINK_KIND_DISPATCH = 'agent_dispatch';
|
|
217
|
+
/** Vendor keys on the SDK Resource only, never on spans or logs; the file exporters drop them. */
|
|
218
|
+
export const RESOURCE_ATTRIBUTES = {
|
|
219
|
+
/** The hook process's own cwd basename. */
|
|
220
|
+
PROJECT_DIRECTORY: 'integritystudio.project.directory',
|
|
221
|
+
};
|
|
222
|
+
export const CONTEXT_ATTRIBUTES = {
|
|
223
|
+
ESTIMATED_TOKENS: 'integritystudio.context.estimated_tokens',
|
|
224
|
+
UTILIZATION_PERCENT: 'integritystudio.context.utilization_percent',
|
|
225
|
+
FREE_SPACE: 'integritystudio.context.free_space',
|
|
226
|
+
TRANSCRIPT_SIZE: 'integritystudio.context.transcript_size',
|
|
227
|
+
MESSAGE_COUNT: 'integritystudio.context.message_count',
|
|
228
|
+
IS_RESUME: 'integritystudio.context.is_resume',
|
|
229
|
+
/** Which `CONTEXT_TYPE` a `SESSION_METRICS.CONTEXT_BY_TYPE` sample measures. */
|
|
230
|
+
TYPE: 'integritystudio.context.type',
|
|
231
|
+
CACHE_HIT_RATE: 'integritystudio.context.cache_hit_rate',
|
|
232
|
+
CACHE_EFFICIENCY: 'integritystudio.context.cache_efficiency',
|
|
233
|
+
};
|
|
234
|
+
/** `integritystudio.git.*` keys; a commit SHA belongs in semconv `vcs.ref.head.revision`, not here. */
|
|
235
|
+
export const GIT_ATTRIBUTES = {
|
|
236
|
+
UNCOMMITTED: 'integritystudio.git.uncommitted',
|
|
237
|
+
COMMAND: 'integritystudio.git.command',
|
|
238
|
+
FILES_CHANGED: 'integritystudio.git.files_changed',
|
|
239
|
+
/** One Bash command can make several commits (/git-commit-smart) */
|
|
240
|
+
COMMIT_COUNT: 'integritystudio.git.commit_count',
|
|
241
|
+
FILES_OUTCOME: 'integritystudio.git.files_outcome',
|
|
242
|
+
DOCS_ONLY: 'integritystudio.git.docs_only',
|
|
243
|
+
/** Repo-relative, capped; FILES_TRUNCATED marks the cap */
|
|
244
|
+
FILES: 'integritystudio.git.files',
|
|
245
|
+
FILES_TRUNCATED: 'integritystudio.git.files_truncated',
|
|
246
|
+
/** Why the post-commit review was not requested: `docs_only` or `already_reviewed`; older rows also carry `generated_only` */
|
|
247
|
+
REVIEW_SKIP_REASON: 'integritystudio.git.review_skip_reason',
|
|
248
|
+
};
|
|
249
|
+
/** Anything else means the payload changed. */
|
|
250
|
+
export const COMPACTION_TRIGGERS = {
|
|
251
|
+
MANUAL: 'manual',
|
|
252
|
+
AUTO: 'auto',
|
|
253
|
+
};
|
|
254
|
+
export const COMPACTION_ATTRIBUTES = {
|
|
255
|
+
TRIGGER: 'integritystudio.compaction.trigger',
|
|
256
|
+
TRIGGER_RAW: 'integritystudio.compaction.trigger_raw',
|
|
257
|
+
HAS_CUSTOM_INSTRUCTIONS: 'integritystudio.compaction.has_custom_instructions',
|
|
258
|
+
CUSTOM_INSTRUCTIONS_LENGTH: 'integritystudio.compaction.custom_instructions_length',
|
|
259
|
+
SUMMARY_LENGTH: 'integritystudio.compaction.summary_length',
|
|
260
|
+
};
|
|
261
|
+
// ---- Metric names and histogram buckets ----
|
|
262
|
+
/** The one instrument with an exponential-histogram View in otel.ts. Spec: docs/gen-ai/execute-tool.md. */
|
|
263
|
+
export const GENAI_EXECUTE_TOOL_DURATION_METRIC = 'gen_ai.execute_tool.duration';
|
|
264
|
+
/** In seconds, unlike every other duration here. Spec: docs/gen-ai/gen-ai-metrics.md. */
|
|
265
|
+
export const GENAI_INVOKE_AGENT_DURATION_METRIC = 'gen_ai.invoke_agent.duration';
|
|
266
|
+
/** The judge's span, one per outbound model call. */
|
|
267
|
+
export const GENAI_CLIENT_OPERATION_SPAN = 'gen_ai.client.operation';
|
|
268
|
+
/**
|
|
269
|
+
* The judge's call duration, with `error.type` on failure. Spec: docs/gen-ai/client-inference.md
|
|
270
|
+
* (semantic-conventions-genai #521, merged 2026-10-06), which replaced `gen_ai.client.operation.duration` for inference.
|
|
271
|
+
*/
|
|
272
|
+
export const GENAI_CLIENT_INFERENCE_DURATION_METRIC = 'gen_ai.client.inference.duration';
|
|
273
|
+
/**
|
|
274
|
+
* GenAI semconv bucket advisories in seconds, verbatim from semantic-conventions-genai
|
|
275
|
+
* docs/gen-ai/execute-tool.md (TOOL) and docs/gen-ai/gen-ai-metrics.md (AGENT); source dated above.
|
|
276
|
+
* Without them the SDK's millisecond-shaped defaults put every sample in the first bucket.
|
|
277
|
+
* Instruments register once per name, so every call site must pass the same list.
|
|
278
|
+
*/
|
|
279
|
+
export const GENAI_DURATION_BUCKETS_S = {
|
|
280
|
+
/** `gen_ai.execute_tool.duration` and `gen_ai.client.inference.duration`. */
|
|
281
|
+
TOOL: [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92],
|
|
282
|
+
/** `gen_ai.invoke_agent.duration`, up to 24 hours. Rows before 2026-11-21 (UTC) stop at 409.6. */
|
|
283
|
+
AGENT: [0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 51.2, 102.4, 204.8, 409.6, 900, 1800, 3600, 7200, 14400, 43200, 86400],
|
|
284
|
+
};
|
|
285
|
+
export const MCP_METRICS = {
|
|
286
|
+
COMPLETIONS: 'integritystudio.mcp.completions',
|
|
287
|
+
/** Same name as the span key. */
|
|
288
|
+
OUTPUT_SIZE: MCP_ATTRS.OUTPUT_SIZE,
|
|
289
|
+
};
|
|
290
|
+
/** `runHandlers` rejections; only PostToolUse fans out handler groups. */
|
|
291
|
+
export const POST_TOOL_HANDLER_ATTRS = {
|
|
292
|
+
GROUP: 'integritystudio.post_tool.handler_group',
|
|
293
|
+
INDEX: 'integritystudio.post_tool.handler_index',
|
|
294
|
+
};
|
|
295
|
+
export const POST_TOOL_METRICS = {
|
|
296
|
+
HANDLER_REJECTIONS: 'integritystudio.post_tool.handler_rejections',
|
|
297
|
+
};
|
|
298
|
+
export const GIT_METRICS = {
|
|
299
|
+
COMMITS_REVIEWED: 'integritystudio.git.commits_reviewed',
|
|
300
|
+
COMMITS_REVIEW_SKIPPED: 'integritystudio.git.commits_review_skipped',
|
|
301
|
+
};
|
|
302
|
+
export const CONTEXT_METRICS = {
|
|
303
|
+
COMPACTIONS: 'integritystudio.context.compactions',
|
|
304
|
+
COMPACTION_SUMMARY_SIZE: 'integritystudio.context.compaction_summary_size',
|
|
305
|
+
COMPACTIONS_COMPLETED: 'integritystudio.context.compactions_completed',
|
|
306
|
+
/** Per-prompt snapshot; its own instrument because `SESSION_METRICS.CACHE_HIT_RATE` is the Stop hook's cumulative ratio. */
|
|
307
|
+
CACHE_HIT_RATE: CONTEXT_ATTRIBUTES.CACHE_HIT_RATE,
|
|
308
|
+
};
|
|
309
|
+
/**
|
|
310
|
+
* Whole-session figures: Stop totals and context-window gauges. No model label, since a
|
|
311
|
+
* `/model` switch makes them belong to neither. Vendor-prefixed: semconv owns `session.*`.
|
|
312
|
+
*/
|
|
313
|
+
export const SESSION_METRICS = {
|
|
314
|
+
TOTAL_TOKENS: 'integritystudio.session.total_tokens',
|
|
315
|
+
CACHE_HIT_RATE: 'integritystudio.session.cache_hit_rate',
|
|
316
|
+
CACHE_EFFICIENCY: 'integritystudio.session.cache_efficiency',
|
|
317
|
+
CONTEXT_SIZE: 'integritystudio.session.context.size',
|
|
318
|
+
CONTEXT_UTILIZATION: 'integritystudio.session.context.utilization',
|
|
319
|
+
CONTEXT_BY_TYPE: 'integritystudio.session.context.by_type',
|
|
320
|
+
FREE_SPACE: 'integritystudio.session.free_space',
|
|
321
|
+
STARTS: 'integritystudio.session.starts',
|
|
322
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluation attribute keys, evaluator kinds and cohorts (OBP16). No imports, so
|
|
3
|
+
* scripts needing only the names never load `otel.js` (RDB4); `quality-signals.ts`
|
|
4
|
+
* re-exports it. Kind, cohort and producer were one overloaded field until
|
|
5
|
+
* OBP16; `cohort` is now required so it cannot be omitted again.
|
|
6
|
+
*/
|
|
7
|
+
/** GenAI evaluation event and its `gen_ai.evaluation.*` keys: semantic-conventions-genai, source dated in otel/attributes.ts. */
|
|
8
|
+
export const EVALUATION_RESULT_EVENT = 'gen_ai.evaluation.result';
|
|
9
|
+
/**
|
|
10
|
+
* Stamped on every record (AA3 § Migration) so the next rename of an
|
|
11
|
+
* `integritystudio.evaluation.*` key is a `rename_attributes` entry, not a
|
|
12
|
+
* dual-reader. Bump the version and publish a schema file on any rename or
|
|
13
|
+
* removal; mirrored in the toolkit's `dashboard/scripts/eval-record.ts`.
|
|
14
|
+
*/
|
|
15
|
+
export const EVALUATION_SCHEMA_URL = 'https://integritystudio.ai/schemas/evaluation/1.0.0';
|
|
16
|
+
export const EVALUATOR_KIND = {
|
|
17
|
+
RULE: 'rule',
|
|
18
|
+
LLM: 'llm',
|
|
19
|
+
HUMAN: 'human',
|
|
20
|
+
GROUND_TRUTH: 'ground_truth',
|
|
21
|
+
};
|
|
22
|
+
/** `CANARY` scores are random by design and must be excluded from any analysis of judge behaviour. */
|
|
23
|
+
export const EVALUATION_COHORT = {
|
|
24
|
+
NORMAL: 'normal',
|
|
25
|
+
SEED: 'seed',
|
|
26
|
+
CANARY: 'canary',
|
|
27
|
+
BACKFILL: 'backfill',
|
|
28
|
+
};
|
|
29
|
+
/** `score.unit` (OBP20) and `judge.model` (OBP22) were removed: every score is [0,1], and the judge span carries the model. */
|
|
30
|
+
export const EVALUATION_ATTRS = {
|
|
31
|
+
NAME: 'gen_ai.evaluation.name',
|
|
32
|
+
SCORE_VALUE: 'gen_ai.evaluation.score.value',
|
|
33
|
+
EXPLANATION: 'gen_ai.evaluation.explanation',
|
|
34
|
+
EVALUATOR_KIND: 'integritystudio.evaluation.evaluator.kind',
|
|
35
|
+
COHORT: 'integritystudio.evaluation.cohort',
|
|
36
|
+
PRODUCER: 'integritystudio.evaluation.producer',
|
|
37
|
+
/** Semconv's link to the evaluated response when no span of it exists: the judged turn's last API `message.id`. */
|
|
38
|
+
RESPONSE_ID: 'gen_ai.response.id',
|
|
39
|
+
/**
|
|
40
|
+
* The judge's own `gen_ai.client.operation` span (OBP21), so Gate 3b can read the judge model off it.
|
|
41
|
+
* A vendor link, never the event's parent: semconv parents the event to the operation evaluated, and the judge call is not that.
|
|
42
|
+
*/
|
|
43
|
+
JUDGE_TRACE_ID: 'integritystudio.evaluation.judge.trace_id',
|
|
44
|
+
JUDGE_SPAN_ID: 'integritystudio.evaluation.judge.span_id',
|
|
45
|
+
/** Which turn an LLM judge scored (G3B1): index plus `userText` hash, never the text. */
|
|
46
|
+
TURN_INDEX: 'integritystudio.evaluation.turn.index',
|
|
47
|
+
TURN_USER_TEXT_HASH: 'integritystudio.evaluation.turn.user_text_hash',
|
|
48
|
+
META_ORIGINAL_EVALUATION: 'integritystudio.meta.original_evaluation',
|
|
49
|
+
/** Content-derived, written on both the file and log-event copies so ingest keeps whichever arrives first (HDF12). */
|
|
50
|
+
ID: 'integritystudio.evaluation.id',
|
|
51
|
+
};
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The telemetry JSONL files: their record shapes, the one appender that stamps the
|
|
3
|
+
* writing account, and the SDK exporters that feed it. The detached shipper
|
|
4
|
+
* (span-shipper.ts) reads these files and forwards them.
|
|
5
|
+
*/
|
|
6
|
+
// Before constants.js: TELEMETRY_DIR is fixed when constants loads, and .envrc may set it.
|
|
7
|
+
import '../core/load-envrc.js';
|
|
8
|
+
import * as fs from 'node:fs';
|
|
9
|
+
import * as path from 'node:path';
|
|
10
|
+
import { ExportResultCode } from '@opentelemetry/core';
|
|
11
|
+
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
12
|
+
import { TELEMETRY_DIR } from '../core/constants.js';
|
|
13
|
+
import { ensureDirExists } from '../core/file-utils.js';
|
|
14
|
+
import { telemetryFileName, writeStderr } from '../core/format-utils.js';
|
|
15
|
+
import { resolveKeySecretName } from '../core/identity-key-resolver.js';
|
|
16
|
+
/** A signal prefix must stay an `OtlpSignal`: shipper-cursor.ts finds these files by `OTLP_SIGNALS`, so a drift is a silently unshipped signal. */
|
|
17
|
+
export function telemetryFilePath(prefix, date = new Date()) {
|
|
18
|
+
ensureDirExists(TELEMETRY_DIR);
|
|
19
|
+
return path.join(TELEMETRY_DIR, telemetryFileName(prefix, date));
|
|
20
|
+
}
|
|
21
|
+
let cachedIdentityKeyRef;
|
|
22
|
+
/**
|
|
23
|
+
* The writing account as its Doppler secret name (never the email; it never leaves the
|
|
24
|
+
* machine). The shipper keys each record by this, not by whoever is signed in when it
|
|
25
|
+
* ships — the latter sent the tail of every session across an account switch to the
|
|
26
|
+
* wrong org (TKR6). Exported for `appendEvaluation`, which bypasses these exporters.
|
|
27
|
+
*/
|
|
28
|
+
export function writerIdentityKeyRef() {
|
|
29
|
+
cachedIdentityKeyRef ??= resolveKeySecretName() ?? null;
|
|
30
|
+
return cachedIdentityKeyRef;
|
|
31
|
+
}
|
|
32
|
+
/** Drop the cached stamp. Tests only. */
|
|
33
|
+
export function resetWriterIdentityKeyRef() {
|
|
34
|
+
cachedIdentityKeyRef = undefined;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The one appender for every telemetry record, so the account stamp (TKR6) cannot be
|
|
38
|
+
* forgotten by a writer that bypasses the SDK exporters. Throws on a write failure: the
|
|
39
|
+
* exporters turn that into an ExportResult, appendEvaluation into a metric.
|
|
40
|
+
*/
|
|
41
|
+
export function appendTelemetryRecords(prefix, records, date = new Date()) {
|
|
42
|
+
const identityKeyRef = writerIdentityKeyRef();
|
|
43
|
+
fs.appendFileSync(telemetryFilePath(prefix, date), records.map((record) => `${JSON.stringify({ ...record, identityKeyRef })}\n`).join(''));
|
|
44
|
+
}
|
|
45
|
+
/** `buildRecords` runs inside the try so a serialization failure fails the export rather than throwing into the SDK. */
|
|
46
|
+
function appendJsonl(prefix, signal, buildRecords, resultCallback) {
|
|
47
|
+
try {
|
|
48
|
+
appendTelemetryRecords(prefix, buildRecords());
|
|
49
|
+
resultCallback({ code: ExportResultCode.SUCCESS });
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
writeStderr(`Failed to export ${signal} to file:`, error);
|
|
53
|
+
resultCallback({ code: ExportResultCode.FAILED, error: error });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/** Spans built outside the SDK, stamped with the writing account like every exported record. */
|
|
57
|
+
export function appendSpanRecords(records) {
|
|
58
|
+
appendJsonl('traces', 'spans', () => records, () => { });
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* One SDK exporter per signal. `Batch` is whatever that signal's `export` receives: an array of
|
|
62
|
+
* spans or log records, but a single ResourceMetrics. Nothing is ever buffered, so shutdown and
|
|
63
|
+
* flush have no work.
|
|
64
|
+
*/
|
|
65
|
+
class FileExporter {
|
|
66
|
+
prefix;
|
|
67
|
+
signal;
|
|
68
|
+
toRecords;
|
|
69
|
+
constructor(prefix, signal, toRecords) {
|
|
70
|
+
this.prefix = prefix;
|
|
71
|
+
this.signal = signal;
|
|
72
|
+
this.toRecords = toRecords;
|
|
73
|
+
// Best-effort eager mkdir; first export re-runs this inside appendJsonl's try/catch and reports non-fatally.
|
|
74
|
+
try {
|
|
75
|
+
ensureDirExists(TELEMETRY_DIR);
|
|
76
|
+
}
|
|
77
|
+
catch { /* intentionally swallowed */ }
|
|
78
|
+
}
|
|
79
|
+
export(batch, resultCallback) {
|
|
80
|
+
appendJsonl(this.prefix, this.signal, () => this.toRecords(batch), resultCallback);
|
|
81
|
+
}
|
|
82
|
+
shutdown() {
|
|
83
|
+
return Promise.resolve();
|
|
84
|
+
}
|
|
85
|
+
forceFlush() {
|
|
86
|
+
return Promise.resolve();
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
function serializeSpan(span) {
|
|
90
|
+
const { traceId, spanId } = span.spanContext();
|
|
91
|
+
return {
|
|
92
|
+
traceId,
|
|
93
|
+
spanId,
|
|
94
|
+
parentSpanId: span.parentSpanContext?.spanId,
|
|
95
|
+
name: span.name,
|
|
96
|
+
kind: span.kind,
|
|
97
|
+
startTime: span.startTime,
|
|
98
|
+
endTime: span.endTime,
|
|
99
|
+
duration: span.duration,
|
|
100
|
+
status: span.status,
|
|
101
|
+
attributes: span.attributes,
|
|
102
|
+
events: span.events,
|
|
103
|
+
links: span.links,
|
|
104
|
+
resource: {
|
|
105
|
+
serviceName: span.resource.attributes[ATTR_SERVICE_NAME],
|
|
106
|
+
serviceVersion: span.resource.attributes[ATTR_SERVICE_VERSION],
|
|
107
|
+
},
|
|
108
|
+
instrumentationScope: {
|
|
109
|
+
name: span.instrumentationScope.name,
|
|
110
|
+
version: span.instrumentationScope.version,
|
|
111
|
+
},
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
export class FileSpanExporter extends FileExporter {
|
|
115
|
+
constructor() {
|
|
116
|
+
super('traces', 'spans', (spans) => spans.map(serializeSpan));
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
export const LOGS_FILE_PREFIX = 'logs';
|
|
120
|
+
function serializeLog(log) {
|
|
121
|
+
return {
|
|
122
|
+
timestamp: log.hrTime,
|
|
123
|
+
observedTimestamp: log.hrTimeObserved,
|
|
124
|
+
severityNumber: log.severityNumber,
|
|
125
|
+
severityText: log.severityText,
|
|
126
|
+
body: log.body,
|
|
127
|
+
eventName: log.eventName ?? undefined,
|
|
128
|
+
attributes: log.attributes,
|
|
129
|
+
traceId: log.spanContext?.traceId,
|
|
130
|
+
spanId: log.spanContext?.spanId,
|
|
131
|
+
resource: {
|
|
132
|
+
serviceName: log.resource.attributes[ATTR_SERVICE_NAME],
|
|
133
|
+
serviceVersion: log.resource.attributes[ATTR_SERVICE_VERSION],
|
|
134
|
+
},
|
|
135
|
+
instrumentationScope: {
|
|
136
|
+
name: log.instrumentationScope.name,
|
|
137
|
+
version: log.instrumentationScope.version,
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
/** Every severity is written; the log file is the record of what a hook said, not only of what went wrong. */
|
|
142
|
+
export class FileLogExporter extends FileExporter {
|
|
143
|
+
constructor() {
|
|
144
|
+
super(LOGS_FILE_PREFIX, 'logs', (logs) => logs.map(serializeLog));
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/** Drops the empty `description` and `advice` the SDK puts on every descriptor; the OTLP serializer reads neither when absent. */
|
|
148
|
+
function compactDescriptor({ description, advice, ...rest }) {
|
|
149
|
+
return {
|
|
150
|
+
...rest,
|
|
151
|
+
...(description ? { description } : {}),
|
|
152
|
+
...(advice && Object.keys(advice).length > 0 ? { advice } : {}),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
function serializeMetrics(metrics) {
|
|
156
|
+
return {
|
|
157
|
+
resource: { attributes: metrics.resource.attributes },
|
|
158
|
+
scopeMetrics: metrics.scopeMetrics.map((scopeMetric) => ({
|
|
159
|
+
...scopeMetric,
|
|
160
|
+
metrics: scopeMetric.metrics.map((metric) => ({ ...metric, descriptor: compactDescriptor(metric.descriptor) })),
|
|
161
|
+
})),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
/** The SDK ships no metrics file exporter of its own. */
|
|
165
|
+
export class FileMetricExporter extends FileExporter {
|
|
166
|
+
constructor() {
|
|
167
|
+
super('metrics', 'metrics', (metrics) => [serializeMetrics(metrics)]);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `hostDetector` with `host.id` remembered across processes (HSF25): on darwin it
|
|
3
|
+
* execs `ioreg` and holds the resource `asyncAttributesPending`, costing every hook
|
|
4
|
+
* process a median 17-28 ms. A record is trusted only while `os.arch()` matches and
|
|
5
|
+
* it is younger than `HOST_RECORD_TTL_MS`; otherwise `hostDetector` runs and rewrites it.
|
|
6
|
+
*/
|
|
7
|
+
import { arch, hostname } from 'node:os';
|
|
8
|
+
import { join } from 'node:path';
|
|
9
|
+
import { hostDetector } from '@opentelemetry/resources';
|
|
10
|
+
import { loadJson, writeJsonAtomic } from '../core/file-utils.js';
|
|
11
|
+
import { TIME_MS } from '../core/units.js';
|
|
12
|
+
import { hostStateDir } from '../session/session-state.js';
|
|
13
|
+
/** The registered `host.*` keys `hostDetector` fills (semconv, Development). */
|
|
14
|
+
const HOST_ATTRS = {
|
|
15
|
+
NAME: 'host.name',
|
|
16
|
+
ARCH: 'host.arch',
|
|
17
|
+
ID: 'host.id',
|
|
18
|
+
};
|
|
19
|
+
const HOST_RECORD_FILE = 'host-resource.json';
|
|
20
|
+
/** How long a recorded id is trusted before `hostDetector` is asked again. */
|
|
21
|
+
export const HOST_RECORD_TTL_MS = TIME_MS.WEEK;
|
|
22
|
+
/** The record, or undefined when there is none worth trusting now. */
|
|
23
|
+
function readHostRecord(recordPath) {
|
|
24
|
+
const record = loadJson(recordPath);
|
|
25
|
+
if (typeof record?.id !== 'string' || !record.id ||
|
|
26
|
+
typeof record.arch !== 'string' ||
|
|
27
|
+
record.nodeArch !== arch() ||
|
|
28
|
+
typeof record.recordedAtMs !== 'number' ||
|
|
29
|
+
Date.now() - record.recordedAtMs >= HOST_RECORD_TTL_MS) {
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
return record;
|
|
33
|
+
}
|
|
34
|
+
/** A host detector that reads and writes its record at `recordPath`. */
|
|
35
|
+
export function createHostResourceDetector(recordPath = join(hostStateDir(), HOST_RECORD_FILE)) {
|
|
36
|
+
return {
|
|
37
|
+
detect(config) {
|
|
38
|
+
const record = readHostRecord(recordPath);
|
|
39
|
+
if (record) {
|
|
40
|
+
return {
|
|
41
|
+
attributes: {
|
|
42
|
+
[HOST_ATTRS.NAME]: hostname(),
|
|
43
|
+
[HOST_ATTRS.ARCH]: record.arch,
|
|
44
|
+
[HOST_ATTRS.ID]: record.id,
|
|
45
|
+
},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
const detected = hostDetector.detect(config);
|
|
49
|
+
const attributes = { ...detected.attributes };
|
|
50
|
+
const hostArch = attributes[HOST_ATTRS.ARCH];
|
|
51
|
+
attributes[HOST_ATTRS.ID] = Promise.resolve(attributes[HOST_ATTRS.ID]).then((id) => {
|
|
52
|
+
if (typeof id === 'string' && id && typeof hostArch === 'string') {
|
|
53
|
+
writeJsonAtomic(recordPath, { id, arch: hostArch, nodeArch: arch(), recordedAtMs: Date.now() }, { silent: true });
|
|
54
|
+
}
|
|
55
|
+
return id;
|
|
56
|
+
});
|
|
57
|
+
return { ...detected, attributes };
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
export const hostResourceDetector = createHostResourceDetector();
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `otelHeadersHelper` for Claude Code's own OTLP exporters (TKR1): Claude Code
|
|
3
|
+
* runs it periodically and merges the printed JSON headers into its export, so
|
|
4
|
+
* the credential follows the signed-in account. Stdout is a JSON object of
|
|
5
|
+
* string headers, nothing else; no resolvable key prints `{}` so the export is
|
|
6
|
+
* refused at ingest rather than landing in the wrong org.
|
|
7
|
+
*/
|
|
8
|
+
import { bearerHeaders } from '../core/http-utils.js';
|
|
9
|
+
import { resolveOtlpApiKey } from '../core/identity-key-resolver.js';
|
|
10
|
+
/** Headers for a resolved key, or an empty bag when there is none. */
|
|
11
|
+
export function buildOtelHeaders(apiKey) {
|
|
12
|
+
return apiKey ? bearerHeaders(apiKey) : {};
|
|
13
|
+
}
|
|
14
|
+
export function main() {
|
|
15
|
+
const key = resolveOtlpApiKey();
|
|
16
|
+
process.stdout.write(JSON.stringify(buildOtelHeaders(key)));
|
|
17
|
+
}
|
|
18
|
+
if (import.meta.main) {
|
|
19
|
+
try {
|
|
20
|
+
main();
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
// An empty bag is the fail-closed answer; the exporter's 401 is the symptom.
|
|
24
|
+
process.stdout.write('{}');
|
|
25
|
+
}
|
|
26
|
+
}
|