observability-toolkit-hooks 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG.md +144 -0
- package/LICENSE +95 -0
- package/README.md +201 -2
- package/dist/handlers/compact.js +38 -0
- package/dist/handlers/mcp-status.js +66 -0
- package/dist/handlers/notification.js +45 -0
- package/dist/handlers/post-tool-agent.js +157 -0
- package/dist/handlers/post-tool-changelog-sync.js +148 -0
- package/dist/handlers/post-tool-code-structure.js +116 -0
- package/dist/handlers/post-tool-edit-ledger.js +15 -0
- package/dist/handlers/post-tool-failure.js +83 -0
- package/dist/handlers/post-tool-git.js +79 -0
- package/dist/handlers/post-tool-mcp.js +105 -0
- package/dist/handlers/post-tool-outcome.js +48 -0
- package/dist/handlers/post-tool-task-state.js +74 -0
- package/dist/handlers/post-tool.js +132 -0
- package/dist/handlers/pre-tool-agent.js +192 -0
- package/dist/handlers/pre-tool.js +218 -0
- package/dist/handlers/session-end.js +39 -0
- package/dist/handlers/session-start.js +262 -0
- package/dist/handlers/stop-error-reminder.js +195 -0
- package/dist/handlers/stop-quality-evaluation.js +58 -0
- package/dist/handlers/stop-session-summary.js +62 -0
- package/dist/handlers/stop-token-metrics.js +83 -0
- package/dist/handlers/stop.js +25 -0
- package/dist/handlers/subagent.js +105 -0
- package/dist/handlers/tool-event-router.js +39 -0
- package/dist/handlers/user-prompt.js +218 -0
- package/dist/hook-runner.js +10 -0
- package/dist/hook-runner.js.map +7 -0
- package/dist/lib/agent/agent-context.js +123 -0
- package/dist/lib/agent/agent-evaluations.js +155 -0
- package/dist/lib/agent/agent-file-budget.js +65 -0
- package/dist/lib/agent/agent-sources.js +165 -0
- package/dist/lib/agent/synth-spans.js +51 -0
- package/dist/lib/core/bounded-label.js +6 -0
- package/dist/lib/core/category-rules.js +11 -0
- package/dist/lib/core/child-result.js +19 -0
- package/dist/lib/core/config-dir-recovery.js +22 -0
- package/dist/lib/core/constants.js +171 -0
- package/dist/lib/core/detached-runner.js +55 -0
- package/dist/lib/core/file-utils.js +353 -0
- package/dist/lib/core/format-utils.js +51 -0
- package/dist/lib/core/http-utils.js +74 -0
- package/dist/lib/core/http1-fetch.js +14 -0
- package/dist/lib/core/identity-key-resolver.js +149 -0
- package/dist/lib/core/jsonl-tail.js +207 -0
- package/dist/lib/core/load-envrc.js +72 -0
- package/dist/lib/core/parsers.js +43 -0
- package/dist/lib/core/regex.js +27 -0
- package/dist/lib/core/settings-env.js +31 -0
- package/dist/lib/core/shell-command.js +122 -0
- package/dist/lib/core/units.js +97 -0
- package/dist/lib/git/git-cache.js +158 -0
- package/dist/lib/git/git-commit.js +189 -0
- package/dist/lib/git/git-exec.js +30 -0
- package/dist/lib/mcp/mcp-completeness.js +40 -0
- package/dist/lib/mcp/mcp-config.js +118 -0
- package/dist/lib/mcp/mcp-failure-latch.js +22 -0
- package/dist/lib/mcp/mcp-identity-launcher.js +43 -0
- package/dist/lib/otel/attributes.js +322 -0
- package/dist/lib/otel/evaluation-attrs.js +51 -0
- package/dist/lib/otel/file-exporters.js +169 -0
- package/dist/lib/otel/host-resource.js +61 -0
- package/dist/lib/otel/otel-headers-helper.js +26 -0
- package/dist/lib/otel/otel-monitor.js +181 -0
- package/dist/lib/otel/otel.js +235 -0
- package/dist/lib/otel/otlp-headers.js +101 -0
- package/dist/lib/otel/otlp-post.js +122 -0
- package/dist/lib/otel/otlp-serialize.js +142 -0
- package/dist/lib/otel/otlp-signal.js +2 -0
- package/dist/lib/otel/remote-spans.js +142 -0
- package/dist/lib/otel/service-identity.js +10 -0
- package/dist/lib/otel/telemetry-alerts.js +43 -0
- package/dist/lib/otel/token-metrics.js +122 -0
- package/dist/lib/otel/trace-context.js +81 -0
- package/dist/lib/quality/judge-meta-eval.js +70 -0
- package/dist/lib/quality/judge-prompts.js +104 -0
- package/dist/lib/quality/judge-providers.js +193 -0
- package/dist/lib/quality/judge-transport.js +93 -0
- package/dist/lib/quality/llm-judge.js +170 -0
- package/dist/lib/quality/quality-budget.js +99 -0
- package/dist/lib/quality/quality-degradation.js +156 -0
- package/dist/lib/quality/quality-eval-handoff.js +74 -0
- package/dist/lib/quality/quality-evaluator.js +178 -0
- package/dist/lib/quality/quality-sampler.js +43 -0
- package/dist/lib/quality/quality-signals.js +150 -0
- package/dist/lib/runtime/categorizers.js +101 -0
- package/dist/lib/runtime/hook-input.js +16 -0
- package/dist/lib/runtime/hook-output.js +50 -0
- package/dist/lib/runtime/output-analyzer.js +160 -0
- package/dist/lib/runtime/run-handlers.js +32 -0
- package/dist/lib/runtime/tool-input.js +111 -0
- package/dist/lib/runtime/tool-output.js +44 -0
- package/dist/lib/runtime/unknown-tool.js +32 -0
- package/dist/lib/session/edit-ledger.js +29 -0
- package/dist/lib/session/forked-skill.js +39 -0
- package/dist/lib/session/handoffs.js +77 -0
- package/dist/lib/session/session-state.js +169 -0
- package/dist/lib/session/skill-rules.js +196 -0
- package/dist/lib/session/task-tracker.js +83 -0
- package/dist/lib/shipper/shipper-credentials.js +38 -0
- package/dist/lib/shipper/shipper-cursor.js +89 -0
- package/dist/lib/shipper/shipper-status.js +174 -0
- package/dist/lib/shipper/span-shipper.js +272 -0
- package/dist/lib/shipper/telemetry-shipper.js +39 -0
- package/dist/lib/transcript/agent-transcript-metrics.js +34 -0
- package/dist/lib/transcript/context-breakdown.js +134 -0
- package/dist/lib/transcript/context-history.js +69 -0
- package/dist/lib/transcript/context-tracker.js +226 -0
- package/dist/lib/transcript/message-count.js +49 -0
- package/dist/lib/transcript/model-ids.js +36 -0
- package/dist/lib/transcript/model-pricing.js +153 -0
- package/dist/lib/transcript/token-metrics-checkpoint.js +152 -0
- package/dist/lib/transcript/tool-call-durations.js +64 -0
- package/dist/lib/transcript/transcript-entry.js +67 -0
- package/dist/lib/transcript/transcript-fold.js +130 -0
- package/dist/lib/transcript/transcript-layout.js +39 -0
- package/dist/lib/transcript/transcript-parser.js +117 -0
- package/dist/runner/agent-context-4XTCLMLZ.js +23 -0
- package/dist/runner/agent-context-4XTCLMLZ.js.map +7 -0
- package/dist/runner/agent-file-budget-Q33ESBBE.js +68 -0
- package/dist/runner/agent-file-budget-Q33ESBBE.js.map +7 -0
- package/dist/runner/agent-transcript-metrics-W5CZZFMI.js +39 -0
- package/dist/runner/agent-transcript-metrics-W5CZZFMI.js.map +7 -0
- package/dist/runner/chunk-2NTNAJZE.js +49 -0
- package/dist/runner/chunk-2NTNAJZE.js.map +7 -0
- package/dist/runner/chunk-4KWBQTTG.js +78 -0
- package/dist/runner/chunk-4KWBQTTG.js.map +7 -0
- package/dist/runner/chunk-52RY6LQT.js +699 -0
- package/dist/runner/chunk-52RY6LQT.js.map +7 -0
- package/dist/runner/chunk-7OM5CYES.js +8695 -0
- package/dist/runner/chunk-7OM5CYES.js.map +7 -0
- package/dist/runner/chunk-7PEOKGYR.js +1568 -0
- package/dist/runner/chunk-7PEOKGYR.js.map +7 -0
- package/dist/runner/chunk-AGSGKELQ.js +126 -0
- package/dist/runner/chunk-AGSGKELQ.js.map +7 -0
- package/dist/runner/chunk-AVM7PJFW.js +12 -0
- package/dist/runner/chunk-AVM7PJFW.js.map +7 -0
- package/dist/runner/chunk-C3WHUMU5.js +178 -0
- package/dist/runner/chunk-C3WHUMU5.js.map +7 -0
- package/dist/runner/chunk-DGB4NSSD.js +157 -0
- package/dist/runner/chunk-DGB4NSSD.js.map +7 -0
- package/dist/runner/chunk-DNSTWC26.js +151 -0
- package/dist/runner/chunk-DNSTWC26.js.map +7 -0
- package/dist/runner/chunk-FDJJ3TYU.js +449 -0
- package/dist/runner/chunk-FDJJ3TYU.js.map +7 -0
- package/dist/runner/chunk-FGKYDQHZ.js +137 -0
- package/dist/runner/chunk-FGKYDQHZ.js.map +7 -0
- package/dist/runner/chunk-GG2I4TGI.js +582 -0
- package/dist/runner/chunk-GG2I4TGI.js.map +7 -0
- package/dist/runner/chunk-GNNKUMRF.js +60 -0
- package/dist/runner/chunk-GNNKUMRF.js.map +7 -0
- package/dist/runner/chunk-I7TD2QVR.js +118 -0
- package/dist/runner/chunk-I7TD2QVR.js.map +7 -0
- package/dist/runner/chunk-IJ4SLUBV.js +78 -0
- package/dist/runner/chunk-IJ4SLUBV.js.map +7 -0
- package/dist/runner/chunk-IY7AP7NA.js +32 -0
- package/dist/runner/chunk-IY7AP7NA.js.map +7 -0
- package/dist/runner/chunk-J3F3Z3EQ.js +177 -0
- package/dist/runner/chunk-J3F3Z3EQ.js.map +7 -0
- package/dist/runner/chunk-JLEO5OHU.js +104 -0
- package/dist/runner/chunk-JLEO5OHU.js.map +7 -0
- package/dist/runner/chunk-JRNNLOPR.js +168 -0
- package/dist/runner/chunk-JRNNLOPR.js.map +7 -0
- package/dist/runner/chunk-KGSA5JYP.js +82 -0
- package/dist/runner/chunk-KGSA5JYP.js.map +7 -0
- package/dist/runner/chunk-LBCZAYFR.js +125 -0
- package/dist/runner/chunk-LBCZAYFR.js.map +7 -0
- package/dist/runner/chunk-LHWRXY2G.js +41 -0
- package/dist/runner/chunk-LHWRXY2G.js.map +7 -0
- package/dist/runner/chunk-LQ2LGUYJ.js +41 -0
- package/dist/runner/chunk-LQ2LGUYJ.js.map +7 -0
- package/dist/runner/chunk-LYY6RCIE.js +158 -0
- package/dist/runner/chunk-LYY6RCIE.js.map +7 -0
- package/dist/runner/chunk-M4R4QIQH.js +41 -0
- package/dist/runner/chunk-M4R4QIQH.js.map +7 -0
- package/dist/runner/chunk-MC75VXMX.js +207 -0
- package/dist/runner/chunk-MC75VXMX.js.map +7 -0
- package/dist/runner/chunk-NKZU4RRG.js +104 -0
- package/dist/runner/chunk-NKZU4RRG.js.map +7 -0
- package/dist/runner/chunk-PJHWHJ3K.js +56 -0
- package/dist/runner/chunk-PJHWHJ3K.js.map +7 -0
- package/dist/runner/chunk-RIGVYJ2O.js +114 -0
- package/dist/runner/chunk-RIGVYJ2O.js.map +7 -0
- package/dist/runner/chunk-RWRGC7ZK.js +93 -0
- package/dist/runner/chunk-RWRGC7ZK.js.map +7 -0
- package/dist/runner/chunk-SRMAJQMV.js +54 -0
- package/dist/runner/chunk-SRMAJQMV.js.map +7 -0
- package/dist/runner/chunk-TRMFD36J.js +47 -0
- package/dist/runner/chunk-TRMFD36J.js.map +7 -0
- package/dist/runner/chunk-U4IOZYQ3.js +21 -0
- package/dist/runner/chunk-U4IOZYQ3.js.map +7 -0
- package/dist/runner/chunk-UEXDE3WP.js +83 -0
- package/dist/runner/chunk-UEXDE3WP.js.map +7 -0
- package/dist/runner/chunk-UW3GQZSM.js +50 -0
- package/dist/runner/chunk-UW3GQZSM.js.map +7 -0
- package/dist/runner/chunk-V2GSP6VR.js +46 -0
- package/dist/runner/chunk-V2GSP6VR.js.map +7 -0
- package/dist/runner/chunk-VOV4Y5VU.js +81 -0
- package/dist/runner/chunk-VOV4Y5VU.js.map +7 -0
- package/dist/runner/chunk-VT5T6OEI.js +65 -0
- package/dist/runner/chunk-VT5T6OEI.js.map +7 -0
- package/dist/runner/chunk-VUYVFWIN.js +12 -0
- package/dist/runner/chunk-VUYVFWIN.js.map +7 -0
- package/dist/runner/chunk-VVGOPPEN.js +36 -0
- package/dist/runner/chunk-VVGOPPEN.js.map +7 -0
- package/dist/runner/chunk-VXFSLM6P.js +14 -0
- package/dist/runner/chunk-VXFSLM6P.js.map +7 -0
- package/dist/runner/chunk-W3EK7JUT.js +26125 -0
- package/dist/runner/chunk-W3EK7JUT.js.map +7 -0
- package/dist/runner/chunk-WJHNCOUF.js +52 -0
- package/dist/runner/chunk-WJHNCOUF.js.map +7 -0
- package/dist/runner/chunk-WU7ANGCW.js +269 -0
- package/dist/runner/chunk-WU7ANGCW.js.map +7 -0
- package/dist/runner/chunk-X7XLMSZX.js +44 -0
- package/dist/runner/chunk-X7XLMSZX.js.map +7 -0
- package/dist/runner/chunk-XSO3B4QN.js +124 -0
- package/dist/runner/chunk-XSO3B4QN.js.map +7 -0
- package/dist/runner/chunk-YGFN3LN2.js +174 -0
- package/dist/runner/chunk-YGFN3LN2.js.map +7 -0
- package/dist/runner/chunk-YHYDBLVB.js +33 -0
- package/dist/runner/chunk-YHYDBLVB.js.map +7 -0
- package/dist/runner/compact-36NP3PKT.js +60 -0
- package/dist/runner/compact-36NP3PKT.js.map +7 -0
- package/dist/runner/getMachineId-bsd-AGENNHN6.js +42 -0
- package/dist/runner/getMachineId-bsd-AGENNHN6.js.map +7 -0
- package/dist/runner/getMachineId-bsd-I5WZGCL4.js +33 -0
- package/dist/runner/getMachineId-bsd-I5WZGCL4.js.map +7 -0
- package/dist/runner/getMachineId-bsd-ZXK225P7.js +33 -0
- package/dist/runner/getMachineId-bsd-ZXK225P7.js.map +7 -0
- package/dist/runner/getMachineId-darwin-47GDLEXL.js +42 -0
- package/dist/runner/getMachineId-darwin-47GDLEXL.js.map +7 -0
- package/dist/runner/getMachineId-darwin-5CRLMDD6.js +33 -0
- package/dist/runner/getMachineId-darwin-5CRLMDD6.js.map +7 -0
- package/dist/runner/getMachineId-darwin-ADG4I32N.js +33 -0
- package/dist/runner/getMachineId-darwin-ADG4I32N.js.map +7 -0
- package/dist/runner/getMachineId-linux-5KVS3T2M.js +27 -0
- package/dist/runner/getMachineId-linux-5KVS3T2M.js.map +7 -0
- package/dist/runner/getMachineId-linux-GDFB5GSO.js +34 -0
- package/dist/runner/getMachineId-linux-GDFB5GSO.js.map +7 -0
- package/dist/runner/getMachineId-linux-NQMWCRO3.js +27 -0
- package/dist/runner/getMachineId-linux-NQMWCRO3.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-5PFIGMVX.js +18 -0
- package/dist/runner/getMachineId-unsupported-5PFIGMVX.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-DW27K3ZX.js +18 -0
- package/dist/runner/getMachineId-unsupported-DW27K3ZX.js.map +7 -0
- package/dist/runner/getMachineId-unsupported-XV3FHTHD.js +25 -0
- package/dist/runner/getMachineId-unsupported-XV3FHTHD.js.map +7 -0
- package/dist/runner/getMachineId-win-5JVMW3EI.js +44 -0
- package/dist/runner/getMachineId-win-5JVMW3EI.js.map +7 -0
- package/dist/runner/getMachineId-win-SNESQF7R.js +35 -0
- package/dist/runner/getMachineId-win-SNESQF7R.js.map +7 -0
- package/dist/runner/getMachineId-win-UJYHSBM2.js +35 -0
- package/dist/runner/getMachineId-win-UJYHSBM2.js.map +7 -0
- package/dist/runner/git-commit-BA53DOJQ.js +30 -0
- package/dist/runner/git-commit-BA53DOJQ.js.map +7 -0
- package/dist/runner/hook-runner-JHH6PGKE.js +220 -0
- package/dist/runner/hook-runner-JHH6PGKE.js.map +7 -0
- package/dist/runner/judge-transport-R4TF5LEH.js +91 -0
- package/dist/runner/judge-transport-R4TF5LEH.js.map +7 -0
- package/dist/runner/mcp-status-QEUH2KGX.js +89 -0
- package/dist/runner/mcp-status-QEUH2KGX.js.map +7 -0
- package/dist/runner/notification-FKMBCSXF.js +69 -0
- package/dist/runner/notification-FKMBCSXF.js.map +7 -0
- package/dist/runner/post-tool-3MRHTBKH.js +192 -0
- package/dist/runner/post-tool-3MRHTBKH.js.map +7 -0
- package/dist/runner/post-tool-agent-BB34MNHI.js +370 -0
- package/dist/runner/post-tool-agent-BB34MNHI.js.map +7 -0
- package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js +147 -0
- package/dist/runner/post-tool-changelog-sync-OP2TT2I7.js.map +7 -0
- package/dist/runner/post-tool-code-structure-LE7J2SC2.js +130 -0
- package/dist/runner/post-tool-code-structure-LE7J2SC2.js.map +7 -0
- package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js +30 -0
- package/dist/runner/post-tool-edit-ledger-IJMEYHKF.js.map +7 -0
- package/dist/runner/post-tool-failure-XTH2I2Z7.js +111 -0
- package/dist/runner/post-tool-failure-XTH2I2Z7.js.map +7 -0
- package/dist/runner/post-tool-git-E46HZGZO.js +94 -0
- package/dist/runner/post-tool-git-E46HZGZO.js.map +7 -0
- package/dist/runner/post-tool-mcp-AMCDY5GH.js +144 -0
- package/dist/runner/post-tool-mcp-AMCDY5GH.js.map +7 -0
- package/dist/runner/post-tool-task-state-TLDOI33H.js +100 -0
- package/dist/runner/post-tool-task-state-TLDOI33H.js.map +7 -0
- package/dist/runner/pre-tool-SQE2FZQO.js +245 -0
- package/dist/runner/pre-tool-SQE2FZQO.js.map +7 -0
- package/dist/runner/pre-tool-agent-OZUPVFUI.js +218 -0
- package/dist/runner/pre-tool-agent-OZUPVFUI.js.map +7 -0
- package/dist/runner/quality-evaluator-XYNDKDRG.js +901 -0
- package/dist/runner/quality-evaluator-XYNDKDRG.js.map +7 -0
- package/dist/runner/session-end-3MJTDLLA.js +57 -0
- package/dist/runner/session-end-3MJTDLLA.js.map +7 -0
- package/dist/runner/session-start-5ZMCLKBS.js +352 -0
- package/dist/runner/session-start-5ZMCLKBS.js.map +7 -0
- package/dist/runner/shipper-status-MADX6SPG.js +37 -0
- package/dist/runner/shipper-status-MADX6SPG.js.map +7 -0
- package/dist/runner/span-shipper-4HQDHC47.js +3408 -0
- package/dist/runner/span-shipper-4HQDHC47.js.map +7 -0
- package/dist/runner/stop-RTKSIYJV.js +596 -0
- package/dist/runner/stop-RTKSIYJV.js.map +7 -0
- package/dist/runner/subagent-CBCG4OLZ.js +116 -0
- package/dist/runner/subagent-CBCG4OLZ.js.map +7 -0
- package/dist/runner/token-metrics-AD2RAFGM.js +31 -0
- package/dist/runner/token-metrics-AD2RAFGM.js.map +7 -0
- package/dist/runner/user-prompt-T5NZ2UJD.js +376 -0
- package/dist/runner/user-prompt-T5NZ2UJD.js.map +7 -0
- package/package.json +70 -5
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stored telemetry records → OTLP protobuf request bodies. The JSONL records were
|
|
3
|
+
* never SDK objects, so each is duck-typed into the shape the otlp-transformer
|
|
4
|
+
* serializers read.
|
|
5
|
+
*/
|
|
6
|
+
import { SpanKind, SpanStatusCode, TraceFlags } from '@opentelemetry/api';
|
|
7
|
+
import { ProtobufTraceSerializer, ProtobufLogsSerializer, ProtobufMetricsSerializer, } from '@opentelemetry/otlp-transformer';
|
|
8
|
+
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
9
|
+
import { SHIPPER_LIMITS } from '../core/constants.js';
|
|
10
|
+
import { isJsonObject } from '../core/file-utils.js';
|
|
11
|
+
import { getServiceName, SERVICE_VERSION } from './service-identity.js';
|
|
12
|
+
const DEFAULT_SCOPE = { name: 'claude-hooks', version: SERVICE_VERSION };
|
|
13
|
+
function sampledCtx(traceId, spanId) {
|
|
14
|
+
return { traceId, spanId, traceFlags: TraceFlags.SAMPLED, isRemote: false };
|
|
15
|
+
}
|
|
16
|
+
function resourceAttrs(resource) {
|
|
17
|
+
return {
|
|
18
|
+
[ATTR_SERVICE_NAME]: resource?.serviceName ?? getServiceName(),
|
|
19
|
+
[ATTR_SERVICE_VERSION]: resource?.serviceVersion ?? SERVICE_VERSION,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/** Duck-typed ReadableSpan carrying both the current and nested sdk-trace-base field names. */
|
|
23
|
+
export function exportedSpanToReadable(s) {
|
|
24
|
+
const spanCtx = sampledCtx(s.traceId, s.spanId);
|
|
25
|
+
const instrumentationScope = {
|
|
26
|
+
name: s.instrumentationScope?.name ?? DEFAULT_SCOPE.name,
|
|
27
|
+
version: s.instrumentationScope?.version ?? DEFAULT_SCOPE.version,
|
|
28
|
+
};
|
|
29
|
+
return {
|
|
30
|
+
name: s.name,
|
|
31
|
+
kind: s.kind ?? SpanKind.INTERNAL,
|
|
32
|
+
spanContext: () => spanCtx,
|
|
33
|
+
parentSpanId: s.parentSpanId,
|
|
34
|
+
parentSpanContext: s.parentSpanId ? sampledCtx(s.traceId, s.parentSpanId) : undefined,
|
|
35
|
+
startTime: s.startTime,
|
|
36
|
+
endTime: s.endTime,
|
|
37
|
+
duration: s.duration,
|
|
38
|
+
status: s.status ?? { code: SpanStatusCode.UNSET },
|
|
39
|
+
attributes: s.attributes ?? {},
|
|
40
|
+
events: s.events ?? [],
|
|
41
|
+
links: s.links ?? [],
|
|
42
|
+
resource: { attributes: resourceAttrs(s.resource), merge: () => null },
|
|
43
|
+
instrumentationLibrary: instrumentationScope,
|
|
44
|
+
instrumentationScope,
|
|
45
|
+
droppedAttributesCount: 0,
|
|
46
|
+
droppedEventsCount: 0,
|
|
47
|
+
droppedLinksCount: 0,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
export function exportedLogToReadable(l) {
|
|
51
|
+
const instrumentationScope = l.instrumentationScope ?? { name: getServiceName(), version: SERVICE_VERSION };
|
|
52
|
+
return {
|
|
53
|
+
hrTime: l.timestamp,
|
|
54
|
+
hrTimeObserved: l.observedTimestamp ?? l.timestamp,
|
|
55
|
+
severityNumber: l.severityNumber,
|
|
56
|
+
severityText: l.severityText,
|
|
57
|
+
body: l.body,
|
|
58
|
+
eventName: l.eventName,
|
|
59
|
+
attributes: l.attributes ?? {},
|
|
60
|
+
droppedAttributesCount: 0,
|
|
61
|
+
spanContext: l.traceId && l.spanId ? sampledCtx(l.traceId, l.spanId) : undefined,
|
|
62
|
+
resource: { attributes: resourceAttrs(l.resource) },
|
|
63
|
+
instrumentationScope,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Records carried by `bodies`: what a byte-offset cursor may advance past once they are delivered. */
|
|
67
|
+
export function recordsCarried(bodies) {
|
|
68
|
+
return bodies.reduce((sum, body) => sum + body.recordCount, 0);
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Protobuf defines concatenation as a merge (repeated fields append), so N request
|
|
72
|
+
* bodies joined decode as one request carrying N `resource_metrics`, which the
|
|
73
|
+
* serializer cannot produce itself. A single message over the cap stays whole.
|
|
74
|
+
*/
|
|
75
|
+
export function packBodies(bodies, maxBytes) {
|
|
76
|
+
const packed = [];
|
|
77
|
+
let group = [];
|
|
78
|
+
let groupBytes = 0;
|
|
79
|
+
const flush = () => {
|
|
80
|
+
if (group.length === 0)
|
|
81
|
+
return;
|
|
82
|
+
if (group.length === 1) {
|
|
83
|
+
packed.push(...group);
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
const joined = new Uint8Array(groupBytes);
|
|
87
|
+
let offset = 0;
|
|
88
|
+
for (const { bytes } of group) {
|
|
89
|
+
joined.set(bytes, offset);
|
|
90
|
+
offset += bytes.byteLength;
|
|
91
|
+
}
|
|
92
|
+
packed.push({ bytes: joined, recordCount: recordsCarried(group) });
|
|
93
|
+
}
|
|
94
|
+
group = [];
|
|
95
|
+
groupBytes = 0;
|
|
96
|
+
};
|
|
97
|
+
for (const body of bodies) {
|
|
98
|
+
if (groupBytes + body.bytes.byteLength > maxBytes)
|
|
99
|
+
flush();
|
|
100
|
+
group.push(body);
|
|
101
|
+
groupBytes += body.bytes.byteLength;
|
|
102
|
+
}
|
|
103
|
+
flush();
|
|
104
|
+
return packed;
|
|
105
|
+
}
|
|
106
|
+
const RECORDS_PER_METRICS_BODY = 1;
|
|
107
|
+
/** The fields `serializeBatch` cannot do without; it casts on the strength of this check. */
|
|
108
|
+
export function isStoredRecord(signal, r) {
|
|
109
|
+
if (!isJsonObject(r))
|
|
110
|
+
return false;
|
|
111
|
+
switch (signal) {
|
|
112
|
+
case 'traces':
|
|
113
|
+
return typeof r.traceId === 'string' && typeof r.spanId === 'string';
|
|
114
|
+
case 'logs':
|
|
115
|
+
return Array.isArray(r.timestamp);
|
|
116
|
+
case 'metrics':
|
|
117
|
+
return Array.isArray(r.scopeMetrics);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
export function serializeBatch(signal, records) {
|
|
121
|
+
switch (signal) {
|
|
122
|
+
case 'traces': {
|
|
123
|
+
const body = ProtobufTraceSerializer.serializeRequest(records.map(exportedSpanToReadable));
|
|
124
|
+
return body ? [{ bytes: body, recordCount: records.length }] : [];
|
|
125
|
+
}
|
|
126
|
+
case 'logs': {
|
|
127
|
+
const body = ProtobufLogsSerializer.serializeRequest(records.map(exportedLogToReadable));
|
|
128
|
+
return body ? [{ bytes: body, recordCount: records.length }] : [];
|
|
129
|
+
}
|
|
130
|
+
case 'metrics': {
|
|
131
|
+
// One body per record: the resource carries process.pid, so records from
|
|
132
|
+
// spawn-per-event hooks almost never share one. `packBodies` joins them per POST.
|
|
133
|
+
const bodies = [];
|
|
134
|
+
for (const { resource, scopeMetrics } of records) {
|
|
135
|
+
const body = ProtobufMetricsSerializer.serializeRequest({ resource, scopeMetrics });
|
|
136
|
+
if (body)
|
|
137
|
+
bodies.push({ bytes: body, recordCount: RECORDS_PER_METRICS_BODY });
|
|
138
|
+
}
|
|
139
|
+
return packBodies(bodies, SHIPPER_LIMITS.MAX_BYTES_PER_POST);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one reader of obtool-api `/v1/traces`, for detached jobs and skills (never an event
|
|
3
|
+
* hook: the key lookup can spawn Doppler).
|
|
4
|
+
*
|
|
5
|
+
* The store is org-scoped by API key and holds only what the shipper has uploaded. A read
|
|
6
|
+
* with the signed-in account's key misses spans written under another account;
|
|
7
|
+
* `fetchTraceRowsEveryOrg` reads every mapped org. Whole-number
|
|
8
|
+
* attribute values come back as strings.
|
|
9
|
+
*/
|
|
10
|
+
import { errorMessage } from '../core/format-utils.js';
|
|
11
|
+
import { fetchWithTimeout, isRetryableHttpStatus } from '../core/http-utils.js';
|
|
12
|
+
import { mappedKeySecretNames, readMappedKey, resolveKeySecretName } from '../core/identity-key-resolver.js';
|
|
13
|
+
import { OTLP_ENV } from '../core/settings-env.js';
|
|
14
|
+
import { DURATION_MS } from '../core/units.js';
|
|
15
|
+
import { buildOtelHeaders } from './otel-headers-helper.js';
|
|
16
|
+
const API_URL_ENV = 'OBTOOL_API_URL';
|
|
17
|
+
const DEFAULT_API_URL = 'https://api.integritystudio.ai';
|
|
18
|
+
const TRACES_PATH = '/v1/traces';
|
|
19
|
+
const PAGE_SIZE = 1000;
|
|
20
|
+
export const MAX_PAGES = 50;
|
|
21
|
+
const REQUEST_TIMEOUT_MS = DURATION_MS.TEN_SECONDS;
|
|
22
|
+
/** The first read after the store has sat idle can take the whole timeout; the repeat is warm. */
|
|
23
|
+
const MAX_ATTEMPTS = 2;
|
|
24
|
+
/** `nameMatch` value that makes `/v1/traces` match `spanName` exactly instead of as a substring. */
|
|
25
|
+
export const NAME_MATCH_EXACT = 'exact';
|
|
26
|
+
/** The read API base URL, distinct from the OTLP write endpoint. */
|
|
27
|
+
export function traceApiBaseUrl() {
|
|
28
|
+
return process.env[API_URL_ENV] ?? DEFAULT_API_URL;
|
|
29
|
+
}
|
|
30
|
+
/** The key for the org the signed-in account's spans ship to, else `OBTOOL_API_KEY`; empty when neither exists. */
|
|
31
|
+
export function readApiKey(debug) {
|
|
32
|
+
const secretName = resolveKeySecretName();
|
|
33
|
+
const mappedKey = secretName ? readMappedKey(secretName) : undefined;
|
|
34
|
+
debug?.(`Reading with the ${mappedKey && secretName ? secretName : OTLP_ENV.API_KEY} key`);
|
|
35
|
+
return mappedKey ?? process.env[OTLP_ENV.API_KEY] ?? '';
|
|
36
|
+
}
|
|
37
|
+
/** One key per mapped org, or the signed-in read key when nothing is mapped. Throws on a mapped key it cannot read: the result would be a slice. */
|
|
38
|
+
export function readEveryOrgApiKey() {
|
|
39
|
+
const secretNames = mappedKeySecretNames();
|
|
40
|
+
if (secretNames.length === 0)
|
|
41
|
+
return [readApiKey()];
|
|
42
|
+
return secretNames.map(secretName => {
|
|
43
|
+
const key = readMappedKey(secretName);
|
|
44
|
+
if (!key)
|
|
45
|
+
throw new Error(`${TRACES_PATH} read has no key for ${secretName}`);
|
|
46
|
+
return key;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
function pageUrl(apiUrl, params, pageSize, cursor) {
|
|
50
|
+
const url = new URL(`${apiUrl}${TRACES_PATH}`);
|
|
51
|
+
for (const [key, value] of Object.entries(params))
|
|
52
|
+
url.searchParams.set(key, value);
|
|
53
|
+
url.searchParams.set('limit', String(pageSize));
|
|
54
|
+
if (cursor)
|
|
55
|
+
url.searchParams.set('cursor', cursor);
|
|
56
|
+
return url;
|
|
57
|
+
}
|
|
58
|
+
function nextCursor(page) {
|
|
59
|
+
return page.hasMore && page.nextCursor ? page.nextCursor : undefined;
|
|
60
|
+
}
|
|
61
|
+
/** The parsed body. Repeats a request that timed out (waiting or mid-body), failed to connect or met a retryable status (408, 409, 429, 5xx); any other status or a malformed body is final. */
|
|
62
|
+
async function fetchJsonBody(urlStr, headers) {
|
|
63
|
+
let failure;
|
|
64
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
65
|
+
try {
|
|
66
|
+
// eslint-disable-next-line no-await-in-loop -- the repeat happens only after the first attempt failed
|
|
67
|
+
const response = await fetchWithTimeout(urlStr, { headers }, REQUEST_TIMEOUT_MS);
|
|
68
|
+
// eslint-disable-next-line no-await-in-loop -- the timeout covers the body, so reading it belongs to the attempt
|
|
69
|
+
if (response.ok)
|
|
70
|
+
return await response.json();
|
|
71
|
+
failure = new Error(`API error: ${response.status} ${response.statusText} (${urlStr})`);
|
|
72
|
+
void response.body?.cancel().catch(() => undefined);
|
|
73
|
+
if (!isRetryableHttpStatus(response.status))
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
catch (err) {
|
|
77
|
+
if (err instanceof SyntaxError) {
|
|
78
|
+
throw new Error(`${TRACES_PATH} response is not valid JSON (${urlStr}): ${errorMessage(err)}`, { cause: err });
|
|
79
|
+
}
|
|
80
|
+
failure = new Error(`${TRACES_PATH} request failed (${urlStr}): ${errorMessage(err)}`, { cause: err });
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
throw failure ?? new Error(`${TRACES_PATH} request not attempted (${urlStr})`);
|
|
84
|
+
}
|
|
85
|
+
async function fetchPage(url, headers) {
|
|
86
|
+
const urlStr = url.toString();
|
|
87
|
+
const body = await fetchJsonBody(urlStr, headers);
|
|
88
|
+
if (!Array.isArray(body?.data))
|
|
89
|
+
throw new Error(`${TRACES_PATH} response has no data array (${urlStr})`);
|
|
90
|
+
return { ...body, data: body.data };
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Raw rows matching `params`, following the cursor page by page, newest first. Throws on an
|
|
94
|
+
* unreachable API, an error status, or a body without a `data` array. A full read stops at
|
|
95
|
+
* `MAX_PAGES` and reports `truncated`: the oldest rows are then missing.
|
|
96
|
+
*/
|
|
97
|
+
export async function fetchTraceRows(params, { apiUrl = traceApiBaseUrl(), newest, apiKey, debug } = {}) {
|
|
98
|
+
const pageSize = newest ?? PAGE_SIZE;
|
|
99
|
+
const maxPages = newest === undefined ? MAX_PAGES : 1;
|
|
100
|
+
const headers = buildOtelHeaders(apiKey ?? readApiKey(debug));
|
|
101
|
+
const rows = [];
|
|
102
|
+
let cursor;
|
|
103
|
+
let truncated = false;
|
|
104
|
+
let pages = 0;
|
|
105
|
+
do {
|
|
106
|
+
const url = pageUrl(apiUrl, params, pageSize, cursor);
|
|
107
|
+
debug?.(`Querying remote API: ${url.toString()}`);
|
|
108
|
+
// eslint-disable-next-line no-await-in-loop -- each page's cursor comes from the previous response
|
|
109
|
+
const body = await fetchPage(url, headers);
|
|
110
|
+
rows.push(...body.data);
|
|
111
|
+
pages += 1;
|
|
112
|
+
cursor = nextCursor(body);
|
|
113
|
+
if (cursor && pages >= maxPages) {
|
|
114
|
+
truncated = newest === undefined;
|
|
115
|
+
cursor = undefined;
|
|
116
|
+
}
|
|
117
|
+
} while (cursor);
|
|
118
|
+
debug?.(`Received ${rows.length} rows from remote API across ${pages} page(s)`);
|
|
119
|
+
return { rows, truncated };
|
|
120
|
+
}
|
|
121
|
+
function compareStartNs(a, b) {
|
|
122
|
+
const aStart = BigInt(a.start_time_ns);
|
|
123
|
+
const bStart = BigInt(b.start_time_ns);
|
|
124
|
+
return aStart < bStart ? -1 : aStart > bStart ? 1 : 0;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* `fetchTraceRows` over every mapped org: each span once, newest first. `newest` reads that
|
|
128
|
+
* many rows per org and keeps the newest of the merge. `truncated` when any org's read hit
|
|
129
|
+
* the page cap. Throws on a mapped key it cannot read.
|
|
130
|
+
*/
|
|
131
|
+
export async function fetchTraceRowsEveryOrg(params, { apiUrl, newest, debug } = {}) {
|
|
132
|
+
const apiKeys = readEveryOrgApiKey();
|
|
133
|
+
debug?.(`Reading ${apiKeys.length} org(s)`);
|
|
134
|
+
const reads = await Promise.all(apiKeys.map(apiKey => fetchTraceRows(params, { apiUrl, newest, apiKey, debug })));
|
|
135
|
+
const truncated = reads.some(read => read.truncated);
|
|
136
|
+
const [onlyRead] = reads;
|
|
137
|
+
if (onlyRead && reads.length === 1)
|
|
138
|
+
return { rows: onlyRead.rows, truncated };
|
|
139
|
+
const rowsBySpan = new Map(reads.flatMap(read => read.rows).map(row => [`${row.trace_id}/${row.span_id}`, row]));
|
|
140
|
+
const newestFirst = [...rowsBySpan.values()].sort((a, b) => compareStartNs(b, a));
|
|
141
|
+
return { rows: newest === undefined ? newestFirst : newestFirst.slice(0, newest), truncated };
|
|
142
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* service.name / service.version shared by otel.ts's Resource and hand-built spans.
|
|
3
|
+
* Import-free of otel.ts (HP2); lazy env reads so OTEL_SERVICE_NAME resolves after load-envrc.
|
|
4
|
+
*/
|
|
5
|
+
const DEFAULT_SERVICE_NAME = 'claude-code-hooks';
|
|
6
|
+
export const SERVICE_VERSION = '1.0.0';
|
|
7
|
+
/** `||`, not `??`: semconv forbids an empty service.name, and otel.ts resolves the Resource the same way (DEDUP-001). */
|
|
8
|
+
export function getServiceName() {
|
|
9
|
+
return process.env.OTEL_SERVICE_NAME || DEFAULT_SERVICE_NAME;
|
|
10
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Alert rules for hooks telemetry, evaluated by the stop handler. Rules are data:
|
|
3
|
+
* each `ALERT_RULES` entry carries its own threshold comparison.
|
|
4
|
+
* @internal
|
|
5
|
+
*/
|
|
6
|
+
import { THRESHOLDS } from '../core/constants.js';
|
|
7
|
+
/** Scraping has no tolerance band: one failed invocation is already worth reporting. */
|
|
8
|
+
const NO_FAILURES_TOLERATED = 0;
|
|
9
|
+
/** Freeze a row (table freeze is shallow and rows are handed to callers) and pin its literal types via `const`. */
|
|
10
|
+
const defineRule = (rule) => Object.freeze(rule);
|
|
11
|
+
export const ALERT_RULES = Object.freeze({
|
|
12
|
+
taskCompletionLow: defineRule({
|
|
13
|
+
name: 'task-completion-low',
|
|
14
|
+
severity: 'warning',
|
|
15
|
+
message: `Task completion ratio below ${THRESHOLDS.TASK_COMPLETION_RATIO}, investigate incomplete work`,
|
|
16
|
+
comparator: 'below',
|
|
17
|
+
threshold: THRESHOLDS.TASK_COMPLETION_RATIO,
|
|
18
|
+
}),
|
|
19
|
+
mcpScrapeIncomplete: defineRule({
|
|
20
|
+
name: 'mcp-scrape-incomplete',
|
|
21
|
+
severity: 'warning',
|
|
22
|
+
message: 'MCP scraping incomplete, check for failed invocations',
|
|
23
|
+
comparator: 'above',
|
|
24
|
+
threshold: NO_FAILURES_TOLERATED,
|
|
25
|
+
}),
|
|
26
|
+
});
|
|
27
|
+
/** Fire `rule` when `value` crosses its threshold. Non-finite values never alert; range is the caller's job. */
|
|
28
|
+
function evaluate(rule, value) {
|
|
29
|
+
if (!Number.isFinite(value))
|
|
30
|
+
return [];
|
|
31
|
+
const breached = rule.comparator === 'below' ? value < rule.threshold : value > rule.threshold;
|
|
32
|
+
return breached ? [rule] : [];
|
|
33
|
+
}
|
|
34
|
+
/** Task completion alerts against a 0–1 ratio. */
|
|
35
|
+
export function evaluateTaskAlerts(completionRatio) {
|
|
36
|
+
return evaluate(ALERT_RULES.taskCompletionLow, completionRatio);
|
|
37
|
+
}
|
|
38
|
+
/** MCP scraping alerts; nothing scraped or a negative count means nothing to report. */
|
|
39
|
+
export function evaluateScrapingAlerts(failedCount, totalCount) {
|
|
40
|
+
if (totalCount <= 0 || failedCount < 0)
|
|
41
|
+
return [];
|
|
42
|
+
return evaluate(ALERT_RULES.mcpScrapeIncomplete, failedCount);
|
|
43
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/** GenAI-semconv token and cost metrics. The rate card is in model-pricing.ts. */
|
|
2
|
+
import { USER_WAIT_TOOLS } from '../core/constants.js';
|
|
3
|
+
import { UNITS } from '../core/units.js';
|
|
4
|
+
import { providerOfModel } from '../transcript/model-ids.js';
|
|
5
|
+
import { TOOL_ERROR_TYPE } from '../transcript/tool-call-durations.js';
|
|
6
|
+
import { inputTokensIncludingCache } from '../transcript/transcript-entry.js';
|
|
7
|
+
import { ERROR_TYPE_ATTR, GENAI_AGENT_ATTRIBUTES, GENAI_DURATION_BUCKETS_S, GENAI_EXECUTE_TOOL_DURATION_METRIC, GENAI_REQUEST_MODEL_ATTR, GENAI_TOOL_NAME_ATTR, TOOL_ATTRS, TOOL_WAIT_KIND } from './attributes.js';
|
|
8
|
+
import { initTelemetry, recordCounter, recordMetric } from './otel.js';
|
|
9
|
+
// Per-direction instruments from semantic-conventions-genai docs/gen-ai/gen-ai-token-metrics.md (#374), which
|
|
10
|
+
// replaced `gen_ai.client.token.usage` and `gen_ai.token.type`. Source dated in otel/attributes.ts.
|
|
11
|
+
const DIRECTION_METRICS = [
|
|
12
|
+
{
|
|
13
|
+
param: 'inputTokens',
|
|
14
|
+
usage: 'gen_ai.client.inference.usage.input_tokens',
|
|
15
|
+
usageDescription: 'Number of input tokens used',
|
|
16
|
+
operation: 'gen_ai.client.inference.operation.input_tokens',
|
|
17
|
+
operationDescription: 'Distribution of input tokens per operation',
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
param: 'outputTokens',
|
|
21
|
+
usage: 'gen_ai.client.inference.usage.output_tokens',
|
|
22
|
+
usageDescription: 'Number of output tokens used',
|
|
23
|
+
operation: 'gen_ai.client.inference.operation.output_tokens',
|
|
24
|
+
operationDescription: 'Distribution of output tokens per operation',
|
|
25
|
+
},
|
|
26
|
+
];
|
|
27
|
+
// Subsets of the usage.* pair, each its own counter (same doc). Pre-TMX6 rows hold them as datapoints on the parent
|
|
28
|
+
// counter tagged gen_ai.token.type; readers of that history filter on the tag.
|
|
29
|
+
const SUBSET_METRICS = [
|
|
30
|
+
{
|
|
31
|
+
param: 'cacheReadTokens',
|
|
32
|
+
name: 'gen_ai.client.inference.usage.cache_read.input_tokens',
|
|
33
|
+
description: 'Number of input tokens served from a provider-managed cache',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
param: 'cacheWriteTokens',
|
|
37
|
+
name: 'gen_ai.client.inference.usage.cache_write.input_tokens',
|
|
38
|
+
description: 'Number of input tokens written to a provider-managed cache',
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
param: 'reasoningTokens',
|
|
42
|
+
name: 'gen_ai.client.inference.usage.reasoning.output_tokens',
|
|
43
|
+
description: 'Number of output tokens used for reasoning',
|
|
44
|
+
},
|
|
45
|
+
];
|
|
46
|
+
// Vendor-prefixed: the GenAI conventions define no cost metric. Judge calls only: Claude Code's
|
|
47
|
+
// own spend is `claude_code.cost.usage`, priced per request by Claude Code.
|
|
48
|
+
const METRIC_COST = 'integritystudio.gen_ai.cost';
|
|
49
|
+
const COST_DESCRIPTION = 'Estimated cost of LLM operations in USD';
|
|
50
|
+
/** Not a `gen_ai.provider.name` registry member; a caller that knows the provider states it via `attributes`. */
|
|
51
|
+
const UNKNOWN_PROVIDER = 'unknown';
|
|
52
|
+
const ATTR_TOKEN_MODALITY = 'gen_ai.token.modality';
|
|
53
|
+
const TOKEN_MODALITY_TEXT = 'text';
|
|
54
|
+
/** Share of input tokens served from cache; undefined when there was no input. The second argument excludes cache reads, or they count twice. */
|
|
55
|
+
export function cacheHitRate(cacheReadTokens, uncachedInputTokens) {
|
|
56
|
+
const totalInput = cacheReadTokens + uncachedInputTokens;
|
|
57
|
+
return totalInput > 0 ? cacheReadTokens / totalInput : undefined;
|
|
58
|
+
}
|
|
59
|
+
function inferenceBaseAttributes(labels) {
|
|
60
|
+
return {
|
|
61
|
+
[GENAI_REQUEST_MODEL_ATTR]: labels.model,
|
|
62
|
+
[GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: providerOfModel(labels.model) ?? UNKNOWN_PROVIDER,
|
|
63
|
+
...(labels.operation && { [GENAI_AGENT_ATTRIBUTES.OPERATION_NAME]: labels.operation }),
|
|
64
|
+
...labels.attributes,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export function recordTokenUsage(params) {
|
|
68
|
+
initTelemetry();
|
|
69
|
+
const inferenceAttrs = { ...inferenceBaseAttributes(params), [ATTR_TOKEN_MODALITY]: TOKEN_MODALITY_TEXT };
|
|
70
|
+
for (const metric of DIRECTION_METRICS) {
|
|
71
|
+
const tokens = params[metric.param];
|
|
72
|
+
recordCounter(metric.usage, tokens, inferenceAttrs, { description: metric.usageDescription, unit: UNITS.TOKEN });
|
|
73
|
+
recordMetric(metric.operation, tokens, inferenceAttrs, { description: metric.operationDescription, unit: UNITS.TOKEN });
|
|
74
|
+
}
|
|
75
|
+
for (const metric of SUBSET_METRICS) {
|
|
76
|
+
const tokens = params[metric.param];
|
|
77
|
+
if (tokens === undefined)
|
|
78
|
+
continue;
|
|
79
|
+
recordCounter(metric.name, tokens, inferenceAttrs, { description: metric.description, unit: UNITS.TOKEN });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/** Only the observable kinds; a permission prompt the transcript cannot see gets no attribute rather than a guess. */
|
|
83
|
+
export function waitKindAttributes(duration) {
|
|
84
|
+
if (USER_WAIT_TOOLS.has(duration.tool))
|
|
85
|
+
return { [TOOL_ATTRS.WAIT_KIND]: TOOL_WAIT_KIND.INTERACTIVE };
|
|
86
|
+
if (duration.crossTurn)
|
|
87
|
+
return { [TOOL_ATTRS.WAIT_KIND]: TOOL_WAIT_KIND.CROSS_TURN };
|
|
88
|
+
return {};
|
|
89
|
+
}
|
|
90
|
+
/** Both emitters (Stop and SubagentStop) go through here, since unit and buckets apply on the instrument's first creation only. */
|
|
91
|
+
export function recordToolDurations(recordMetricFn, durations, extraAttributes = {}) {
|
|
92
|
+
for (const duration of durations) {
|
|
93
|
+
recordMetricFn(GENAI_EXECUTE_TOOL_DURATION_METRIC, duration.seconds, {
|
|
94
|
+
[GENAI_TOOL_NAME_ATTR]: duration.tool,
|
|
95
|
+
...extraAttributes,
|
|
96
|
+
...waitKindAttributes(duration),
|
|
97
|
+
...(duration.failed ? { [ERROR_TYPE_ATTR]: TOOL_ERROR_TYPE } : {}),
|
|
98
|
+
}, { unit: UNITS.SECOND, boundaries: GENAI_DURATION_BUCKETS_S.TOOL });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** One call's cost, labelled like its token counters so `/v1/cost` can join them by model. */
|
|
102
|
+
export function recordInferenceCost(labels, usd) {
|
|
103
|
+
initTelemetry();
|
|
104
|
+
recordCounter(METRIC_COST, usd, inferenceBaseAttributes(labels), { description: COST_DESCRIPTION, unit: UNITS.USD });
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Records the GenAI token metrics for one model's share of an operation, folding
|
|
108
|
+
* the cache-inclusive input sum (semconv definition of input_tokens) so callers do
|
|
109
|
+
* not reproduce it.
|
|
110
|
+
*/
|
|
111
|
+
export function recordModelTotals(model, totals, operation, attributes) {
|
|
112
|
+
recordTokenUsage({
|
|
113
|
+
inputTokens: inputTokensIncludingCache(totals.totalInputTokens, totals.totalCacheCreationTokens, totals.totalCacheReadTokens),
|
|
114
|
+
outputTokens: totals.totalOutputTokens,
|
|
115
|
+
cacheReadTokens: totals.totalCacheReadTokens,
|
|
116
|
+
cacheWriteTokens: totals.totalCacheCreationTokens,
|
|
117
|
+
reasoningTokens: totals.totalReasoningTokens,
|
|
118
|
+
model,
|
|
119
|
+
operation,
|
|
120
|
+
...(attributes ? { attributes } : {}),
|
|
121
|
+
});
|
|
122
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Three file-backed stores under session-state: the session-start span context
|
|
3
|
+
* (child spans across hook processes, 24 h TTL), the prompt span context (tool
|
|
4
|
+
* hooks link back to it, 10 min TTL), and the last session per project
|
|
5
|
+
* (`session.previous_id`, OTEL5b, no TTL).
|
|
6
|
+
*/
|
|
7
|
+
import { join } from 'node:path';
|
|
8
|
+
import { FILE_LOCK, THRESHOLDS } from '../core/constants.js';
|
|
9
|
+
import { loadJson, writeJson, writeJsonAtomic, withFileLock } from '../core/file-utils.js';
|
|
10
|
+
import { TIME_MS } from '../core/units.js';
|
|
11
|
+
import { projectStateDir, sessionStateDir } from '../session/session-state.js';
|
|
12
|
+
import { ERROR_TYPE_ATTR, UNKNOWN_ERROR_TYPE } from './attributes.js';
|
|
13
|
+
import { log } from './otel.js';
|
|
14
|
+
const SESSION_TRACE_TTL_MS = TIME_MS.DAY;
|
|
15
|
+
export const PREDECESSOR_STORE_UNAVAILABLE_MESSAGE = 'Session predecessor not recorded; this session starts a new chain';
|
|
16
|
+
function getSessionTracePath(sessionId) {
|
|
17
|
+
return join(sessionStateDir(sessionId), 'session-trace.json');
|
|
18
|
+
}
|
|
19
|
+
function saveTraceCtx(path, traceId, spanId) {
|
|
20
|
+
const ctx = { traceId, spanId, timestamp: Date.now() };
|
|
21
|
+
writeJson(path, ctx, false, { silent: true });
|
|
22
|
+
}
|
|
23
|
+
function loadTraceCtx(path, ttlMs) {
|
|
24
|
+
const ctx = loadJson(path);
|
|
25
|
+
if (!ctx || Date.now() - ctx.timestamp > ttlMs)
|
|
26
|
+
return null;
|
|
27
|
+
return { traceId: ctx.traceId, spanId: ctx.spanId };
|
|
28
|
+
}
|
|
29
|
+
export function saveSessionContext(sessionId, traceId, spanId) {
|
|
30
|
+
saveTraceCtx(getSessionTracePath(sessionId), traceId, spanId);
|
|
31
|
+
}
|
|
32
|
+
export function loadSessionContext(sessionId) {
|
|
33
|
+
return loadTraceCtx(getSessionTracePath(sessionId), SESSION_TRACE_TTL_MS);
|
|
34
|
+
}
|
|
35
|
+
function getContextPath(sessionId) {
|
|
36
|
+
return join(sessionStateDir(sessionId), 'prompt-trace.json');
|
|
37
|
+
}
|
|
38
|
+
export function savePromptContext(sessionId, traceId, spanId) {
|
|
39
|
+
saveTraceCtx(getContextPath(sessionId), traceId, spanId);
|
|
40
|
+
}
|
|
41
|
+
export function loadPromptContext(sessionId) {
|
|
42
|
+
return loadTraceCtx(getContextPath(sessionId), THRESHOLDS.PROMPT_CONTEXT_TTL_MS);
|
|
43
|
+
}
|
|
44
|
+
/** Span link attributes for the prompt span, when one was recorded. No `session.id`: the monitor stamps it and drops a caller's copy. */
|
|
45
|
+
export function promptLinkAttrs(sessionId) {
|
|
46
|
+
const promptCtx = sessionId ? loadPromptContext(sessionId) : null;
|
|
47
|
+
return promptCtx ? { 'integritystudio.prompt.trace_id': promptCtx.traceId, 'integritystudio.prompt.span_id': promptCtx.spanId } : {};
|
|
48
|
+
}
|
|
49
|
+
function getLastSessionPath(projectSlug) {
|
|
50
|
+
return join(projectStateDir(projectSlug), 'last-session.json');
|
|
51
|
+
}
|
|
52
|
+
export function saveLastSessionId(projectSlug, sessionId) {
|
|
53
|
+
const record = { sessionId, timestamp: Date.now() };
|
|
54
|
+
writeJsonAtomic(getLastSessionPath(projectSlug), record, { silent: true });
|
|
55
|
+
}
|
|
56
|
+
export function loadLastSessionId(projectSlug) {
|
|
57
|
+
return loadJson(getLastSessionPath(projectSlug))?.sessionId || null;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Read the predecessor and record this session as its successor in one locked
|
|
61
|
+
* section (OTEL5-8b): unserialized, two concurrent starts both read `P`, fork off
|
|
62
|
+
* it, and the earlier one is skipped walking back. Every edge is still truthful;
|
|
63
|
+
* only chain completeness was lost. Fail-open: an untaken lock degrades to a
|
|
64
|
+
* possible fork, never a blocked start. A fine backoff fits the ~1 ms section.
|
|
65
|
+
*/
|
|
66
|
+
export function recordSessionPredecessor(projectSlug, sessionId) {
|
|
67
|
+
try {
|
|
68
|
+
return withFileLock(getLastSessionPath(projectSlug), () => {
|
|
69
|
+
const prior = loadLastSessionId(projectSlug);
|
|
70
|
+
saveLastSessionId(projectSlug, sessionId);
|
|
71
|
+
return prior;
|
|
72
|
+
}, { retryDelayMs: FILE_LOCK.LAST_SESSION_RETRY_DELAY_MS, maxRetries: FILE_LOCK.LAST_SESSION_MAX_RETRIES });
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
// withFileLock creates the lock's parent directory outside its own fail-open handling
|
|
76
|
+
log('WARN', PREDECESSOR_STORE_UNAVAILABLE_MESSAGE, {
|
|
77
|
+
[ERROR_TYPE_ATTR]: error instanceof Error && 'code' in error ? String(error.code) : UNKNOWN_ERROR_TYPE,
|
|
78
|
+
});
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/** Meta-evaluation policy: which judged explanations are re-scored, and by which model (AA3 Gate 3b). */
|
|
2
|
+
import { UNITS } from '../core/units.js';
|
|
3
|
+
import { GENAI_AGENT_ATTRIBUTES, GENAI_REQUEST_MODEL_ATTR } from '../otel/attributes.js';
|
|
4
|
+
import { recordMetric } from '../otel/otel.js';
|
|
5
|
+
import { firstJudgeApiKey, providerForModel } from './judge-providers.js';
|
|
6
|
+
import { JUDGE_MODEL } from './llm-judge.js';
|
|
7
|
+
import { QUALITY_ATTRS } from './quality-eval-handoff.js';
|
|
8
|
+
import { QUALITY_METRIC_NAMES } from './quality-signals.js';
|
|
9
|
+
const META_EVAL_JUDGE_MODEL = process.env.META_EVAL_JUDGE_MODEL || JUDGE_MODEL;
|
|
10
|
+
/** Mirrors META_EVAL_SAMPLE_RATE in the toolkit's llm-judge-config.ts. */
|
|
11
|
+
export const META_EVAL_SAMPLE_RATE = 0.1;
|
|
12
|
+
/** `META_EVAL_SAMPLE_RATE` override (AA3 Gate 3b). Invalid values fall back, not clamp, so a typo cannot toggle meta-eval. */
|
|
13
|
+
export function metaEvalSampleRate() {
|
|
14
|
+
const raw = process.env.META_EVAL_SAMPLE_RATE;
|
|
15
|
+
const rate = raw === undefined || raw.trim() === '' ? NaN : Number(raw);
|
|
16
|
+
return Number.isFinite(rate) && rate >= 0 && rate <= 1 ? rate : META_EVAL_SAMPLE_RATE;
|
|
17
|
+
}
|
|
18
|
+
/** Must match EXPLANATION_QUALITY_CRITERIA.name in the toolkit. */
|
|
19
|
+
export const META_EVAL_METRIC_NAME = 'explanation_quality';
|
|
20
|
+
export function shouldMetaEvaluate(evaluationName, canary) {
|
|
21
|
+
if (canary)
|
|
22
|
+
return false;
|
|
23
|
+
if (evaluationName === META_EVAL_METRIC_NAME)
|
|
24
|
+
return false;
|
|
25
|
+
return Math.random() < metaEvalSampleRate();
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* `META_EVAL_JUDGE_MODELS`, comma-separated: per-evaluation picks run AA3 Gate 3b's arms over one
|
|
29
|
+
* population. Read at module load; settings.json `env` does not reach hooks, so export it beforehand.
|
|
30
|
+
*/
|
|
31
|
+
const META_EVAL_JUDGE_MODEL_POOL = (process.env.META_EVAL_JUDGE_MODELS ?? '')
|
|
32
|
+
.split(',')
|
|
33
|
+
.map((model) => model.trim())
|
|
34
|
+
.filter(Boolean);
|
|
35
|
+
const META_EVAL_UNAVAILABLE_REASONS = {
|
|
36
|
+
UNKNOWN_PROVIDER: 'unknown_provider',
|
|
37
|
+
MISSING_CREDENTIAL: 'missing_credential',
|
|
38
|
+
};
|
|
39
|
+
function recordArmUnavailable(model, reason, extra = {}) {
|
|
40
|
+
recordMetric(QUALITY_METRIC_NAMES.META_EVAL_ARM_UNAVAILABLE, 1, { [GENAI_REQUEST_MODEL_ATTR]: model, [QUALITY_ATTRS.META_EVAL_UNAVAILABLE_REASON]: reason, ...extra }, { unit: UNITS.FAILURE });
|
|
41
|
+
}
|
|
42
|
+
/** Drops and counts unusable arms; one reaching `callJudge` would record nothing and look like slow collection. */
|
|
43
|
+
function resolveUsableMetaEvalModels() {
|
|
44
|
+
const configured = META_EVAL_JUDGE_MODEL_POOL.length > 0
|
|
45
|
+
? META_EVAL_JUDGE_MODEL_POOL
|
|
46
|
+
: [META_EVAL_JUDGE_MODEL];
|
|
47
|
+
const usable = [];
|
|
48
|
+
for (const model of configured) {
|
|
49
|
+
const provider = providerForModel(model);
|
|
50
|
+
if (!provider) {
|
|
51
|
+
recordArmUnavailable(model, META_EVAL_UNAVAILABLE_REASONS.UNKNOWN_PROVIDER);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (firstJudgeApiKey(provider) === undefined) {
|
|
55
|
+
recordArmUnavailable(model, META_EVAL_UNAVAILABLE_REASONS.MISSING_CREDENTIAL, { [GENAI_AGENT_ATTRIBUTES.PROVIDER_NAME]: provider });
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
usable.push(model);
|
|
59
|
+
}
|
|
60
|
+
return usable;
|
|
61
|
+
}
|
|
62
|
+
/** Once per process: pool and credentials are fixed, so each unusable arm is counted once. */
|
|
63
|
+
let usableMetaEvalModels;
|
|
64
|
+
export function pickMetaEvalJudgeModel() {
|
|
65
|
+
usableMetaEvalModels ??= resolveUsableMetaEvalModels();
|
|
66
|
+
if (usableMetaEvalModels.length === 0)
|
|
67
|
+
return undefined;
|
|
68
|
+
const index = Math.floor(Math.random() * usableMetaEvalModels.length);
|
|
69
|
+
return usableMetaEvalModels[index];
|
|
70
|
+
}
|