@agent-native/core 0.109.0 → 0.109.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +14 -0
- package/corpus/core/docs/content/integrations.mdx +87 -253
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/a2a/client.ts +24 -2
- package/corpus/core/src/a2a/correlation.ts +50 -0
- package/corpus/core/src/a2a/handlers.ts +140 -14
- package/corpus/core/src/a2a/index.ts +1 -0
- package/corpus/core/src/a2a/invoke.ts +10 -1
- package/corpus/core/src/a2a/task-store.ts +146 -6
- package/corpus/core/src/a2a/types.ts +16 -0
- package/corpus/core/src/action.ts +2 -0
- package/corpus/core/src/agent/production-agent.ts +23 -0
- package/corpus/core/src/client/blocks/library/diagram.config.ts +10 -1
- package/corpus/core/src/client/blocks/library/diagram.tsx +9 -4
- package/corpus/core/src/client/use-run-stuck-detection.ts +27 -1
- package/corpus/core/src/db/client.ts +55 -0
- package/corpus/core/src/deploy/build.ts +63 -3
- package/corpus/core/src/observability/traces.ts +172 -19
- package/corpus/core/src/scripts/call-agent.ts +53 -2
- package/corpus/core/src/server/agent-chat/action-filters-a2a.ts +72 -25
- package/corpus/core/src/server/agent-chat-plugin.ts +107 -6
- package/corpus/core/src/templates/workspace-core/.agents/skills/a2a-protocol/SKILL.md +21 -0
- package/corpus/core/src/templates/workspace-core/.agents/skills/observability/SKILL.md +7 -1
- package/corpus/core/src/templates/workspace-core/.agents/skills/tracking/SKILL.md +1 -1
- package/corpus/templates/analytics/actions/query-agent-native-analytics.ts +1 -1
- package/corpus/templates/analytics/changelog/2026-07-17-analytics-now-includes-a-default-alert-for-spikes-in-stuck-agent-chats.md +6 -0
- package/corpus/templates/analytics/docs/schemas/first-party-analytics.md +23 -14
- package/corpus/templates/analytics/docs/schemas/tracked-events.md +5 -1
- package/corpus/templates/analytics/server/jobs/analytics-alerts.ts +3 -3
- package/corpus/templates/analytics/server/lib/analytics-alerts.ts +209 -87
- package/corpus/templates/clips/.agents/skills/dictate/SKILL.md +28 -5
- package/corpus/templates/clips/.agents/skills/meetings/SKILL.md +7 -0
- package/corpus/templates/clips/.agents/skills/recording/SKILL.md +23 -0
- package/corpus/templates/clips/AGENTS.md +10 -0
- package/corpus/templates/clips/actions/create-dictation.ts +29 -4
- package/corpus/templates/clips/actions/finalize-recording.ts +542 -65
- package/corpus/templates/clips/actions/get-recording-player-data.ts +16 -6
- package/corpus/templates/clips/actions/set-mobile-capture-state.ts +37 -0
- package/corpus/templates/clips/actions/update-dictation.ts +9 -1
- package/corpus/templates/clips/app/components/player/playback-comment-overlay.tsx +17 -2
- package/corpus/templates/clips/app/components/player/video-player.tsx +14 -2
- package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +2 -2
- package/corpus/templates/clips/app/routes/r.$recordingId.tsx +14 -3
- package/corpus/templates/clips/app/routes/record.tsx +3 -3
- package/corpus/templates/clips/app/routes/share.$shareId.tsx +19 -5
- package/corpus/templates/clips/changelog/2026-07-17-capture-dictation-background-meeting-audio-and-camera-videos.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-17-clips-now-retries-storage-verification-in-the-background-ins.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-17-delayed-upload-verification-keeps-local-backups-retryable.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-17-fixed-broken-thumbnail-icons-appearing-over-playable-clip-em.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-17-playback-comments-stay-readable-at-faster-playback-speeds.md +6 -0
- package/corpus/templates/clips/chrome-extension/src/finalize-recovery.ts +3 -0
- package/corpus/templates/clips/chrome-extension/src/offscreen.ts +3 -3
- package/corpus/templates/clips/desktop/src/app.tsx +35 -6
- package/corpus/templates/clips/desktop/src/lib/processing-backup-recovery.ts +19 -0
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +141 -18
- package/corpus/templates/clips/desktop/src/lib/upload-verification.ts +28 -7
- package/corpus/templates/clips/desktop/src-tauri/src/lib.rs +3 -4
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen/live_upload.rs +32 -15
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +59 -18
- package/corpus/templates/clips/server/db/schema.ts +9 -1
- package/corpus/templates/clips/server/jobs/media-verification.ts +130 -0
- package/corpus/templates/clips/server/lib/builder-media-compression.ts +74 -7
- package/corpus/templates/clips/server/lib/media-verification-state.ts +55 -0
- package/corpus/templates/clips/server/lib/player-thumbnail-url.ts +31 -0
- package/corpus/templates/clips/server/lib/post-finalize-dispatch.ts +10 -3
- package/corpus/templates/clips/server/plugins/auth.ts +5 -5
- package/corpus/templates/clips/server/plugins/calendar-jobs.ts +2 -0
- package/corpus/templates/clips/server/routes/api/_agent-native-background/post-finalize-worker.post.ts +12 -2
- package/corpus/templates/clips/server/routes/api/public-recording.get.ts +13 -1
- package/corpus/templates/clips/server/routes/api/thumbnail/[recordingId].get.ts +340 -0
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/abort.post.ts +24 -7
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/chunk.post.ts +91 -7
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/reset-chunks.post.ts +22 -1
- package/corpus/templates/clips/server/routes/api/uploads/[recordingId]/status.get.ts +14 -0
- package/corpus/templates/clips/shared/finalize-recovery.ts +61 -2
- package/corpus/templates/clips/shared/share-meta.ts +1 -1
- package/corpus/templates/plan/changelog/2026-07-17-diagrams-can-opt-into-a-polished-clean-rendering-mode-withou.md +6 -0
- package/corpus/templates/plan/shared/plan-content.ts +3 -0
- package/dist/a2a/client.d.ts +11 -2
- package/dist/a2a/client.d.ts.map +1 -1
- package/dist/a2a/client.js +16 -3
- package/dist/a2a/client.js.map +1 -1
- package/dist/a2a/correlation.d.ts +10 -0
- package/dist/a2a/correlation.d.ts.map +1 -0
- package/dist/a2a/correlation.js +40 -0
- package/dist/a2a/correlation.js.map +1 -0
- package/dist/a2a/handlers.d.ts.map +1 -1
- package/dist/a2a/handlers.js +100 -15
- package/dist/a2a/handlers.js.map +1 -1
- package/dist/a2a/index.d.ts +1 -1
- package/dist/a2a/index.d.ts.map +1 -1
- package/dist/a2a/index.js.map +1 -1
- package/dist/a2a/invoke.d.ts +4 -1
- package/dist/a2a/invoke.d.ts.map +1 -1
- package/dist/a2a/invoke.js +3 -0
- package/dist/a2a/invoke.js.map +1 -1
- package/dist/a2a/task-store.d.ts +12 -1
- package/dist/a2a/task-store.d.ts.map +1 -1
- package/dist/a2a/task-store.js +96 -6
- package/dist/a2a/task-store.js.map +1 -1
- package/dist/a2a/types.d.ts +15 -0
- package/dist/a2a/types.d.ts.map +1 -1
- package/dist/a2a/types.js.map +1 -1
- package/dist/action.d.ts +2 -0
- package/dist/action.d.ts.map +1 -1
- package/dist/action.js.map +1 -1
- package/dist/agent/production-agent.d.ts +9 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +14 -0
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/client/blocks/library/diagram.config.d.ts +2 -0
- package/dist/client/blocks/library/diagram.config.d.ts.map +1 -1
- package/dist/client/blocks/library/diagram.config.js +8 -1
- package/dist/client/blocks/library/diagram.config.js.map +1 -1
- package/dist/client/blocks/library/diagram.d.ts.map +1 -1
- package/dist/client/blocks/library/diagram.js +9 -4
- package/dist/client/blocks/library/diagram.js.map +1 -1
- package/dist/client/use-run-stuck-detection.d.ts.map +1 -1
- package/dist/client/use-run-stuck-detection.js +25 -1
- package/dist/client/use-run-stuck-detection.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/db/client.d.ts +7 -0
- package/dist/db/client.d.ts.map +1 -1
- package/dist/db/client.js +33 -0
- package/dist/db/client.js.map +1 -1
- package/dist/deploy/build.d.ts +3 -0
- package/dist/deploy/build.d.ts.map +1 -1
- package/dist/deploy/build.js +45 -1
- package/dist/deploy/build.js.map +1 -1
- package/dist/observability/routes.d.ts +1 -1
- package/dist/observability/traces.d.ts +9 -0
- package/dist/observability/traces.d.ts.map +1 -1
- package/dist/observability/traces.js +137 -19
- package/dist/observability/traces.js.map +1 -1
- package/dist/progress/routes.d.ts +1 -1
- package/dist/scripts/call-agent.d.ts.map +1 -1
- package/dist/scripts/call-agent.js +35 -3
- package/dist/scripts/call-agent.js.map +1 -1
- package/dist/server/agent-chat/action-filters-a2a.d.ts +18 -2
- package/dist/server/agent-chat/action-filters-a2a.d.ts.map +1 -1
- package/dist/server/agent-chat/action-filters-a2a.js +35 -7
- package/dist/server/agent-chat/action-filters-a2a.js.map +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +93 -6
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/templates/workspace-core/.agents/skills/a2a-protocol/SKILL.md +21 -0
- package/dist/templates/workspace-core/.agents/skills/observability/SKILL.md +7 -1
- package/dist/templates/workspace-core/.agents/skills/tracking/SKILL.md +1 -1
- package/docs/content/integrations.mdx +87 -253
- package/package.json +1 -1
- package/src/templates/workspace-core/.agents/skills/a2a-protocol/SKILL.md +21 -0
- package/src/templates/workspace-core/.agents/skills/observability/SKILL.md +7 -1
- package/src/templates/workspace-core/.agents/skills/tracking/SKILL.md +1 -1
|
@@ -18,6 +18,7 @@ export declare function instrumentAgentLoop(opts: {
|
|
|
18
18
|
send: (event: AgentChatEvent) => void;
|
|
19
19
|
signal: AbortSignal;
|
|
20
20
|
providerOptions?: any;
|
|
21
|
+
runId?: string;
|
|
21
22
|
}) => Promise<AgentLoopUsage>;
|
|
22
23
|
loopOpts: {
|
|
23
24
|
engine: any;
|
|
@@ -29,6 +30,7 @@ export declare function instrumentAgentLoop(opts: {
|
|
|
29
30
|
send: (event: AgentChatEvent) => void;
|
|
30
31
|
signal: AbortSignal;
|
|
31
32
|
providerOptions?: any;
|
|
33
|
+
runId?: string;
|
|
32
34
|
};
|
|
33
35
|
runId: string;
|
|
34
36
|
threadId: string | null;
|
|
@@ -44,6 +46,13 @@ export declare function instrumentAgentLoop(opts: {
|
|
|
44
46
|
variantId: string;
|
|
45
47
|
}>;
|
|
46
48
|
modelSelectionSource?: string;
|
|
49
|
+
delegation?: {
|
|
50
|
+
protocol: "a2a" | "mcp";
|
|
51
|
+
callerApp?: string;
|
|
52
|
+
taskId?: string;
|
|
53
|
+
parentRunId?: string;
|
|
54
|
+
parentTurnId?: string;
|
|
55
|
+
};
|
|
47
56
|
/** Raw user-authored message before prompt/context enrichment. */
|
|
48
57
|
sentimentInput?: string;
|
|
49
58
|
classifyError?: (error: unknown) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,mBAAmB,CAAC;AAGxE,OAAO,KAAK,EAA2B,mBAAmB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,mBAAmB,CAAC;AAGxE,OAAO,KAAK,EAA2B,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAmK/E;;;wEAGwE;AACxE,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAE7D;AAoBD,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAc3E;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE;IAC9C,YAAY,EAAE,CAAC,QAAQ,EAAE;QACvB,MAAM,EAAE,GAAG,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,GAAG,EAAE,CAAC;QACb,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,WAAW,CAAC;QACpB,eAAe,CAAC,EAAE,GAAG,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAC9B,QAAQ,EAAE;QACR,MAAM,EAAE,GAAG,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,GAAG,EAAE,CAAC;QACb,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,WAAW,CAAC;QACpB,eAAe,CAAC,EAAE,GAAG,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB;;;iBAGa;IACb,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,qBAAqB,CAAC,EAAE,KAAK,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,UAAU,CAAC,EAAE;QACX,QAAQ,EAAE,KAAK,GAAG,KAAK,CAAC;QACxB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,kEAAkE;IAClE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAC3B;QACE,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;KAC3C,GACD,IAAI,GACJ,SAAS,CAAC;CACf,GAAG,OAAO,CAAC,cAAc,CAAC,CA0f1B"}
|
|
@@ -21,6 +21,7 @@ function llmProviderFromEngine(engineName, model) {
|
|
|
21
21
|
function costUsdFromCenticents(value) {
|
|
22
22
|
return Math.round((value / 10_000) * 1_000_000) / 1_000_000;
|
|
23
23
|
}
|
|
24
|
+
const MAX_TRACKED_GENERATION_TOOL_CALLS = 50;
|
|
24
25
|
function emitLlmGenerationTrackingEvent(args) {
|
|
25
26
|
const provider = llmProviderFromEngine(args.engineName, args.model);
|
|
26
27
|
const costUsd = costUsdFromCenticents(args.costCentsX100);
|
|
@@ -47,6 +48,14 @@ function emitLlmGenerationTrackingEvent(args) {
|
|
|
47
48
|
tool_calls: args.toolCalls,
|
|
48
49
|
successful_tools: args.successfulTools,
|
|
49
50
|
failed_tools: args.failedTools,
|
|
51
|
+
tools: args.tools,
|
|
52
|
+
tools_truncated: args.toolsTruncated,
|
|
53
|
+
delegated: args.delegation ? true : undefined,
|
|
54
|
+
delegation_protocol: args.delegation?.protocol,
|
|
55
|
+
caller_app: args.delegation?.callerApp,
|
|
56
|
+
a2a_task_id: args.delegation?.taskId,
|
|
57
|
+
parent_run_id: args.delegation?.parentRunId,
|
|
58
|
+
parent_turn_id: args.delegation?.parentTurnId,
|
|
50
59
|
model_selection_source: args.modelSelectionSource,
|
|
51
60
|
created_at: new Date(args.createdAt).toISOString(),
|
|
52
61
|
created_at_ms: args.createdAt,
|
|
@@ -181,12 +190,11 @@ export async function instrumentAgentLoop(opts) {
|
|
|
181
190
|
let toolInvocationCounter = 0;
|
|
182
191
|
// Keyed by counter to handle concurrent calls to the same tool name
|
|
183
192
|
const pendingTools = new Map();
|
|
184
|
-
// Secondary index
|
|
185
|
-
//
|
|
186
|
-
// pair starts and dones correctly when the agent runs concurrent calls to the
|
|
187
|
-
// same tool name (read-only / parallelSafe batches via Promise.all), we keep a
|
|
188
|
-
// queue per name and match each done to the OLDEST still-pending start.
|
|
193
|
+
// Secondary index for legacy emitters without call ids. Current tool events
|
|
194
|
+
// are paired by id first; same-name FIFO remains as a compatibility fallback.
|
|
189
195
|
const toolNameToCounters = new Map();
|
|
196
|
+
const toolCallIdToCounter = new Map();
|
|
197
|
+
const generationToolCalls = new Map();
|
|
190
198
|
let toolCallCount = 0;
|
|
191
199
|
let successfulTools = 0;
|
|
192
200
|
let failedTools = 0;
|
|
@@ -204,12 +212,15 @@ export async function instrumentAgentLoop(opts) {
|
|
|
204
212
|
// microtask gap by recording the span on the pending entry when ready.
|
|
205
213
|
const entry = {
|
|
206
214
|
spanId: sid,
|
|
215
|
+
...(event.id ? { callId: event.id } : {}),
|
|
207
216
|
startMs: Date.now(),
|
|
208
217
|
toolName: event.tool,
|
|
209
218
|
input: event.input,
|
|
210
219
|
otelSpan: null,
|
|
211
220
|
};
|
|
212
221
|
pendingTools.set(counter, entry);
|
|
222
|
+
if (event.id)
|
|
223
|
+
toolCallIdToCounter.set(event.id, counter);
|
|
213
224
|
void startAgentSpan("tool.call", {
|
|
214
225
|
"tool.name": event.tool,
|
|
215
226
|
}).then((span) => {
|
|
@@ -236,14 +247,34 @@ export async function instrumentAgentLoop(opts) {
|
|
|
236
247
|
}
|
|
237
248
|
else if (event.type === "tool_done") {
|
|
238
249
|
const queue = toolNameToCounters.get(event.tool);
|
|
239
|
-
const
|
|
250
|
+
const counterFromId = event.id
|
|
251
|
+
? toolCallIdToCounter.get(event.id)
|
|
252
|
+
: undefined;
|
|
253
|
+
const legacyQueueIndex = event.id && counterFromId === undefined && queue
|
|
254
|
+
? queue.findIndex((candidate) => !pendingTools.get(candidate)?.callId)
|
|
255
|
+
: -1;
|
|
256
|
+
const counter = counterFromId ??
|
|
257
|
+
(event.id
|
|
258
|
+
? legacyQueueIndex >= 0
|
|
259
|
+
? queue?.[legacyQueueIndex]
|
|
260
|
+
: undefined
|
|
261
|
+
: queue?.shift());
|
|
240
262
|
const pending = counter !== undefined ? pendingTools.get(counter) : undefined;
|
|
241
263
|
if (counter !== undefined) {
|
|
242
264
|
pendingTools.delete(counter);
|
|
265
|
+
if (pending?.callId)
|
|
266
|
+
toolCallIdToCounter.delete(pending.callId);
|
|
267
|
+
if ((counterFromId !== undefined || legacyQueueIndex >= 0) && queue) {
|
|
268
|
+
const queueIndex = queue.indexOf(counter);
|
|
269
|
+
if (queueIndex >= 0)
|
|
270
|
+
queue.splice(queueIndex, 1);
|
|
271
|
+
}
|
|
243
272
|
if (queue && queue.length === 0)
|
|
244
273
|
toolNameToCounters.delete(event.tool);
|
|
245
274
|
}
|
|
246
275
|
toolCallCount++;
|
|
276
|
+
const finishedAt = Date.now();
|
|
277
|
+
const explicitError = event.isError === true;
|
|
247
278
|
const isError = typeof event.isError === "boolean"
|
|
248
279
|
? event.isError
|
|
249
280
|
: typeof event.result === "string" &&
|
|
@@ -253,6 +284,21 @@ export async function instrumentAgentLoop(opts) {
|
|
|
253
284
|
failedTools++;
|
|
254
285
|
else
|
|
255
286
|
successfulTools++;
|
|
287
|
+
if (counter !== undefined &&
|
|
288
|
+
counter < MAX_TRACKED_GENERATION_TOOL_CALLS &&
|
|
289
|
+
pending) {
|
|
290
|
+
generationToolCalls.set(counter, {
|
|
291
|
+
name: pending.toolName,
|
|
292
|
+
started_offset_ms: Math.max(0, pending.startMs - runStart),
|
|
293
|
+
duration_ms: Math.max(0, finishedAt - pending.startMs),
|
|
294
|
+
status: isError ? "error" : "success",
|
|
295
|
+
error_class: !isError
|
|
296
|
+
? null
|
|
297
|
+
: explicitError
|
|
298
|
+
? "tool_error"
|
|
299
|
+
: "legacy_inferred_error",
|
|
300
|
+
});
|
|
301
|
+
}
|
|
256
302
|
// Finalize the OTel tool span. If the span promise hasn't resolved yet
|
|
257
303
|
// we record the result on the entry so its `.then` handler ends it.
|
|
258
304
|
const otelEndResult = {
|
|
@@ -283,7 +329,7 @@ export async function instrumentAgentLoop(opts) {
|
|
|
283
329
|
cacheReadTokens: 0,
|
|
284
330
|
cacheWriteTokens: 0,
|
|
285
331
|
costCentsX100: 0,
|
|
286
|
-
durationMs: pending ?
|
|
332
|
+
durationMs: pending ? Math.max(0, finishedAt - pending.startMs) : 0,
|
|
287
333
|
status: isError ? "error" : "success",
|
|
288
334
|
errorMessage: isError ? event.result : null,
|
|
289
335
|
metadata: config.captureToolArgs && pending
|
|
@@ -308,7 +354,11 @@ export async function instrumentAgentLoop(opts) {
|
|
|
308
354
|
let errorMessage = null;
|
|
309
355
|
let runMetadata = opts.metadata ?? null;
|
|
310
356
|
try {
|
|
311
|
-
usage = await runAgentLoop({
|
|
357
|
+
usage = await runAgentLoop({
|
|
358
|
+
...loopOpts,
|
|
359
|
+
runId,
|
|
360
|
+
send: instrumentedSend,
|
|
361
|
+
});
|
|
312
362
|
}
|
|
313
363
|
catch (err) {
|
|
314
364
|
const classification = opts.classifyError?.(err) ?? null;
|
|
@@ -327,6 +377,62 @@ export async function instrumentAgentLoop(opts) {
|
|
|
327
377
|
finally {
|
|
328
378
|
const runEnd = Date.now();
|
|
329
379
|
const totalDurationMs = runEnd - runStart;
|
|
380
|
+
if (pendingTools.size > 0) {
|
|
381
|
+
if (runStatus === "success") {
|
|
382
|
+
runStatus = "error";
|
|
383
|
+
errorMessage ??= "Agent run ended with interrupted tool calls";
|
|
384
|
+
}
|
|
385
|
+
for (const [counter, pending] of pendingTools) {
|
|
386
|
+
toolCallCount += 1;
|
|
387
|
+
failedTools += 1;
|
|
388
|
+
if (counter < MAX_TRACKED_GENERATION_TOOL_CALLS) {
|
|
389
|
+
generationToolCalls.set(counter, {
|
|
390
|
+
name: pending.toolName,
|
|
391
|
+
started_offset_ms: Math.max(0, pending.startMs - runStart),
|
|
392
|
+
duration_ms: Math.max(0, runEnd - pending.startMs),
|
|
393
|
+
status: "error",
|
|
394
|
+
error_class: "interrupted",
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
const interruptedMessage = "Tool call interrupted before completion";
|
|
398
|
+
if (pending.otelSpan) {
|
|
399
|
+
openOtelToolSpans.delete(pending.otelSpan);
|
|
400
|
+
endAgentSpan(pending.otelSpan, {
|
|
401
|
+
status: "error",
|
|
402
|
+
errorMessage: interruptedMessage,
|
|
403
|
+
attributes: { "tool.name": pending.toolName },
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
pending.endResult = {
|
|
408
|
+
status: "error",
|
|
409
|
+
errorMessage: interruptedMessage,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
spans.push({
|
|
413
|
+
id: pending.spanId,
|
|
414
|
+
runId,
|
|
415
|
+
threadId,
|
|
416
|
+
userId,
|
|
417
|
+
parentSpanId,
|
|
418
|
+
spanType: "tool_call",
|
|
419
|
+
name: pending.toolName,
|
|
420
|
+
inputTokens: 0,
|
|
421
|
+
outputTokens: 0,
|
|
422
|
+
cacheReadTokens: 0,
|
|
423
|
+
cacheWriteTokens: 0,
|
|
424
|
+
costCentsX100: 0,
|
|
425
|
+
durationMs: Math.max(0, runEnd - pending.startMs),
|
|
426
|
+
status: "error",
|
|
427
|
+
errorMessage: interruptedMessage,
|
|
428
|
+
metadata: null,
|
|
429
|
+
createdAt: runEnd,
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
pendingTools.clear();
|
|
433
|
+
toolNameToCounters.clear();
|
|
434
|
+
toolCallIdToCounter.clear();
|
|
435
|
+
}
|
|
330
436
|
let costCentsX100 = 0;
|
|
331
437
|
try {
|
|
332
438
|
const { calculateCost } = await import("../usage/store.js");
|
|
@@ -336,8 +442,15 @@ export async function instrumentAgentLoop(opts) {
|
|
|
336
442
|
}
|
|
337
443
|
catch { }
|
|
338
444
|
let llmCallCount = 0;
|
|
339
|
-
if (usage) {
|
|
445
|
+
if (usage || runStatus === "error") {
|
|
340
446
|
llmCallCount = 1;
|
|
447
|
+
const generationUsage = usage ?? {
|
|
448
|
+
inputTokens: 0,
|
|
449
|
+
outputTokens: 0,
|
|
450
|
+
cacheReadTokens: 0,
|
|
451
|
+
cacheWriteTokens: 0,
|
|
452
|
+
model: loopOpts.model,
|
|
453
|
+
};
|
|
341
454
|
const llmSpanId = spanId();
|
|
342
455
|
const llmSpan = {
|
|
343
456
|
id: llmSpanId,
|
|
@@ -346,11 +459,11 @@ export async function instrumentAgentLoop(opts) {
|
|
|
346
459
|
userId,
|
|
347
460
|
parentSpanId,
|
|
348
461
|
spanType: "llm_call",
|
|
349
|
-
name:
|
|
350
|
-
inputTokens:
|
|
351
|
-
outputTokens:
|
|
352
|
-
cacheReadTokens:
|
|
353
|
-
cacheWriteTokens:
|
|
462
|
+
name: generationUsage.model,
|
|
463
|
+
inputTokens: generationUsage.inputTokens,
|
|
464
|
+
outputTokens: generationUsage.outputTokens,
|
|
465
|
+
cacheReadTokens: generationUsage.cacheReadTokens,
|
|
466
|
+
cacheWriteTokens: generationUsage.cacheWriteTokens,
|
|
354
467
|
costCentsX100,
|
|
355
468
|
durationMs: totalDurationMs,
|
|
356
469
|
status: runStatus,
|
|
@@ -368,11 +481,11 @@ export async function instrumentAgentLoop(opts) {
|
|
|
368
481
|
engineName: typeof loopOpts.engine?.name === "string"
|
|
369
482
|
? loopOpts.engine.name
|
|
370
483
|
: undefined,
|
|
371
|
-
model:
|
|
372
|
-
inputTokens:
|
|
373
|
-
outputTokens:
|
|
374
|
-
cacheReadTokens:
|
|
375
|
-
cacheWriteTokens:
|
|
484
|
+
model: generationUsage.model,
|
|
485
|
+
inputTokens: generationUsage.inputTokens,
|
|
486
|
+
outputTokens: generationUsage.outputTokens,
|
|
487
|
+
cacheReadTokens: generationUsage.cacheReadTokens,
|
|
488
|
+
cacheWriteTokens: generationUsage.cacheWriteTokens,
|
|
376
489
|
costCentsX100,
|
|
377
490
|
durationMs: totalDurationMs,
|
|
378
491
|
status: runStatus,
|
|
@@ -380,6 +493,11 @@ export async function instrumentAgentLoop(opts) {
|
|
|
380
493
|
toolCalls: toolCallCount,
|
|
381
494
|
successfulTools,
|
|
382
495
|
failedTools,
|
|
496
|
+
tools: [...generationToolCalls.entries()]
|
|
497
|
+
.sort(([a], [b]) => a - b)
|
|
498
|
+
.map(([, detail]) => detail),
|
|
499
|
+
toolsTruncated: toolInvocationCounter > MAX_TRACKED_GENERATION_TOOL_CALLS,
|
|
500
|
+
delegation: opts.delegation,
|
|
383
501
|
createdAt: runStart,
|
|
384
502
|
experimentAssignments: opts.experimentAssignments,
|
|
385
503
|
modelSelectionSource: opts.modelSelectionSource,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"traces.js","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAEA,OAAO,EAAkB,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAEpE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,SAAS,MAAM;IACb,OAAO,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,qBAAqB,CAC5B,UAA8B,EAC9B,KAAa;IAEb,MAAM,MAAM,GAAG,UAAU,EAAE,IAAI,EAAE,CAAC;IAClC,IAAI,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACzE,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC;IACxD,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IACrD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAClD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;AAC9D,CAAC;AAED,SAAS,8BAA8B,CAAC,IAyBvC;IACC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;IAC7C,MAAM,UAAU,GAA4B;QAC1C,GAAG,0BAA0B,EAAE;QAC/B,MAAM,EAAE,qBAAqB;QAC7B,SAAS,EAAE,UAAU;QACrB,MAAM,EAAE,IAAI,CAAC,KAAK;QAClB,SAAS,EAAE,IAAI,CAAC,QAAQ;QACxB,cAAc,EAAE,IAAI,CAAC,YAAY;QACjC,OAAO,EAAE,IAAI,CAAC,SAAS;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ;QACR,YAAY,EAAE,IAAI,CAAC,WAAW;QAC9B,aAAa,EAAE,IAAI,CAAC,YAAY;QAChC,YAAY,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY;QAClD,iBAAiB,EAAE,IAAI,CAAC,eAAe;QACvC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;QACzC,eAAe,EAAE,IAAI,CAAC,aAAa;QACnC,QAAQ,EAAE,OAAO;QACjB,WAAW,EAAE,IAAI,CAAC,UAAU;QAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,gBAAgB,EAAE,IAAI,CAAC,eAAe;QACtC,YAAY,EAAE,IAAI,CAAC,WAAW;QAC9B,sBAAsB,EAAE,IAAI,CAAC,oBAAoB;QACjD,UAAU,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAClD,aAAa,EAAE,IAAI,CAAC,SAAS;QAC7B,YAAY,EAAE,IAAI,CAAC,KAAK;QACxB,cAAc,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS;QAC1C,WAAW,EAAE,IAAI,CAAC,SAAS;QAC3B,aAAa,EAAE,WAAW;QAC1B,aAAa,EAAE,IAAI,CAAC,YAAY;QAChC,SAAS,EAAE,IAAI,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ;QACtB,gBAAgB,EAAE,IAAI,CAAC,WAAW;QAClC,iBAAiB,EAAE,IAAI,CAAC,YAAY;QACpC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI;QAC/D,YAAY,EAAE,IAAI,CAAC,MAAM,KAAK,OAAO;QACrC,SAAS,EAAE,KAAK;QAChB,2BAA2B,EAAE,IAAI,CAAC,eAAe;QACjD,+BAA+B,EAAE,IAAI,CAAC,gBAAgB;QACtD,iBAAiB,EAAE,CAAC;QACpB,kBAAkB,EAAE,OAAO;KAC5B,CAAC;IACF,IAAI,IAAI,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAC;QACvC,UAAU,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB;aACnD,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;aAC5C,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,qBAAqB;aACxD,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;aACzC,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,IAAI,IAAI,CAAC,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,UAAU,CAAC,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;YACtE,UAAU,CAAC,kBAAkB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,IAAI,KAAK;QAAE,UAAU,CAAC,aAAa,GAAG,KAAK,CAAC;IAE5C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,yBAAyB,CAAC;aACnC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;YAClB,KAAK,CAAC,gBAAgB,EAAE,UAAU,EAAE;gBAClC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,SAAS;aACjC,CAAC,CAAC;QACL,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,iEAAiE;IACnE,CAAC;AACH,CAAC;AAED;;;;;;2BAM2B;AAC3B,MAAM,uBAAuB,GAC3B,uGAAuG,CAAC;AAE1G;;;wEAGwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,OAAO,EAAU,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,IAAqB;IACvD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC;QAAE,OAAO,YAAY,CAAC;IACnD,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;IAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QACtE,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,IAAI,MAAM,GAAwC,IAAI,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC5D,MAAM,GAAG,CAAC,MAAM,UAAU,CACxB,sBAAsB,CACvB,CAAwC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,MAAM,EAAE,8BAA8B,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1E,OAAO;QACL,GAAG,4BAA4B;QAC/B,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;QACjB,GAAG,8BAA8B,CAAC,MAAM,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IA+CzC;IACC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;IAC9B,MAAM,wBAAwB,GAC5B,MAAM,CAAC,wBAAwB,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,IAAI,MAAM;QAC1E,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;aACjB,IAAI,CAAC,CAAC,EAAE,8BAA8B,EAAE,EAAE,EAAE,CAC3C,8BAA8B,CAAC,QAAQ,EAAE;YACvC,MAAM;YACN,YAAY,EAAE,KAAK;SACpB,CAAC,CACH;aACA,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,yEAAyE;IACzE,oDAAoD;IACpD,MAAM,kBAAkB,GAAG,cAAc,CAAC,WAAW,EAAE;QACrD,cAAc,EAAE,KAAK;QACrB,iBAAiB,EAAE,QAAQ,IAAI,SAAS;QACxC,eAAe,EAAE,MAAM,IAAI,SAAS;QACpC,aAAa,EAAE,QAAQ,CAAC,KAAK;QAC7B,8BAA8B,EAAE,IAAI,CAAC,oBAAoB;QACzD,qBAAqB,EACnB,IAAI,CAAC,qBAAqB,EAAE,MAAM,KAAK,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY;YAC5C,CAAC,CAAC,SAAS;QACf,0BAA0B,EACxB,IAAI,CAAC,qBAAqB,EAAE,MAAM,KAAK,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS;YACzC,CAAC,CAAC,SAAS;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,oEAAoE;IACpE,MAAM,YAAY,GAAG,IAAI,GAAG,EAUzB,CAAC;IACJ,0EAA0E;IAC1E,2EAA2E;IAC3E,8EAA8E;IAC9E,+EAA+E;IAC/E,wEAAwE;IACxE,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEvD,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,wEAAwE;IACxE,gDAAgD;IAChD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C,MAAM,gBAAgB,GAAG,CAAC,KAAqB,EAAQ,EAAE;QACvD,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;gBACrB,iEAAiE;gBACjE,oEAAoE;gBACpE,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,KAAK,GAYP;oBACF,MAAM,EAAE,GAAG;oBACX,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;oBACnB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC;gBACF,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACjC,KAAK,cAAc,CAAC,WAAW,EAAE;oBAC/B,WAAW,EAAE,KAAK,CAAC,IAAI;iBACxB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;oBACf,IAAI,CAAC,IAAI;wBAAE,OAAO;oBAClB,sEAAsE;oBACtE,+DAA+D;oBAC/D,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;wBACpB,YAAY,CAAC,IAAI,EAAE;4BACjB,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM;4BAC9B,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,YAAY;yBAC3C,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;wBACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,KAAK;oBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;oBAC1B,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC;gBAC/B,MAAM,OAAO,GACX,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAChE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7B,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAC7B,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,OAAO,KAAK,SAAS;oBAChC,CAAC,CAAC,KAAK,CAAC,OAAO;oBACf,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;wBAChC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;4BAC/B,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACnD,IAAI,OAAO;oBAAE,WAAW,EAAE,CAAC;;oBACtB,eAAe,EAAE,CAAC;gBAEvB,uEAAuE;gBACvE,oEAAoE;gBACpE,MAAM,aAAa,GAAG;oBACpB,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAwB;oBAC9D,YAAY,EAAE,OAAO,CAAC,CAAC,CAAE,KAAK,CAAC,MAAiB,CAAC,CAAC,CAAC,IAAI;iBACxD,CAAC;gBACF,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;oBACtB,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAC3C,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE;wBAC7B,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,YAAY,EAAE,aAAa,CAAC,YAAY;wBACxC,UAAU,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE;qBACxC,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,OAAO,EAAE,CAAC;oBACnB,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC;gBACpC,CAAC;gBAED,MAAM,IAAI,GAAc;oBACtB,EAAE,EAAE,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE;oBAC/B,KAAK;oBACL,QAAQ;oBACR,MAAM;oBACN,YAAY;oBACZ,QAAQ,EAAE,WAAW;oBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,WAAW,EAAE,CAAC;oBACd,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;oBAClB,gBAAgB,EAAE,CAAC;oBACnB,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBACtD,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;oBACrC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;oBAC3C,QAAQ,EACN,MAAM,CAAC,eAAe,IAAI,OAAO;wBAC/B,CAAC,CAAC,yDAAyD;4BACzD,sDAAsD;4BACtD,uDAAuD;4BACvD,oCAAoC;4BACpC;gCACE,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAGzC;6BACF;wBACH,CAAC,CAAC,IAAI;oBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,IAAI,KAAiC,CAAC;IACtC,IAAI,SAAS,GAAwB,SAAS,CAAC;IAC/C,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,WAAW,GAAmC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IACxE,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QACzD,SAAS,GAAG,cAAc,EAAE,MAAM,IAAI,OAAO,CAAC;QAC9C,YAAY;YACV,cAAc,EAAE,YAAY,KAAK,SAAS;gBACxC,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC,CAAC,cAAc,CAAC,YAAY,CAAC;QAClC,MAAM,aAAa,GAAG,cAAc,EAAE,QAAQ,IAAI,IAAI,CAAC;QACvD,WAAW;YACT,WAAW,IAAI,aAAa;gBAC1B,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,EAAE;gBACtD,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;QAE1C,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,aAAa,GAAG,aAAa,CAC3B,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,eAAe,EACrB,KAAK,CAAC,gBAAgB,CACvB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,EAAE,CAAC;YACV,YAAY,GAAG,CAAC,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAc;gBACzB,EAAE,EAAE,SAAS;gBACb,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,YAAY;gBACZ,QAAQ,EAAE,UAAU;gBACpB,IAAI,EAAE,KAAK,CAAC,KAAK;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,aAAa;gBACb,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,QAAQ;aACpB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,8BAA8B,CAAC;gBAC7B,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,YAAY;gBACZ,SAAS;gBACT,UAAU,EACR,OAAO,QAAQ,CAAC,MAAM,EAAE,IAAI,KAAK,QAAQ;oBACvC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI;oBACtB,CAAC,CAAC,SAAS;gBACf,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,aAAa;gBACb,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,SAAS,EAAE,aAAa;gBACxB,eAAe;gBACf,WAAW;gBACX,SAAS,EAAE,QAAQ;gBACnB,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;gBACjD,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;aAChD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAc;YAC5B,EAAE,EAAE,YAAY;YAChB,KAAK;YACL,QAAQ;YACR,MAAM;YACN,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,WAAW;YACrB,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACpC,YAAY,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YACtC,eAAe,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;YAC5C,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,IAAI,CAAC;YAC9C,aAAa;YACb,UAAU,EAAE,eAAe;YAC3B,MAAM,EAAE,SAAS;YACjB,YAAY;YACZ,QAAQ,EAAE,WAAW;YACrB,SAAS,EAAE,QAAQ;SACpB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAiB;YAC5B,KAAK;YACL,QAAQ;YACR,MAAM;YACN,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,aAAa;YACxB,eAAe;YACf,WAAW;YACX,eAAe;YACf,kBAAkB,EAAE,aAAa;YACjC,gBAAgB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACzC,iBAAiB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YAC3C,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;YACrC,SAAS,EAAE,QAAQ;SACpB,CAAC;QAEF,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9D,uEAAuE;QACvE,uEAAuE;QACvE,6EAA6E;QAC7E,0EAA0E;QAC1E,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE;oBACjD,MAAM,EAAE,SAAS;oBACjB,YAAY;oBACZ,UAAU,EAAE;wBACV,WAAW,EAAE,KAAK,CAAC,KAAK;wBACxB,kBAAkB,EAAE,KAAK,CAAC,WAAW;wBACrC,mBAAmB,EAAE,KAAK,CAAC,YAAY;wBACvC,uBAAuB,EAAE,KAAK,CAAC,eAAe;wBAC9C,wBAAwB,EAAE,KAAK,CAAC,gBAAgB;wBAChD,qBAAqB,EAAE,aAAa;qBACrC;iBACF,CAAC,CAAC;YACL,CAAC;YACD,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACzC,YAAY,CAAC,QAAQ,EAAE;oBACrB,MAAM,EAAE,OAAO;oBACf,YAAY,EAAE,mCAAmC;iBAClD,CAAC,CAAC;YACL,CAAC;YACD,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC1B,YAAY,CAAC,MAAM,kBAAkB,EAAE;gBACrC,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,UAAU,EAAE;oBACV,kBAAkB,EAAE,aAAa;oBACjC,wBAAwB,EAAE,eAAe;oBACzC,oBAAoB,EAAE,WAAW;oBACjC,mBAAmB,EAAE,eAAe;oBACpC,oBAAoB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC7C,qBAAqB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;oBAC/C,uBAAuB,EAAE,aAAa;iBACvC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,8EAA8E;IAC9E,wEAAwE;IACxE,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,MAAM,wBAAwB,CAAC;YACzD,IAAI,iBAAiB,EAAE,CAAC;gBACtB,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBAClE,MAAM,sBAAsB,CAAC;oBAC3B,eAAe,EAAE,MAAM,CAAC,sBAAsB;oBAC9C,sBAAsB,EAAE,iBAAiB,CAAC,KAAK;oBAC/C,IAAI,EAAE,IAAI,CAAC,cAAc;oBACzB,cAAc,EAAE,iBAAiB,CAAC,KAAK;oBACvC,0BAA0B,EAAE,KAAK;oBACjC,QAAQ;oBACR,MAAM;oBACN,UAAU,EAAE,MAAM,CAAC,2BAA2B;iBAC/C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;IACH,CAAC;IAED,OAAO,KAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,KAAkB,EAClB,OAAqB,EACrB,KAAa,EACb,MAA2B;IAE3B,MAAM,EAAE,eAAe,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3E,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAElD,qDAAqD;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,WAAW,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC","sourcesContent":["import type { AgentLoopUsage } from \"../agent/production-agent.js\";\nimport type { AgentChatEvent, AgentToolInput } from \"../agent/types.js\";\nimport { type AgentSpan, endAgentSpan, startAgentSpan } from \"./tracing.js\";\nimport { trackingIdentityProperties } from \"./tracking-identity.js\";\nimport type { TraceSpan, TraceSummary, ObservabilityConfig } from \"./types.js\";\nimport { DEFAULT_OBSERVABILITY_CONFIG } from \"./types.js\";\n\nfunction spanId(): string {\n return `span-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;\n}\n\nfunction llmProviderFromEngine(\n engineName: string | undefined,\n model: string,\n): string {\n const engine = engineName?.trim();\n if (engine?.startsWith(\"ai-sdk:\")) return engine.slice(\"ai-sdk:\".length);\n if (engine) return engine;\n if (/claude|anthropic/i.test(model)) return \"anthropic\";\n if (/gpt|openai|codex/i.test(model)) return \"openai\";\n if (/gemini|google/i.test(model)) return \"google\";\n return \"unknown\";\n}\n\nfunction costUsdFromCenticents(value: number): number {\n return Math.round((value / 10_000) * 1_000_000) / 1_000_000;\n}\n\nfunction emitLlmGenerationTrackingEvent(args: {\n runId: string;\n threadId: string | null;\n userId: string | null;\n parentSpanId: string;\n llmSpanId: string;\n engineName: string | undefined;\n model: string;\n inputTokens: number;\n outputTokens: number;\n cacheReadTokens: number;\n cacheWriteTokens: number;\n costCentsX100: number;\n durationMs: number;\n status: \"success\" | \"error\";\n errorMessage: string | null;\n toolCalls: number;\n successfulTools: number;\n failedTools: number;\n createdAt: number;\n experimentAssignments?: Array<{\n experimentId: string;\n variantId: string;\n }>;\n modelSelectionSource?: string;\n}): void {\n const provider = llmProviderFromEngine(args.engineName, args.model);\n const costUsd = costUsdFromCenticents(args.costCentsX100);\n const error = args.errorMessage ?? undefined;\n const properties: Record<string, unknown> = {\n ...trackingIdentityProperties(),\n source: \"agent_observability\",\n span_type: \"llm_call\",\n run_id: args.runId,\n thread_id: args.threadId,\n parent_span_id: args.parentSpanId,\n span_id: args.llmSpanId,\n model: args.model,\n provider,\n input_tokens: args.inputTokens,\n output_tokens: args.outputTokens,\n total_tokens: args.inputTokens + args.outputTokens,\n cache_read_tokens: args.cacheReadTokens,\n cache_write_tokens: args.cacheWriteTokens,\n cost_cents_x100: args.costCentsX100,\n cost_usd: costUsd,\n duration_ms: args.durationMs,\n status: args.status,\n tool_calls: args.toolCalls,\n successful_tools: args.successfulTools,\n failed_tools: args.failedTools,\n model_selection_source: args.modelSelectionSource,\n created_at: new Date(args.createdAt).toISOString(),\n created_at_ms: args.createdAt,\n $ai_trace_id: args.runId,\n $ai_session_id: args.threadId ?? undefined,\n $ai_span_id: args.llmSpanId,\n $ai_span_name: \"agent_run\",\n $ai_parent_id: args.parentSpanId,\n $ai_model: args.model,\n $ai_provider: provider,\n $ai_input_tokens: args.inputTokens,\n $ai_output_tokens: args.outputTokens,\n $ai_latency: Math.round((args.durationMs / 1000) * 1000) / 1000,\n $ai_is_error: args.status === \"error\",\n $ai_error: error,\n $ai_cache_read_input_tokens: args.cacheReadTokens,\n $ai_cache_creation_input_tokens: args.cacheWriteTokens,\n $ai_request_count: 1,\n $ai_total_cost_usd: costUsd,\n };\n if (args.experimentAssignments?.length) {\n properties.experiment_ids = args.experimentAssignments\n .map((assignment) => assignment.experimentId)\n .join(\",\");\n properties.experiment_variants = args.experimentAssignments\n .map((assignment) => assignment.variantId)\n .join(\",\");\n if (args.experimentAssignments.length === 1) {\n properties.experiment_id = args.experimentAssignments[0].experimentId;\n properties.experiment_variant = args.experimentAssignments[0].variantId;\n }\n }\n if (error) properties.error_message = error;\n\n for (const key of Object.keys(properties)) {\n if (properties[key] === undefined) delete properties[key];\n }\n\n try {\n void import(\"../tracking/registry.js\")\n .then(({ track }) => {\n track(\"$ai_generation\", properties, {\n userId: args.userId ?? undefined,\n });\n })\n .catch(() => {});\n } catch {\n // Tracking must never affect the agent run or trace persistence.\n }\n}\n\n/** Keys whose values are stripped from persisted tool inputs when\n * `captureToolArgs` is enabled. Matched case-insensitively and tolerant\n * of `_` / `-` separators. M14 in the MCP/A2A audit: tool calls\n * routinely receive credentials verbatim (db-exec INSERTs, fetchTool\n * Authorization headers, ad-hoc bearer tokens) — keeping those values\n * out of agent_trace_spans.metadata avoids long-term storage of\n * short-lived secrets. */\nconst SENSITIVE_FIELD_PATTERN =\n /^(authorization|cookie|api[_-]?key|password|secret|token|access[_-]?token|refresh[_-]?token|bearer)$/i;\n\n/** Recursively walk a structured value and replace sensitive field\n * values with the literal string \"[REDACTED]\". Pure (returns a copy);\n * the original input is never mutated. Cycles are tolerated via a\n * small WeakSet seen-tracker that returns \"[Circular]\" for repeats. */\nexport function redactSensitiveFields(value: unknown): unknown {\n return redactWalk(value, new WeakSet<object>());\n}\n\nfunction redactWalk(value: unknown, seen: WeakSet<object>): unknown {\n if (value === null || typeof value !== \"object\") return value;\n if (seen.has(value as object)) return \"[Circular]\";\n seen.add(value as object);\n if (Array.isArray(value)) {\n return value.map((v) => redactWalk(v, seen));\n }\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n if (SENSITIVE_FIELD_PATTERN.test(k)) {\n out[k] = \"[REDACTED]\";\n } else {\n out[k] = redactWalk(v, seen);\n }\n }\n return out;\n}\n\nexport async function getObservabilityConfig(): Promise<ObservabilityConfig> {\n let stored: Partial<ObservabilityConfig> | null = null;\n try {\n const { getSetting } = await import(\"../settings/store.js\");\n stored = (await getSetting(\n \"observability-config\",\n )) as Partial<ObservabilityConfig> | null;\n } catch {}\n const { resolveInferredSentimentConfig } = await import(\"./sentiment.js\");\n return {\n ...DEFAULT_OBSERVABILITY_CONFIG,\n ...(stored ?? {}),\n ...resolveInferredSentimentConfig(stored),\n };\n}\n\nexport async function instrumentAgentLoop(opts: {\n runAgentLoop: (loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n }) => Promise<AgentLoopUsage>;\n loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n };\n runId: string;\n threadId: string | null;\n /** Owner of this run; persisted on every span + summary so dashboard\n * reads can filter to a single user. Null for unauthenticated callers\n * (background tasks, etc.) — those rows aren't returned by per-user\n * reads. */\n userId: string | null;\n config: ObservabilityConfig;\n metadata?: Record<string, unknown> | null;\n experimentAssignments?: Array<{\n experimentId: string;\n variantId: string;\n }>;\n modelSelectionSource?: string;\n /** Raw user-authored message before prompt/context enrichment. */\n sentimentInput?: string;\n classifyError?: (error: unknown) =>\n | {\n status?: \"success\" | \"error\";\n errorMessage?: string | null;\n metadata?: Record<string, unknown> | null;\n }\n | null\n | undefined;\n}): Promise<AgentLoopUsage> {\n const { runAgentLoop, loopOpts, runId, threadId, userId, config } = opts;\n const runStart = Date.now();\n const parentSpanId = spanId();\n const precedingResponsePromise =\n config.inferredSentimentEnabled && opts.sentimentInput && threadId && userId\n ? import(\"./store.js\")\n .then(({ getLatestTraceSummaryForThread }) =>\n getLatestTraceSummaryForThread(threadId, {\n userId,\n excludeRunId: runId,\n }),\n )\n .catch(() => null)\n : Promise.resolve(null);\n\n // Optional OpenTelemetry root span for this run. No-ops unless a host has\n // installed `@opentelemetry/api` and registered a provider. The promise is\n // resolved before the loop runs so child tool/model spans can parent under\n // it conceptually (we keep them flat in the same tracer, which is enough\n // for the dashboards an embedding app would build).\n const otelRunSpanPromise = startAgentSpan(\"agent.run\", {\n \"agent.run_id\": runId,\n \"agent.thread_id\": threadId ?? undefined,\n \"agent.user_id\": userId ?? undefined,\n \"agent.model\": loopOpts.model,\n \"agent.model_selection_source\": opts.modelSelectionSource,\n \"agent.experiment_id\":\n opts.experimentAssignments?.length === 1\n ? opts.experimentAssignments[0].experimentId\n : undefined,\n \"agent.experiment_variant\":\n opts.experimentAssignments?.length === 1\n ? opts.experimentAssignments[0].variantId\n : undefined,\n });\n\n const spans: TraceSpan[] = [];\n let toolInvocationCounter = 0;\n // Keyed by counter to handle concurrent calls to the same tool name\n const pendingTools = new Map<\n number,\n {\n spanId: string;\n startMs: number;\n toolName: string;\n input: AgentToolInput;\n otelSpan: AgentSpan | null;\n endResult?: { status: \"success\" | \"error\"; errorMessage: string | null };\n }\n >();\n // Secondary index: tool name → FIFO queue of pending invocation counters.\n // tool_start/tool_done events carry only the tool name (no call id), so to\n // pair starts and dones correctly when the agent runs concurrent calls to the\n // same tool name (read-only / parallelSafe batches via Promise.all), we keep a\n // queue per name and match each done to the OLDEST still-pending start.\n const toolNameToCounters = new Map<string, number[]>();\n\n let toolCallCount = 0;\n let successfulTools = 0;\n let failedTools = 0;\n\n // Track in-flight OTel tool spans so they're all ended even if the loop\n // throws before a matching `tool_done` arrives.\n const openOtelToolSpans = new Set<AgentSpan>();\n\n const instrumentedSend = (event: AgentChatEvent): void => {\n try {\n if (event.type === \"tool_start\") {\n const counter = toolInvocationCounter++;\n const sid = spanId();\n // Start the OTel tool span synchronously-ish: kick off the async\n // resolution and stash the span once it lands. Tool spans are short\n // and the api tracer is synchronous in practice, but we tolerate the\n // microtask gap by recording the span on the pending entry when ready.\n const entry: {\n spanId: string;\n startMs: number;\n toolName: string;\n input: AgentToolInput;\n otelSpan: AgentSpan | null;\n // Set by the done handler if it fires before the span promise\n // resolves, so the resolved span is ended with the correct status.\n endResult?: {\n status: \"success\" | \"error\";\n errorMessage: string | null;\n };\n } = {\n spanId: sid,\n startMs: Date.now(),\n toolName: event.tool,\n input: event.input,\n otelSpan: null,\n };\n pendingTools.set(counter, entry);\n void startAgentSpan(\"tool.call\", {\n \"tool.name\": event.tool,\n }).then((span) => {\n if (!span) return;\n // If `tool_done` already ran for this call, end the span now with the\n // status it recorded; otherwise stash it for the done handler.\n if (entry.endResult) {\n endAgentSpan(span, {\n status: entry.endResult.status,\n errorMessage: entry.endResult.errorMessage,\n });\n } else {\n entry.otelSpan = span;\n openOtelToolSpans.add(span);\n }\n });\n const queue = toolNameToCounters.get(event.tool);\n if (queue) queue.push(counter);\n else toolNameToCounters.set(event.tool, [counter]);\n } else if (event.type === \"tool_done\") {\n const queue = toolNameToCounters.get(event.tool);\n const counter = queue?.shift();\n const pending =\n counter !== undefined ? pendingTools.get(counter) : undefined;\n if (counter !== undefined) {\n pendingTools.delete(counter);\n if (queue && queue.length === 0)\n toolNameToCounters.delete(event.tool);\n }\n toolCallCount++;\n\n const isError =\n typeof event.isError === \"boolean\"\n ? event.isError\n : typeof event.result === \"string\" &&\n (event.result.startsWith(\"Error\") ||\n event.result.startsWith(\"Error running \"));\n if (isError) failedTools++;\n else successfulTools++;\n\n // Finalize the OTel tool span. If the span promise hasn't resolved yet\n // we record the result on the entry so its `.then` handler ends it.\n const otelEndResult = {\n status: (isError ? \"error\" : \"success\") as \"success\" | \"error\",\n errorMessage: isError ? (event.result as string) : null,\n };\n if (pending?.otelSpan) {\n openOtelToolSpans.delete(pending.otelSpan);\n endAgentSpan(pending.otelSpan, {\n status: otelEndResult.status,\n errorMessage: otelEndResult.errorMessage,\n attributes: { \"tool.name\": event.tool },\n });\n } else if (pending) {\n pending.endResult = otelEndResult;\n }\n\n const span: TraceSpan = {\n id: pending?.spanId ?? spanId(),\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"tool_call\",\n name: event.tool,\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n costCentsX100: 0,\n durationMs: pending ? Date.now() - pending.startMs : 0,\n status: isError ? \"error\" : \"success\",\n errorMessage: isError ? event.result : null,\n metadata:\n config.captureToolArgs && pending\n ? // Strip Authorization/api-key/token-shaped values before\n // persisting (M14 in the MCP/A2A audit). Tool-runtime\n // execution still sees the unredacted input — only the\n // long-lived span row is sanitized.\n {\n input: redactSensitiveFields(pending.input) as Record<\n string,\n string\n >,\n }\n : null,\n createdAt: Date.now(),\n };\n spans.push(span);\n }\n } catch {}\n\n loopOpts.send(event);\n };\n\n let usage: AgentLoopUsage | undefined;\n let runStatus: \"success\" | \"error\" = \"success\";\n let errorMessage: string | null = null;\n let runMetadata: Record<string, unknown> | null = opts.metadata ?? null;\n try {\n usage = await runAgentLoop({ ...loopOpts, send: instrumentedSend });\n } catch (err: any) {\n const classification = opts.classifyError?.(err) ?? null;\n runStatus = classification?.status ?? \"error\";\n errorMessage =\n classification?.errorMessage === undefined\n ? (err?.message ?? String(err))\n : classification.errorMessage;\n const errorMetadata = classification?.metadata ?? null;\n runMetadata =\n runMetadata || errorMetadata\n ? { ...(runMetadata ?? {}), ...(errorMetadata ?? {}) }\n : null;\n throw err;\n } finally {\n const runEnd = Date.now();\n const totalDurationMs = runEnd - runStart;\n\n let costCentsX100 = 0;\n try {\n const { calculateCost } = await import(\"../usage/store.js\");\n if (usage) {\n costCentsX100 = calculateCost(\n usage.inputTokens,\n usage.outputTokens,\n usage.model,\n usage.cacheReadTokens,\n usage.cacheWriteTokens,\n );\n }\n } catch {}\n\n let llmCallCount = 0;\n if (usage) {\n llmCallCount = 1;\n const llmSpanId = spanId();\n const llmSpan: TraceSpan = {\n id: llmSpanId,\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"llm_call\",\n name: usage.model,\n inputTokens: usage.inputTokens,\n outputTokens: usage.outputTokens,\n cacheReadTokens: usage.cacheReadTokens,\n cacheWriteTokens: usage.cacheWriteTokens,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: null,\n createdAt: runStart,\n };\n spans.push(llmSpan);\n emitLlmGenerationTrackingEvent({\n runId,\n threadId,\n userId,\n parentSpanId,\n llmSpanId,\n engineName:\n typeof loopOpts.engine?.name === \"string\"\n ? loopOpts.engine.name\n : undefined,\n model: usage.model,\n inputTokens: usage.inputTokens,\n outputTokens: usage.outputTokens,\n cacheReadTokens: usage.cacheReadTokens,\n cacheWriteTokens: usage.cacheWriteTokens,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n toolCalls: toolCallCount,\n successfulTools,\n failedTools,\n createdAt: runStart,\n experimentAssignments: opts.experimentAssignments,\n modelSelectionSource: opts.modelSelectionSource,\n });\n }\n\n const parentSpan: TraceSpan = {\n id: parentSpanId,\n runId,\n threadId,\n userId,\n parentSpanId: null,\n spanType: \"agent_run\",\n name: \"agent_run\",\n inputTokens: usage?.inputTokens ?? 0,\n outputTokens: usage?.outputTokens ?? 0,\n cacheReadTokens: usage?.cacheReadTokens ?? 0,\n cacheWriteTokens: usage?.cacheWriteTokens ?? 0,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: runMetadata,\n createdAt: runStart,\n };\n spans.push(parentSpan);\n\n const summary: TraceSummary = {\n runId,\n threadId,\n userId,\n totalSpans: spans.length,\n llmCalls: llmCallCount,\n toolCalls: toolCallCount,\n successfulTools,\n failedTools,\n totalDurationMs,\n totalCostCentsX100: costCentsX100,\n totalInputTokens: usage?.inputTokens ?? 0,\n totalOutputTokens: usage?.outputTokens ?? 0,\n model: usage?.model ?? loopOpts.model,\n createdAt: runStart,\n };\n\n writeTraceData(spans, summary, runId, config).catch(() => {});\n\n // OpenTelemetry export (no-op unless a provider is registered). Emit a\n // self-contained `llm.call` span carrying model + token usage, end any\n // tool spans still open (loop threw mid-tool), and end the run span. Awaited\n // so the spans are emitted before the function returns; cheap when no-op.\n try {\n if (usage) {\n endAgentSpan(await startAgentSpan(\"llm.call\", {}), {\n status: runStatus,\n errorMessage,\n attributes: {\n \"llm.model\": usage.model,\n \"llm.input_tokens\": usage.inputTokens,\n \"llm.output_tokens\": usage.outputTokens,\n \"llm.cache_read_tokens\": usage.cacheReadTokens,\n \"llm.cache_write_tokens\": usage.cacheWriteTokens,\n \"llm.cost_cents_x100\": costCentsX100,\n },\n });\n }\n for (const toolSpan of openOtelToolSpans) {\n endAgentSpan(toolSpan, {\n status: \"error\",\n errorMessage: \"Agent run ended before tool_done.\",\n });\n }\n openOtelToolSpans.clear();\n endAgentSpan(await otelRunSpanPromise, {\n status: runStatus,\n errorMessage,\n attributes: {\n \"agent.tool_calls\": toolCallCount,\n \"agent.successful_tools\": successfulTools,\n \"agent.failed_tools\": failedTools,\n \"agent.duration_ms\": totalDurationMs,\n \"agent.input_tokens\": usage?.inputTokens ?? 0,\n \"agent.output_tokens\": usage?.outputTokens ?? 0,\n \"agent.cost_cents_x100\": costCentsX100,\n },\n });\n } catch {\n // OTel export must never break the run.\n }\n }\n\n // Classify only after the main loop has finished so the tiny managed Luna\n // request cannot contend with the user's response for a gateway slot. This\n // short, awaited tail keeps serverless runtimes alive long enough to emit the\n // event, while the response content has already streamed to the client.\n if (usage && opts.sentimentInput) {\n try {\n const precedingResponse = await precedingResponsePromise;\n if (precedingResponse) {\n const { inferAndTrackSentiment } = await import(\"./sentiment.js\");\n await inferAndTrackSentiment({\n classifierModel: config.inferredSentimentModel,\n precedingResponseModel: precedingResponse.model,\n text: opts.sentimentInput,\n precedingRunId: precedingResponse.runId,\n classificationTriggerRunId: runId,\n threadId,\n userId,\n sampleRate: config.inferredSentimentSampleRate,\n });\n }\n } catch {\n // Optional inference must never alter the result of the main run.\n }\n }\n\n return usage!;\n}\n\nasync function writeTraceData(\n spans: TraceSpan[],\n summary: TraceSummary,\n runId: string,\n config: ObservabilityConfig,\n): Promise<void> {\n const { insertTraceSpan, upsertTraceSummary } = await import(\"./store.js\");\n await Promise.all(spans.map((s) => insertTraceSpan(s).catch(() => {})));\n await upsertTraceSummary(summary).catch(() => {});\n\n // Fire automated evals after trace data is persisted\n try {\n const { evaluateRun } = await import(\"./evals.js\");\n await evaluateRun(runId, { sampleRate: config.evalSampleRate });\n } catch {}\n}\n"]}
|
|
1
|
+
{"version":3,"file":"traces.js","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAEA,OAAO,EAAkB,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAEpE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,SAAS,MAAM;IACb,OAAO,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,qBAAqB,CAC5B,UAA8B,EAC9B,KAAa;IAEb,MAAM,MAAM,GAAG,UAAU,EAAE,IAAI,EAAE,CAAC;IAClC,IAAI,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACzE,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC;IACxD,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IACrD,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAClD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;AAC9D,CAAC;AAED,MAAM,iCAAiC,GAAG,EAAE,CAAC;AAU7C,SAAS,8BAA8B,CAAC,IAkCvC;IACC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACpE,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;IAC7C,MAAM,UAAU,GAA4B;QAC1C,GAAG,0BAA0B,EAAE;QAC/B,MAAM,EAAE,qBAAqB;QAC7B,SAAS,EAAE,UAAU;QACrB,MAAM,EAAE,IAAI,CAAC,KAAK;QAClB,SAAS,EAAE,IAAI,CAAC,QAAQ;QACxB,cAAc,EAAE,IAAI,CAAC,YAAY;QACjC,OAAO,EAAE,IAAI,CAAC,SAAS;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ;QACR,YAAY,EAAE,IAAI,CAAC,WAAW;QAC9B,aAAa,EAAE,IAAI,CAAC,YAAY;QAChC,YAAY,EAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY;QAClD,iBAAiB,EAAE,IAAI,CAAC,eAAe;QACvC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB;QACzC,eAAe,EAAE,IAAI,CAAC,aAAa;QACnC,QAAQ,EAAE,OAAO;QACjB,WAAW,EAAE,IAAI,CAAC,UAAU;QAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,gBAAgB,EAAE,IAAI,CAAC,eAAe;QACtC,YAAY,EAAE,IAAI,CAAC,WAAW;QAC9B,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,eAAe,EAAE,IAAI,CAAC,cAAc;QACpC,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAC7C,mBAAmB,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ;QAC9C,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS;QACtC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM;QACpC,aAAa,EAAE,IAAI,CAAC,UAAU,EAAE,WAAW;QAC3C,cAAc,EAAE,IAAI,CAAC,UAAU,EAAE,YAAY;QAC7C,sBAAsB,EAAE,IAAI,CAAC,oBAAoB;QACjD,UAAU,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QAClD,aAAa,EAAE,IAAI,CAAC,SAAS;QAC7B,YAAY,EAAE,IAAI,CAAC,KAAK;QACxB,cAAc,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS;QAC1C,WAAW,EAAE,IAAI,CAAC,SAAS;QAC3B,aAAa,EAAE,WAAW;QAC1B,aAAa,EAAE,IAAI,CAAC,YAAY;QAChC,SAAS,EAAE,IAAI,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ;QACtB,gBAAgB,EAAE,IAAI,CAAC,WAAW;QAClC,iBAAiB,EAAE,IAAI,CAAC,YAAY;QACpC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI;QAC/D,YAAY,EAAE,IAAI,CAAC,MAAM,KAAK,OAAO;QACrC,SAAS,EAAE,KAAK;QAChB,2BAA2B,EAAE,IAAI,CAAC,eAAe;QACjD,+BAA+B,EAAE,IAAI,CAAC,gBAAgB;QACtD,iBAAiB,EAAE,CAAC;QACpB,kBAAkB,EAAE,OAAO;KAC5B,CAAC;IACF,IAAI,IAAI,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAC;QACvC,UAAU,CAAC,cAAc,GAAG,IAAI,CAAC,qBAAqB;aACnD,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC;aAC5C,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,qBAAqB;aACxD,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC;aACzC,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,IAAI,IAAI,CAAC,qBAAqB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5C,UAAU,CAAC,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;YACtE,UAAU,CAAC,kBAAkB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,IAAI,KAAK;QAAE,UAAU,CAAC,aAAa,GAAG,KAAK,CAAC;IAE5C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,yBAAyB,CAAC;aACnC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;YAClB,KAAK,CAAC,gBAAgB,EAAE,UAAU,EAAE;gBAClC,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,SAAS;aACjC,CAAC,CAAC;QACL,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,iEAAiE;IACnE,CAAC;AACH,CAAC;AAED;;;;;;2BAM2B;AAC3B,MAAM,uBAAuB,GAC3B,uGAAuG,CAAC;AAE1G;;;wEAGwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,OAAO,EAAU,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,IAAqB;IACvD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC;QAAE,OAAO,YAAY,CAAC;IACnD,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;IAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QACtE,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,IAAI,MAAM,GAAwC,IAAI,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC5D,MAAM,GAAG,CAAC,MAAM,UAAU,CACxB,sBAAsB,CACvB,CAAwC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,MAAM,EAAE,8BAA8B,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1E,OAAO;QACL,GAAG,4BAA4B;QAC/B,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC;QACjB,GAAG,8BAA8B,CAAC,MAAM,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAwDzC;IACC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;IAC9B,MAAM,wBAAwB,GAC5B,MAAM,CAAC,wBAAwB,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,IAAI,MAAM;QAC1E,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;aACjB,IAAI,CAAC,CAAC,EAAE,8BAA8B,EAAE,EAAE,EAAE,CAC3C,8BAA8B,CAAC,QAAQ,EAAE;YACvC,MAAM;YACN,YAAY,EAAE,KAAK;SACpB,CAAC,CACH;aACA,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,yEAAyE;IACzE,oDAAoD;IACpD,MAAM,kBAAkB,GAAG,cAAc,CAAC,WAAW,EAAE;QACrD,cAAc,EAAE,KAAK;QACrB,iBAAiB,EAAE,QAAQ,IAAI,SAAS;QACxC,eAAe,EAAE,MAAM,IAAI,SAAS;QACpC,aAAa,EAAE,QAAQ,CAAC,KAAK;QAC7B,8BAA8B,EAAE,IAAI,CAAC,oBAAoB;QACzD,qBAAqB,EACnB,IAAI,CAAC,qBAAqB,EAAE,MAAM,KAAK,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,YAAY;YAC5C,CAAC,CAAC,SAAS;QACf,0BAA0B,EACxB,IAAI,CAAC,qBAAqB,EAAE,MAAM,KAAK,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,SAAS;YACzC,CAAC,CAAC,SAAS;KAChB,CAAC,CAAC;IAEH,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,oEAAoE;IACpE,MAAM,YAAY,GAAG,IAAI,GAAG,EAWzB,CAAC;IACJ,4EAA4E;IAC5E,8EAA8E;IAC9E,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAoB,CAAC;IACvD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,wEAAwE;IACxE,gDAAgD;IAChD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C,MAAM,gBAAgB,GAAG,CAAC,KAAqB,EAAQ,EAAE;QACvD,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;gBACrB,iEAAiE;gBACjE,oEAAoE;gBACpE,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,KAAK,GAaP;oBACF,MAAM,EAAE,GAAG;oBACX,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;oBACnB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC;gBACF,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACjC,IAAI,KAAK,CAAC,EAAE;oBAAE,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBACzD,KAAK,cAAc,CAAC,WAAW,EAAE;oBAC/B,WAAW,EAAE,KAAK,CAAC,IAAI;iBACxB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;oBACf,IAAI,CAAC,IAAI;wBAAE,OAAO;oBAClB,sEAAsE;oBACtE,+DAA+D;oBAC/D,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;wBACpB,YAAY,CAAC,IAAI,EAAE;4BACjB,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM;4BAC9B,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,YAAY;yBAC3C,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;wBACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,KAAK;oBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;oBAC1B,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,MAAM,aAAa,GAAG,KAAK,CAAC,EAAE;oBAC5B,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBACnC,CAAC,CAAC,SAAS,CAAC;gBACd,MAAM,gBAAgB,GACpB,KAAK,CAAC,EAAE,IAAI,aAAa,KAAK,SAAS,IAAI,KAAK;oBAC9C,CAAC,CAAC,KAAK,CAAC,SAAS,CACb,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,MAAM,CACpD;oBACH,CAAC,CAAC,CAAC,CAAC,CAAC;gBACT,MAAM,OAAO,GACX,aAAa;oBACb,CAAC,KAAK,CAAC,EAAE;wBACP,CAAC,CAAC,gBAAgB,IAAI,CAAC;4BACrB,CAAC,CAAC,KAAK,EAAE,CAAC,gBAAgB,CAAC;4BAC3B,CAAC,CAAC,SAAS;wBACb,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;gBACtB,MAAM,OAAO,GACX,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAChE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7B,IAAI,OAAO,EAAE,MAAM;wBAAE,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBAChE,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,gBAAgB,IAAI,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;wBACpE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC1C,IAAI,UAAU,IAAI,CAAC;4BAAE,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;oBACnD,CAAC;oBACD,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAC7B,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE9B,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC;gBAC7C,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,OAAO,KAAK,SAAS;oBAChC,CAAC,CAAC,KAAK,CAAC,OAAO;oBACf,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;wBAChC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;4BAC/B,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBACnD,IAAI,OAAO;oBAAE,WAAW,EAAE,CAAC;;oBACtB,eAAe,EAAE,CAAC;gBAEvB,IACE,OAAO,KAAK,SAAS;oBACrB,OAAO,GAAG,iCAAiC;oBAC3C,OAAO,EACP,CAAC;oBACD,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE;wBAC/B,IAAI,EAAE,OAAO,CAAC,QAAQ;wBACtB,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,GAAG,QAAQ,CAAC;wBAC1D,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC;wBACtD,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;wBACrC,WAAW,EAAE,CAAC,OAAO;4BACnB,CAAC,CAAC,IAAI;4BACN,CAAC,CAAC,aAAa;gCACb,CAAC,CAAC,YAAY;gCACd,CAAC,CAAC,uBAAuB;qBAC9B,CAAC,CAAC;gBACL,CAAC;gBAED,uEAAuE;gBACvE,oEAAoE;gBACpE,MAAM,aAAa,GAAG;oBACpB,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAwB;oBAC9D,YAAY,EAAE,OAAO,CAAC,CAAC,CAAE,KAAK,CAAC,MAAiB,CAAC,CAAC,CAAC,IAAI;iBACxD,CAAC;gBACF,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;oBACtB,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAC3C,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE;wBAC7B,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,YAAY,EAAE,aAAa,CAAC,YAAY;wBACxC,UAAU,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE;qBACxC,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,OAAO,EAAE,CAAC;oBACnB,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC;gBACpC,CAAC;gBAED,MAAM,IAAI,GAAc;oBACtB,EAAE,EAAE,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE;oBAC/B,KAAK;oBACL,QAAQ;oBACR,MAAM;oBACN,YAAY;oBACZ,QAAQ,EAAE,WAAW;oBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,WAAW,EAAE,CAAC;oBACd,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;oBAClB,gBAAgB,EAAE,CAAC;oBACnB,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;oBACrC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;oBAC3C,QAAQ,EACN,MAAM,CAAC,eAAe,IAAI,OAAO;wBAC/B,CAAC,CAAC,yDAAyD;4BACzD,sDAAsD;4BACtD,uDAAuD;4BACvD,oCAAoC;4BACpC;gCACE,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAGzC;6BACF;wBACH,CAAC,CAAC,IAAI;oBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,IAAI,KAAiC,CAAC;IACtC,IAAI,SAAS,GAAwB,SAAS,CAAC;IAC/C,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,WAAW,GAAmC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IACxE,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC;YACzB,GAAG,QAAQ;YACX,KAAK;YACL,IAAI,EAAE,gBAAgB;SACvB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QACzD,SAAS,GAAG,cAAc,EAAE,MAAM,IAAI,OAAO,CAAC;QAC9C,YAAY;YACV,cAAc,EAAE,YAAY,KAAK,SAAS;gBACxC,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC,CAAC,cAAc,CAAC,YAAY,CAAC;QAClC,MAAM,aAAa,GAAG,cAAc,EAAE,QAAQ,IAAI,IAAI,CAAC;QACvD,WAAW;YACT,WAAW,IAAI,aAAa;gBAC1B,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,EAAE;gBACtD,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;QAE1C,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,OAAO,CAAC;gBACpB,YAAY,KAAK,6CAA6C,CAAC;YACjE,CAAC;YACD,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,YAAY,EAAE,CAAC;gBAC9C,aAAa,IAAI,CAAC,CAAC;gBACnB,WAAW,IAAI,CAAC,CAAC;gBACjB,IAAI,OAAO,GAAG,iCAAiC,EAAE,CAAC;oBAChD,mBAAmB,CAAC,GAAG,CAAC,OAAO,EAAE;wBAC/B,IAAI,EAAE,OAAO,CAAC,QAAQ;wBACtB,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,GAAG,QAAQ,CAAC;wBAC1D,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;wBAClD,MAAM,EAAE,OAAO;wBACf,WAAW,EAAE,aAAa;qBAC3B,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,kBAAkB,GAAG,yCAAyC,CAAC;gBACrE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;oBACrB,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAC3C,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE;wBAC7B,MAAM,EAAE,OAAO;wBACf,YAAY,EAAE,kBAAkB;wBAChC,UAAU,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,QAAQ,EAAE;qBAC9C,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,SAAS,GAAG;wBAClB,MAAM,EAAE,OAAO;wBACf,YAAY,EAAE,kBAAkB;qBACjC,CAAC;gBACJ,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC;oBACT,EAAE,EAAE,OAAO,CAAC,MAAM;oBAClB,KAAK;oBACL,QAAQ;oBACR,MAAM;oBACN,YAAY;oBACZ,QAAQ,EAAE,WAAW;oBACrB,IAAI,EAAE,OAAO,CAAC,QAAQ;oBACtB,WAAW,EAAE,CAAC;oBACd,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;oBAClB,gBAAgB,EAAE,CAAC;oBACnB,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;oBACjD,MAAM,EAAE,OAAO;oBACf,YAAY,EAAE,kBAAkB;oBAChC,QAAQ,EAAE,IAAI;oBACd,SAAS,EAAE,MAAM;iBAClB,CAAC,CAAC;YACL,CAAC;YACD,YAAY,CAAC,KAAK,EAAE,CAAC;YACrB,kBAAkB,CAAC,KAAK,EAAE,CAAC;YAC3B,mBAAmB,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;QAED,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,aAAa,GAAG,aAAa,CAC3B,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,eAAe,EACrB,KAAK,CAAC,gBAAgB,CACvB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;YACnC,YAAY,GAAG,CAAC,CAAC;YACjB,MAAM,eAAe,GAAG,KAAK,IAAI;gBAC/B,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,CAAC;gBACf,eAAe,EAAE,CAAC;gBAClB,gBAAgB,EAAE,CAAC;gBACnB,KAAK,EAAE,QAAQ,CAAC,KAAK;aACtB,CAAC;YACF,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAc;gBACzB,EAAE,EAAE,SAAS;gBACb,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,YAAY;gBACZ,QAAQ,EAAE,UAAU;gBACpB,IAAI,EAAE,eAAe,CAAC,KAAK;gBAC3B,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,YAAY,EAAE,eAAe,CAAC,YAAY;gBAC1C,eAAe,EAAE,eAAe,CAAC,eAAe;gBAChD,gBAAgB,EAAE,eAAe,CAAC,gBAAgB;gBAClD,aAAa;gBACb,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,QAAQ;aACpB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,8BAA8B,CAAC;gBAC7B,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,YAAY;gBACZ,SAAS;gBACT,UAAU,EACR,OAAO,QAAQ,CAAC,MAAM,EAAE,IAAI,KAAK,QAAQ;oBACvC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI;oBACtB,CAAC,CAAC,SAAS;gBACf,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,YAAY,EAAE,eAAe,CAAC,YAAY;gBAC1C,eAAe,EAAE,eAAe,CAAC,eAAe;gBAChD,gBAAgB,EAAE,eAAe,CAAC,gBAAgB;gBAClD,aAAa;gBACb,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,SAAS,EAAE,aAAa;gBACxB,eAAe;gBACf,WAAW;gBACX,KAAK,EAAE,CAAC,GAAG,mBAAmB,CAAC,OAAO,EAAE,CAAC;qBACtC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;qBACzB,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC;gBAC9B,cAAc,EACZ,qBAAqB,GAAG,iCAAiC;gBAC3D,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,SAAS,EAAE,QAAQ;gBACnB,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;gBACjD,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;aAChD,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAc;YAC5B,EAAE,EAAE,YAAY;YAChB,KAAK;YACL,QAAQ;YACR,MAAM;YACN,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,WAAW;YACrB,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACpC,YAAY,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YACtC,eAAe,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;YAC5C,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,IAAI,CAAC;YAC9C,aAAa;YACb,UAAU,EAAE,eAAe;YAC3B,MAAM,EAAE,SAAS;YACjB,YAAY;YACZ,QAAQ,EAAE,WAAW;YACrB,SAAS,EAAE,QAAQ;SACpB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAiB;YAC5B,KAAK;YACL,QAAQ;YACR,MAAM;YACN,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,aAAa;YACxB,eAAe;YACf,WAAW;YACX,eAAe;YACf,kBAAkB,EAAE,aAAa;YACjC,gBAAgB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACzC,iBAAiB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YAC3C,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;YACrC,SAAS,EAAE,QAAQ;SACpB,CAAC;QAEF,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9D,uEAAuE;QACvE,uEAAuE;QACvE,6EAA6E;QAC7E,0EAA0E;QAC1E,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE;oBACjD,MAAM,EAAE,SAAS;oBACjB,YAAY;oBACZ,UAAU,EAAE;wBACV,WAAW,EAAE,KAAK,CAAC,KAAK;wBACxB,kBAAkB,EAAE,KAAK,CAAC,WAAW;wBACrC,mBAAmB,EAAE,KAAK,CAAC,YAAY;wBACvC,uBAAuB,EAAE,KAAK,CAAC,eAAe;wBAC9C,wBAAwB,EAAE,KAAK,CAAC,gBAAgB;wBAChD,qBAAqB,EAAE,aAAa;qBACrC;iBACF,CAAC,CAAC;YACL,CAAC;YACD,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACzC,YAAY,CAAC,QAAQ,EAAE;oBACrB,MAAM,EAAE,OAAO;oBACf,YAAY,EAAE,mCAAmC;iBAClD,CAAC,CAAC;YACL,CAAC;YACD,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC1B,YAAY,CAAC,MAAM,kBAAkB,EAAE;gBACrC,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,UAAU,EAAE;oBACV,kBAAkB,EAAE,aAAa;oBACjC,wBAAwB,EAAE,eAAe;oBACzC,oBAAoB,EAAE,WAAW;oBACjC,mBAAmB,EAAE,eAAe;oBACpC,oBAAoB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC7C,qBAAqB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;oBAC/C,uBAAuB,EAAE,aAAa;iBACvC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,8EAA8E;IAC9E,wEAAwE;IACxE,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,MAAM,wBAAwB,CAAC;YACzD,IAAI,iBAAiB,EAAE,CAAC;gBACtB,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;gBAClE,MAAM,sBAAsB,CAAC;oBAC3B,eAAe,EAAE,MAAM,CAAC,sBAAsB;oBAC9C,sBAAsB,EAAE,iBAAiB,CAAC,KAAK;oBAC/C,IAAI,EAAE,IAAI,CAAC,cAAc;oBACzB,cAAc,EAAE,iBAAiB,CAAC,KAAK;oBACvC,0BAA0B,EAAE,KAAK;oBACjC,QAAQ;oBACR,MAAM;oBACN,UAAU,EAAE,MAAM,CAAC,2BAA2B;iBAC/C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;IACH,CAAC;IAED,OAAO,KAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,KAAkB,EAClB,OAAqB,EACrB,KAAa,EACb,MAA2B;IAE3B,MAAM,EAAE,eAAe,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3E,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAElD,qDAAqD;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,WAAW,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC","sourcesContent":["import type { AgentLoopUsage } from \"../agent/production-agent.js\";\nimport type { AgentChatEvent, AgentToolInput } from \"../agent/types.js\";\nimport { type AgentSpan, endAgentSpan, startAgentSpan } from \"./tracing.js\";\nimport { trackingIdentityProperties } from \"./tracking-identity.js\";\nimport type { TraceSpan, TraceSummary, ObservabilityConfig } from \"./types.js\";\nimport { DEFAULT_OBSERVABILITY_CONFIG } from \"./types.js\";\n\nfunction spanId(): string {\n return `span-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;\n}\n\nfunction llmProviderFromEngine(\n engineName: string | undefined,\n model: string,\n): string {\n const engine = engineName?.trim();\n if (engine?.startsWith(\"ai-sdk:\")) return engine.slice(\"ai-sdk:\".length);\n if (engine) return engine;\n if (/claude|anthropic/i.test(model)) return \"anthropic\";\n if (/gpt|openai|codex/i.test(model)) return \"openai\";\n if (/gemini|google/i.test(model)) return \"google\";\n return \"unknown\";\n}\n\nfunction costUsdFromCenticents(value: number): number {\n return Math.round((value / 10_000) * 1_000_000) / 1_000_000;\n}\n\nconst MAX_TRACKED_GENERATION_TOOL_CALLS = 50;\n\ntype GenerationToolCall = {\n name: string;\n started_offset_ms: number;\n duration_ms: number;\n status: \"success\" | \"error\";\n error_class: \"tool_error\" | \"legacy_inferred_error\" | \"interrupted\" | null;\n};\n\nfunction emitLlmGenerationTrackingEvent(args: {\n runId: string;\n threadId: string | null;\n userId: string | null;\n parentSpanId: string;\n llmSpanId: string;\n engineName: string | undefined;\n model: string;\n inputTokens: number;\n outputTokens: number;\n cacheReadTokens: number;\n cacheWriteTokens: number;\n costCentsX100: number;\n durationMs: number;\n status: \"success\" | \"error\";\n errorMessage: string | null;\n toolCalls: number;\n successfulTools: number;\n failedTools: number;\n tools: GenerationToolCall[];\n toolsTruncated: boolean;\n delegation?: {\n protocol: \"a2a\" | \"mcp\";\n callerApp?: string;\n taskId?: string;\n parentRunId?: string;\n parentTurnId?: string;\n };\n createdAt: number;\n experimentAssignments?: Array<{\n experimentId: string;\n variantId: string;\n }>;\n modelSelectionSource?: string;\n}): void {\n const provider = llmProviderFromEngine(args.engineName, args.model);\n const costUsd = costUsdFromCenticents(args.costCentsX100);\n const error = args.errorMessage ?? undefined;\n const properties: Record<string, unknown> = {\n ...trackingIdentityProperties(),\n source: \"agent_observability\",\n span_type: \"llm_call\",\n run_id: args.runId,\n thread_id: args.threadId,\n parent_span_id: args.parentSpanId,\n span_id: args.llmSpanId,\n model: args.model,\n provider,\n input_tokens: args.inputTokens,\n output_tokens: args.outputTokens,\n total_tokens: args.inputTokens + args.outputTokens,\n cache_read_tokens: args.cacheReadTokens,\n cache_write_tokens: args.cacheWriteTokens,\n cost_cents_x100: args.costCentsX100,\n cost_usd: costUsd,\n duration_ms: args.durationMs,\n status: args.status,\n tool_calls: args.toolCalls,\n successful_tools: args.successfulTools,\n failed_tools: args.failedTools,\n tools: args.tools,\n tools_truncated: args.toolsTruncated,\n delegated: args.delegation ? true : undefined,\n delegation_protocol: args.delegation?.protocol,\n caller_app: args.delegation?.callerApp,\n a2a_task_id: args.delegation?.taskId,\n parent_run_id: args.delegation?.parentRunId,\n parent_turn_id: args.delegation?.parentTurnId,\n model_selection_source: args.modelSelectionSource,\n created_at: new Date(args.createdAt).toISOString(),\n created_at_ms: args.createdAt,\n $ai_trace_id: args.runId,\n $ai_session_id: args.threadId ?? undefined,\n $ai_span_id: args.llmSpanId,\n $ai_span_name: \"agent_run\",\n $ai_parent_id: args.parentSpanId,\n $ai_model: args.model,\n $ai_provider: provider,\n $ai_input_tokens: args.inputTokens,\n $ai_output_tokens: args.outputTokens,\n $ai_latency: Math.round((args.durationMs / 1000) * 1000) / 1000,\n $ai_is_error: args.status === \"error\",\n $ai_error: error,\n $ai_cache_read_input_tokens: args.cacheReadTokens,\n $ai_cache_creation_input_tokens: args.cacheWriteTokens,\n $ai_request_count: 1,\n $ai_total_cost_usd: costUsd,\n };\n if (args.experimentAssignments?.length) {\n properties.experiment_ids = args.experimentAssignments\n .map((assignment) => assignment.experimentId)\n .join(\",\");\n properties.experiment_variants = args.experimentAssignments\n .map((assignment) => assignment.variantId)\n .join(\",\");\n if (args.experimentAssignments.length === 1) {\n properties.experiment_id = args.experimentAssignments[0].experimentId;\n properties.experiment_variant = args.experimentAssignments[0].variantId;\n }\n }\n if (error) properties.error_message = error;\n\n for (const key of Object.keys(properties)) {\n if (properties[key] === undefined) delete properties[key];\n }\n\n try {\n void import(\"../tracking/registry.js\")\n .then(({ track }) => {\n track(\"$ai_generation\", properties, {\n userId: args.userId ?? undefined,\n });\n })\n .catch(() => {});\n } catch {\n // Tracking must never affect the agent run or trace persistence.\n }\n}\n\n/** Keys whose values are stripped from persisted tool inputs when\n * `captureToolArgs` is enabled. Matched case-insensitively and tolerant\n * of `_` / `-` separators. M14 in the MCP/A2A audit: tool calls\n * routinely receive credentials verbatim (db-exec INSERTs, fetchTool\n * Authorization headers, ad-hoc bearer tokens) — keeping those values\n * out of agent_trace_spans.metadata avoids long-term storage of\n * short-lived secrets. */\nconst SENSITIVE_FIELD_PATTERN =\n /^(authorization|cookie|api[_-]?key|password|secret|token|access[_-]?token|refresh[_-]?token|bearer)$/i;\n\n/** Recursively walk a structured value and replace sensitive field\n * values with the literal string \"[REDACTED]\". Pure (returns a copy);\n * the original input is never mutated. Cycles are tolerated via a\n * small WeakSet seen-tracker that returns \"[Circular]\" for repeats. */\nexport function redactSensitiveFields(value: unknown): unknown {\n return redactWalk(value, new WeakSet<object>());\n}\n\nfunction redactWalk(value: unknown, seen: WeakSet<object>): unknown {\n if (value === null || typeof value !== \"object\") return value;\n if (seen.has(value as object)) return \"[Circular]\";\n seen.add(value as object);\n if (Array.isArray(value)) {\n return value.map((v) => redactWalk(v, seen));\n }\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n if (SENSITIVE_FIELD_PATTERN.test(k)) {\n out[k] = \"[REDACTED]\";\n } else {\n out[k] = redactWalk(v, seen);\n }\n }\n return out;\n}\n\nexport async function getObservabilityConfig(): Promise<ObservabilityConfig> {\n let stored: Partial<ObservabilityConfig> | null = null;\n try {\n const { getSetting } = await import(\"../settings/store.js\");\n stored = (await getSetting(\n \"observability-config\",\n )) as Partial<ObservabilityConfig> | null;\n } catch {}\n const { resolveInferredSentimentConfig } = await import(\"./sentiment.js\");\n return {\n ...DEFAULT_OBSERVABILITY_CONFIG,\n ...(stored ?? {}),\n ...resolveInferredSentimentConfig(stored),\n };\n}\n\nexport async function instrumentAgentLoop(opts: {\n runAgentLoop: (loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n runId?: string;\n }) => Promise<AgentLoopUsage>;\n loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n runId?: string;\n };\n runId: string;\n threadId: string | null;\n /** Owner of this run; persisted on every span + summary so dashboard\n * reads can filter to a single user. Null for unauthenticated callers\n * (background tasks, etc.) — those rows aren't returned by per-user\n * reads. */\n userId: string | null;\n config: ObservabilityConfig;\n metadata?: Record<string, unknown> | null;\n experimentAssignments?: Array<{\n experimentId: string;\n variantId: string;\n }>;\n modelSelectionSource?: string;\n delegation?: {\n protocol: \"a2a\" | \"mcp\";\n callerApp?: string;\n taskId?: string;\n parentRunId?: string;\n parentTurnId?: string;\n };\n /** Raw user-authored message before prompt/context enrichment. */\n sentimentInput?: string;\n classifyError?: (error: unknown) =>\n | {\n status?: \"success\" | \"error\";\n errorMessage?: string | null;\n metadata?: Record<string, unknown> | null;\n }\n | null\n | undefined;\n}): Promise<AgentLoopUsage> {\n const { runAgentLoop, loopOpts, runId, threadId, userId, config } = opts;\n const runStart = Date.now();\n const parentSpanId = spanId();\n const precedingResponsePromise =\n config.inferredSentimentEnabled && opts.sentimentInput && threadId && userId\n ? import(\"./store.js\")\n .then(({ getLatestTraceSummaryForThread }) =>\n getLatestTraceSummaryForThread(threadId, {\n userId,\n excludeRunId: runId,\n }),\n )\n .catch(() => null)\n : Promise.resolve(null);\n\n // Optional OpenTelemetry root span for this run. No-ops unless a host has\n // installed `@opentelemetry/api` and registered a provider. The promise is\n // resolved before the loop runs so child tool/model spans can parent under\n // it conceptually (we keep them flat in the same tracer, which is enough\n // for the dashboards an embedding app would build).\n const otelRunSpanPromise = startAgentSpan(\"agent.run\", {\n \"agent.run_id\": runId,\n \"agent.thread_id\": threadId ?? undefined,\n \"agent.user_id\": userId ?? undefined,\n \"agent.model\": loopOpts.model,\n \"agent.model_selection_source\": opts.modelSelectionSource,\n \"agent.experiment_id\":\n opts.experimentAssignments?.length === 1\n ? opts.experimentAssignments[0].experimentId\n : undefined,\n \"agent.experiment_variant\":\n opts.experimentAssignments?.length === 1\n ? opts.experimentAssignments[0].variantId\n : undefined,\n });\n\n const spans: TraceSpan[] = [];\n let toolInvocationCounter = 0;\n // Keyed by counter to handle concurrent calls to the same tool name\n const pendingTools = new Map<\n number,\n {\n spanId: string;\n callId?: string;\n startMs: number;\n toolName: string;\n input: AgentToolInput;\n otelSpan: AgentSpan | null;\n endResult?: { status: \"success\" | \"error\"; errorMessage: string | null };\n }\n >();\n // Secondary index for legacy emitters without call ids. Current tool events\n // are paired by id first; same-name FIFO remains as a compatibility fallback.\n const toolNameToCounters = new Map<string, number[]>();\n const toolCallIdToCounter = new Map<string, number>();\n const generationToolCalls = new Map<number, GenerationToolCall>();\n\n let toolCallCount = 0;\n let successfulTools = 0;\n let failedTools = 0;\n\n // Track in-flight OTel tool spans so they're all ended even if the loop\n // throws before a matching `tool_done` arrives.\n const openOtelToolSpans = new Set<AgentSpan>();\n\n const instrumentedSend = (event: AgentChatEvent): void => {\n try {\n if (event.type === \"tool_start\") {\n const counter = toolInvocationCounter++;\n const sid = spanId();\n // Start the OTel tool span synchronously-ish: kick off the async\n // resolution and stash the span once it lands. Tool spans are short\n // and the api tracer is synchronous in practice, but we tolerate the\n // microtask gap by recording the span on the pending entry when ready.\n const entry: {\n spanId: string;\n callId?: string;\n startMs: number;\n toolName: string;\n input: AgentToolInput;\n otelSpan: AgentSpan | null;\n // Set by the done handler if it fires before the span promise\n // resolves, so the resolved span is ended with the correct status.\n endResult?: {\n status: \"success\" | \"error\";\n errorMessage: string | null;\n };\n } = {\n spanId: sid,\n ...(event.id ? { callId: event.id } : {}),\n startMs: Date.now(),\n toolName: event.tool,\n input: event.input,\n otelSpan: null,\n };\n pendingTools.set(counter, entry);\n if (event.id) toolCallIdToCounter.set(event.id, counter);\n void startAgentSpan(\"tool.call\", {\n \"tool.name\": event.tool,\n }).then((span) => {\n if (!span) return;\n // If `tool_done` already ran for this call, end the span now with the\n // status it recorded; otherwise stash it for the done handler.\n if (entry.endResult) {\n endAgentSpan(span, {\n status: entry.endResult.status,\n errorMessage: entry.endResult.errorMessage,\n });\n } else {\n entry.otelSpan = span;\n openOtelToolSpans.add(span);\n }\n });\n const queue = toolNameToCounters.get(event.tool);\n if (queue) queue.push(counter);\n else toolNameToCounters.set(event.tool, [counter]);\n } else if (event.type === \"tool_done\") {\n const queue = toolNameToCounters.get(event.tool);\n const counterFromId = event.id\n ? toolCallIdToCounter.get(event.id)\n : undefined;\n const legacyQueueIndex =\n event.id && counterFromId === undefined && queue\n ? queue.findIndex(\n (candidate) => !pendingTools.get(candidate)?.callId,\n )\n : -1;\n const counter =\n counterFromId ??\n (event.id\n ? legacyQueueIndex >= 0\n ? queue?.[legacyQueueIndex]\n : undefined\n : queue?.shift());\n const pending =\n counter !== undefined ? pendingTools.get(counter) : undefined;\n if (counter !== undefined) {\n pendingTools.delete(counter);\n if (pending?.callId) toolCallIdToCounter.delete(pending.callId);\n if ((counterFromId !== undefined || legacyQueueIndex >= 0) && queue) {\n const queueIndex = queue.indexOf(counter);\n if (queueIndex >= 0) queue.splice(queueIndex, 1);\n }\n if (queue && queue.length === 0)\n toolNameToCounters.delete(event.tool);\n }\n toolCallCount++;\n\n const finishedAt = Date.now();\n\n const explicitError = event.isError === true;\n const isError =\n typeof event.isError === \"boolean\"\n ? event.isError\n : typeof event.result === \"string\" &&\n (event.result.startsWith(\"Error\") ||\n event.result.startsWith(\"Error running \"));\n if (isError) failedTools++;\n else successfulTools++;\n\n if (\n counter !== undefined &&\n counter < MAX_TRACKED_GENERATION_TOOL_CALLS &&\n pending\n ) {\n generationToolCalls.set(counter, {\n name: pending.toolName,\n started_offset_ms: Math.max(0, pending.startMs - runStart),\n duration_ms: Math.max(0, finishedAt - pending.startMs),\n status: isError ? \"error\" : \"success\",\n error_class: !isError\n ? null\n : explicitError\n ? \"tool_error\"\n : \"legacy_inferred_error\",\n });\n }\n\n // Finalize the OTel tool span. If the span promise hasn't resolved yet\n // we record the result on the entry so its `.then` handler ends it.\n const otelEndResult = {\n status: (isError ? \"error\" : \"success\") as \"success\" | \"error\",\n errorMessage: isError ? (event.result as string) : null,\n };\n if (pending?.otelSpan) {\n openOtelToolSpans.delete(pending.otelSpan);\n endAgentSpan(pending.otelSpan, {\n status: otelEndResult.status,\n errorMessage: otelEndResult.errorMessage,\n attributes: { \"tool.name\": event.tool },\n });\n } else if (pending) {\n pending.endResult = otelEndResult;\n }\n\n const span: TraceSpan = {\n id: pending?.spanId ?? spanId(),\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"tool_call\",\n name: event.tool,\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n costCentsX100: 0,\n durationMs: pending ? Math.max(0, finishedAt - pending.startMs) : 0,\n status: isError ? \"error\" : \"success\",\n errorMessage: isError ? event.result : null,\n metadata:\n config.captureToolArgs && pending\n ? // Strip Authorization/api-key/token-shaped values before\n // persisting (M14 in the MCP/A2A audit). Tool-runtime\n // execution still sees the unredacted input — only the\n // long-lived span row is sanitized.\n {\n input: redactSensitiveFields(pending.input) as Record<\n string,\n string\n >,\n }\n : null,\n createdAt: Date.now(),\n };\n spans.push(span);\n }\n } catch {}\n\n loopOpts.send(event);\n };\n\n let usage: AgentLoopUsage | undefined;\n let runStatus: \"success\" | \"error\" = \"success\";\n let errorMessage: string | null = null;\n let runMetadata: Record<string, unknown> | null = opts.metadata ?? null;\n try {\n usage = await runAgentLoop({\n ...loopOpts,\n runId,\n send: instrumentedSend,\n });\n } catch (err: any) {\n const classification = opts.classifyError?.(err) ?? null;\n runStatus = classification?.status ?? \"error\";\n errorMessage =\n classification?.errorMessage === undefined\n ? (err?.message ?? String(err))\n : classification.errorMessage;\n const errorMetadata = classification?.metadata ?? null;\n runMetadata =\n runMetadata || errorMetadata\n ? { ...(runMetadata ?? {}), ...(errorMetadata ?? {}) }\n : null;\n throw err;\n } finally {\n const runEnd = Date.now();\n const totalDurationMs = runEnd - runStart;\n\n if (pendingTools.size > 0) {\n if (runStatus === \"success\") {\n runStatus = \"error\";\n errorMessage ??= \"Agent run ended with interrupted tool calls\";\n }\n for (const [counter, pending] of pendingTools) {\n toolCallCount += 1;\n failedTools += 1;\n if (counter < MAX_TRACKED_GENERATION_TOOL_CALLS) {\n generationToolCalls.set(counter, {\n name: pending.toolName,\n started_offset_ms: Math.max(0, pending.startMs - runStart),\n duration_ms: Math.max(0, runEnd - pending.startMs),\n status: \"error\",\n error_class: \"interrupted\",\n });\n }\n const interruptedMessage = \"Tool call interrupted before completion\";\n if (pending.otelSpan) {\n openOtelToolSpans.delete(pending.otelSpan);\n endAgentSpan(pending.otelSpan, {\n status: \"error\",\n errorMessage: interruptedMessage,\n attributes: { \"tool.name\": pending.toolName },\n });\n } else {\n pending.endResult = {\n status: \"error\",\n errorMessage: interruptedMessage,\n };\n }\n spans.push({\n id: pending.spanId,\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"tool_call\",\n name: pending.toolName,\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n costCentsX100: 0,\n durationMs: Math.max(0, runEnd - pending.startMs),\n status: \"error\",\n errorMessage: interruptedMessage,\n metadata: null,\n createdAt: runEnd,\n });\n }\n pendingTools.clear();\n toolNameToCounters.clear();\n toolCallIdToCounter.clear();\n }\n\n let costCentsX100 = 0;\n try {\n const { calculateCost } = await import(\"../usage/store.js\");\n if (usage) {\n costCentsX100 = calculateCost(\n usage.inputTokens,\n usage.outputTokens,\n usage.model,\n usage.cacheReadTokens,\n usage.cacheWriteTokens,\n );\n }\n } catch {}\n\n let llmCallCount = 0;\n if (usage || runStatus === \"error\") {\n llmCallCount = 1;\n const generationUsage = usage ?? {\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n model: loopOpts.model,\n };\n const llmSpanId = spanId();\n const llmSpan: TraceSpan = {\n id: llmSpanId,\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"llm_call\",\n name: generationUsage.model,\n inputTokens: generationUsage.inputTokens,\n outputTokens: generationUsage.outputTokens,\n cacheReadTokens: generationUsage.cacheReadTokens,\n cacheWriteTokens: generationUsage.cacheWriteTokens,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: null,\n createdAt: runStart,\n };\n spans.push(llmSpan);\n emitLlmGenerationTrackingEvent({\n runId,\n threadId,\n userId,\n parentSpanId,\n llmSpanId,\n engineName:\n typeof loopOpts.engine?.name === \"string\"\n ? loopOpts.engine.name\n : undefined,\n model: generationUsage.model,\n inputTokens: generationUsage.inputTokens,\n outputTokens: generationUsage.outputTokens,\n cacheReadTokens: generationUsage.cacheReadTokens,\n cacheWriteTokens: generationUsage.cacheWriteTokens,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n toolCalls: toolCallCount,\n successfulTools,\n failedTools,\n tools: [...generationToolCalls.entries()]\n .sort(([a], [b]) => a - b)\n .map(([, detail]) => detail),\n toolsTruncated:\n toolInvocationCounter > MAX_TRACKED_GENERATION_TOOL_CALLS,\n delegation: opts.delegation,\n createdAt: runStart,\n experimentAssignments: opts.experimentAssignments,\n modelSelectionSource: opts.modelSelectionSource,\n });\n }\n\n const parentSpan: TraceSpan = {\n id: parentSpanId,\n runId,\n threadId,\n userId,\n parentSpanId: null,\n spanType: \"agent_run\",\n name: \"agent_run\",\n inputTokens: usage?.inputTokens ?? 0,\n outputTokens: usage?.outputTokens ?? 0,\n cacheReadTokens: usage?.cacheReadTokens ?? 0,\n cacheWriteTokens: usage?.cacheWriteTokens ?? 0,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: runMetadata,\n createdAt: runStart,\n };\n spans.push(parentSpan);\n\n const summary: TraceSummary = {\n runId,\n threadId,\n userId,\n totalSpans: spans.length,\n llmCalls: llmCallCount,\n toolCalls: toolCallCount,\n successfulTools,\n failedTools,\n totalDurationMs,\n totalCostCentsX100: costCentsX100,\n totalInputTokens: usage?.inputTokens ?? 0,\n totalOutputTokens: usage?.outputTokens ?? 0,\n model: usage?.model ?? loopOpts.model,\n createdAt: runStart,\n };\n\n writeTraceData(spans, summary, runId, config).catch(() => {});\n\n // OpenTelemetry export (no-op unless a provider is registered). Emit a\n // self-contained `llm.call` span carrying model + token usage, end any\n // tool spans still open (loop threw mid-tool), and end the run span. Awaited\n // so the spans are emitted before the function returns; cheap when no-op.\n try {\n if (usage) {\n endAgentSpan(await startAgentSpan(\"llm.call\", {}), {\n status: runStatus,\n errorMessage,\n attributes: {\n \"llm.model\": usage.model,\n \"llm.input_tokens\": usage.inputTokens,\n \"llm.output_tokens\": usage.outputTokens,\n \"llm.cache_read_tokens\": usage.cacheReadTokens,\n \"llm.cache_write_tokens\": usage.cacheWriteTokens,\n \"llm.cost_cents_x100\": costCentsX100,\n },\n });\n }\n for (const toolSpan of openOtelToolSpans) {\n endAgentSpan(toolSpan, {\n status: \"error\",\n errorMessage: \"Agent run ended before tool_done.\",\n });\n }\n openOtelToolSpans.clear();\n endAgentSpan(await otelRunSpanPromise, {\n status: runStatus,\n errorMessage,\n attributes: {\n \"agent.tool_calls\": toolCallCount,\n \"agent.successful_tools\": successfulTools,\n \"agent.failed_tools\": failedTools,\n \"agent.duration_ms\": totalDurationMs,\n \"agent.input_tokens\": usage?.inputTokens ?? 0,\n \"agent.output_tokens\": usage?.outputTokens ?? 0,\n \"agent.cost_cents_x100\": costCentsX100,\n },\n });\n } catch {\n // OTel export must never break the run.\n }\n }\n\n // Classify only after the main loop has finished so the tiny managed Luna\n // request cannot contend with the user's response for a gateway slot. This\n // short, awaited tail keeps serverless runtimes alive long enough to emit the\n // event, while the response content has already streamed to the client.\n if (usage && opts.sentimentInput) {\n try {\n const precedingResponse = await precedingResponsePromise;\n if (precedingResponse) {\n const { inferAndTrackSentiment } = await import(\"./sentiment.js\");\n await inferAndTrackSentiment({\n classifierModel: config.inferredSentimentModel,\n precedingResponseModel: precedingResponse.model,\n text: opts.sentimentInput,\n precedingRunId: precedingResponse.runId,\n classificationTriggerRunId: runId,\n threadId,\n userId,\n sampleRate: config.inferredSentimentSampleRate,\n });\n }\n } catch {\n // Optional inference must never alter the result of the main run.\n }\n }\n\n return usage!;\n}\n\nasync function writeTraceData(\n spans: TraceSpan[],\n summary: TraceSummary,\n runId: string,\n config: ObservabilityConfig,\n): Promise<void> {\n const { insertTraceSpan, upsertTraceSummary } = await import(\"./store.js\");\n await Promise.all(spans.map((s) => insertTraceSpan(s).catch(() => {})));\n await upsertTraceSummary(summary).catch(() => {});\n\n // Fire automated evals after trace data is persisted\n try {\n const { evaluateRun } = await import(\"./evals.js\");\n await evaluateRun(runId, { sampleRate: config.evalSampleRate });\n } catch {}\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"call-agent.d.ts","sourceRoot":"","sources":["../../src/scripts/call-agent.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"call-agent.d.ts","sourceRoot":"","sources":["../../src/scripts/call-agent.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AACrE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAgHpD,eAAO,MAAM,IAAI,EAAE,UAsDlB,CAAC;AAEF,wBAAsB,GAAG,CACvB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,OAAO,CAAC,EAAE,gBAAgB,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,MAAM,CAAC,CAgXjB;AA+ND,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CASzE"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { A2ATaskTimeoutError, callAgent, shouldPreferGlobalA2ASecret, signA2AToken, } from "../a2a/client.js";
|
|
3
3
|
import { invokeAgentAction } from "../a2a/invoke.js";
|
|
4
4
|
import { formatLlmCredentialErrorMessage, isLlmCredentialError, } from "../agent/engine/credential-errors.js";
|
|
@@ -9,6 +9,27 @@ import { getRequestUserEmail, getRequestOrgId, isIntegrationCallerRequest, getIn
|
|
|
9
9
|
const DEFAULT_SERVERLESS_INTEGRATION_A2A_TIMEOUT_MS = 18_000;
|
|
10
10
|
const NETLIFY_INTEGRATION_A2A_TIMEOUT_MS = 2_000;
|
|
11
11
|
const INTEGRATION_A2A_TOKEN_TTL = "30m";
|
|
12
|
+
function buildDelegationCorrelation(context, selfAppId, invocationId) {
|
|
13
|
+
return {
|
|
14
|
+
...(selfAppId?.trim() ? { callerApp: selfAppId.trim() } : {}),
|
|
15
|
+
...(context?.threadId ? { callerThreadId: context.threadId } : {}),
|
|
16
|
+
...(context?.runId ? { parentRunId: context.runId } : {}),
|
|
17
|
+
...(context?.turnId ? { parentTurnId: context.turnId } : {}),
|
|
18
|
+
...(invocationId ? { invocationId } : {}),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function buildMessageIdempotencyKey(originatingTurnId, target, exactMessage) {
|
|
22
|
+
if (!originatingTurnId)
|
|
23
|
+
return undefined;
|
|
24
|
+
const digest = createHash("sha256")
|
|
25
|
+
.update(JSON.stringify({
|
|
26
|
+
originatingTurnId,
|
|
27
|
+
target,
|
|
28
|
+
message: exactMessage,
|
|
29
|
+
}))
|
|
30
|
+
.digest("hex");
|
|
31
|
+
return `v1:${digest}`;
|
|
32
|
+
}
|
|
12
33
|
function parseTimeoutMs(value) {
|
|
13
34
|
if (!value)
|
|
14
35
|
return undefined;
|
|
@@ -142,12 +163,16 @@ export async function run(args, context, selfAppId) {
|
|
|
142
163
|
.join(", ");
|
|
143
164
|
return `Error: Agent "${agentIdOrName}" not found. Available agents: ${available || "(none)"}`;
|
|
144
165
|
}
|
|
166
|
+
const correlation = buildDelegationCorrelation(context, selfAppId);
|
|
167
|
+
const idempotencyKey = message && !taskId
|
|
168
|
+
? buildMessageIdempotencyKey(context?.turnId, agent.url, message)
|
|
169
|
+
: undefined;
|
|
145
170
|
if (action) {
|
|
146
171
|
if (context?.send) {
|
|
147
172
|
context.send({ type: "agent_call", agent: agent.name, status: "start" });
|
|
148
173
|
}
|
|
149
174
|
try {
|
|
150
|
-
const output = await invokeReadOnlyAppAction(agent, action, input);
|
|
175
|
+
const output = await invokeReadOnlyAppAction(agent, action, input, buildDelegationCorrelation(context, selfAppId, randomUUID()));
|
|
151
176
|
return output;
|
|
152
177
|
}
|
|
153
178
|
finally {
|
|
@@ -330,6 +355,9 @@ export async function run(args, context, selfAppId) {
|
|
|
330
355
|
orgDomain: callerOrgDomain,
|
|
331
356
|
orgSecret: callerOrgSecret,
|
|
332
357
|
approvedActions,
|
|
358
|
+
contextId: context.threadId,
|
|
359
|
+
correlation,
|
|
360
|
+
idempotencyKey,
|
|
333
361
|
...(taskId ? { taskId } : {}),
|
|
334
362
|
onUpdate: onRemotePollUpdate,
|
|
335
363
|
returnRecoverableArtifactsOnTimeout: false,
|
|
@@ -412,6 +440,9 @@ export async function run(args, context, selfAppId) {
|
|
|
412
440
|
orgDomain: domain,
|
|
413
441
|
orgSecret,
|
|
414
442
|
approvedActions,
|
|
443
|
+
contextId: context?.threadId,
|
|
444
|
+
correlation,
|
|
445
|
+
idempotencyKey,
|
|
415
446
|
...(taskId ? { taskId } : {}),
|
|
416
447
|
returnRecoverableArtifactsOnTimeout: false,
|
|
417
448
|
});
|
|
@@ -435,7 +466,7 @@ export async function run(args, context, selfAppId) {
|
|
|
435
466
|
return `Error calling ${agent.name}: ${msg}`;
|
|
436
467
|
}
|
|
437
468
|
}
|
|
438
|
-
async function invokeReadOnlyAppAction(agent, action, input) {
|
|
469
|
+
async function invokeReadOnlyAppAction(agent, action, input, correlation) {
|
|
439
470
|
const callerEmail = getRequestUserEmail();
|
|
440
471
|
if (!callerEmail) {
|
|
441
472
|
return `Error calling ${agent.name} action ${action}: a signed-in user identity is required`;
|
|
@@ -464,6 +495,7 @@ async function invokeReadOnlyAppAction(agent, action, input) {
|
|
|
464
495
|
userEmail: callerEmail,
|
|
465
496
|
orgDomain: callerOrgDomain,
|
|
466
497
|
orgSecret: callerOrgSecret,
|
|
498
|
+
correlation,
|
|
467
499
|
});
|
|
468
500
|
return invocation.result.status === "completed"
|
|
469
501
|
? invocation.result.output
|