@agent-native/core 0.168.13 → 0.169.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/engine/first-event-timeout.d.ts +8 -0
- package/dist/agent/engine/first-event-timeout.js +8 -0
- package/dist/agent/production-agent.d.ts +0 -30
- package/dist/agent/production-agent.js +17 -38
- package/dist/agent/run-loop-with-resume.d.ts +38 -25
- package/dist/agent/run-loop-with-resume.js +140 -55
- package/dist/agent/run-manager.d.ts +83 -68
- package/dist/agent/run-manager.js +280 -94
- package/dist/agent/run-store.d.ts +31 -0
- package/dist/agent/run-store.js +42 -12
- package/dist/app-config/agent.d.ts +2 -0
- package/dist/app-config/agent.js +33 -0
- package/dist/app-config/run-lifecycle-invariants.d.ts +248 -0
- package/dist/app-config/run-lifecycle-invariants.js +342 -0
- package/dist/app-config/schema.d.ts +2 -0
- package/dist/app-config/store.js +9 -1
- package/dist/client/agent-chat-adapter.d.ts +0 -2
- package/dist/client/agent-chat-adapter.js +7 -23
- package/dist/jobs/background-automation-runner.d.ts +25 -0
- package/dist/jobs/background-automation-runner.js +104 -21
- package/dist/jobs/run-history.d.ts +7 -1
- package/dist/jobs/run-history.js +57 -14
- package/dist/observability/traces.d.ts +13 -0
- package/dist/observability/traces.js +369 -317
- package/dist/progress/routes.d.ts +1 -1
- package/dist/server/agent-chat-plugin.js +2 -4
- package/dist/server/realtime-token.d.ts +1 -1
- package/package.json +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { captureError } from "../server/capture-error.js";
|
|
1
2
|
import { getRequestContext } from "../server/request-context.js";
|
|
2
3
|
import { MAX_AI_CONTENT_BYTES, MAX_AI_SPANS_PER_RUN, boundAiContent, emitAiSpanEvent, emitAiTraceEvent, toAiErrorDetail, } from "./posthog-ai.js";
|
|
3
4
|
import { endAgentSpan, startAgentSpan } from "./tracing.js";
|
|
@@ -23,6 +24,32 @@ function llmProviderFromEngine(engineName, model) {
|
|
|
23
24
|
function costUsdFromCenticents(value) {
|
|
24
25
|
return Math.round((value / 10_000) * 1_000_000) / 1_000_000;
|
|
25
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Project run metadata onto flat PostHog trace properties.
|
|
29
|
+
*
|
|
30
|
+
* Prefixed and shallow on purpose: this is operational context (which
|
|
31
|
+
* automation, which trigger, which terminal state), never message content, and
|
|
32
|
+
* nested objects in an analytics property are unqueryable anyway. Values are
|
|
33
|
+
* bounded so a caller cannot turn a metadata bag into a payload channel.
|
|
34
|
+
*/
|
|
35
|
+
function aiTraceMetadataProperties(metadata) {
|
|
36
|
+
if (!metadata)
|
|
37
|
+
return {};
|
|
38
|
+
const properties = {};
|
|
39
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
40
|
+
if (value === undefined || value === null)
|
|
41
|
+
continue;
|
|
42
|
+
const scalar = typeof value === "string"
|
|
43
|
+
? value.slice(0, 200)
|
|
44
|
+
: typeof value === "number" || typeof value === "boolean"
|
|
45
|
+
? value
|
|
46
|
+
: undefined;
|
|
47
|
+
if (scalar === undefined)
|
|
48
|
+
continue;
|
|
49
|
+
properties[`run_${key}`] = scalar;
|
|
50
|
+
}
|
|
51
|
+
return properties;
|
|
52
|
+
}
|
|
26
53
|
const MAX_TRACKED_GENERATION_TOOL_CALLS = 50;
|
|
27
54
|
/**
|
|
28
55
|
* `auto_continue` reasons the server PLANNED, which must not read as failures.
|
|
@@ -289,6 +316,7 @@ export async function getObservabilityConfig() {
|
|
|
289
316
|
}
|
|
290
317
|
export async function instrumentAgentLoop(opts) {
|
|
291
318
|
const { runAgentLoop, loopOpts, runId, threadId, userId, config } = opts;
|
|
319
|
+
const spanName = opts.spanName?.trim() || "agent_run";
|
|
292
320
|
const runStart = Date.now();
|
|
293
321
|
const parentSpanId = spanId();
|
|
294
322
|
const precedingResponsePromise = config.inferredSentimentEnabled && opts.sentimentInput && threadId && userId
|
|
@@ -591,365 +619,389 @@ export async function instrumentAgentLoop(opts) {
|
|
|
591
619
|
throw err;
|
|
592
620
|
}
|
|
593
621
|
finally {
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
if (
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
started_offset_ms: Math.max(0, pending.startMs - runStart),
|
|
609
|
-
duration_ms: Math.max(0, runEnd - pending.startMs),
|
|
610
|
-
status: "error",
|
|
611
|
-
error_class: "interrupted",
|
|
612
|
-
error_message: config.captureToolResults
|
|
613
|
-
? interruptedMessage
|
|
614
|
-
: undefined,
|
|
615
|
-
});
|
|
622
|
+
// A throw from inside a `finally` REPLACES whatever the block was doing —
|
|
623
|
+
// including a successful return — so an assembly failure here (a content
|
|
624
|
+
// builder tripping on an odd payload, a span mapper on a malformed tool
|
|
625
|
+
// result) would report a completed run as a failed one, and a failed run
|
|
626
|
+
// with the wrong error. Every emit below already guards itself; this guards
|
|
627
|
+
// the assembly between them, so the module's contract holds without each
|
|
628
|
+
// future line having to remember it.
|
|
629
|
+
try {
|
|
630
|
+
const runEnd = Date.now();
|
|
631
|
+
const totalDurationMs = runEnd - runStart;
|
|
632
|
+
if (pendingTools.size > 0) {
|
|
633
|
+
if (runStatus === "success") {
|
|
634
|
+
runStatus = "error";
|
|
635
|
+
errorMessage ??= "Agent run ended with interrupted tool calls";
|
|
616
636
|
}
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
637
|
+
for (const [counter, pending] of pendingTools) {
|
|
638
|
+
toolCallCount += 1;
|
|
639
|
+
failedTools += 1;
|
|
640
|
+
const interruptedMessage = "Tool call interrupted before completion";
|
|
641
|
+
if (counter < MAX_TRACKED_GENERATION_TOOL_CALLS) {
|
|
642
|
+
generationToolCalls.set(counter, {
|
|
643
|
+
name: pending.toolName,
|
|
644
|
+
started_offset_ms: Math.max(0, pending.startMs - runStart),
|
|
645
|
+
duration_ms: Math.max(0, runEnd - pending.startMs),
|
|
646
|
+
status: "error",
|
|
647
|
+
error_class: "interrupted",
|
|
648
|
+
error_message: config.captureToolResults
|
|
649
|
+
? interruptedMessage
|
|
650
|
+
: undefined,
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
if (pending.otelSpan) {
|
|
654
|
+
openOtelToolSpans.delete(pending.otelSpan);
|
|
655
|
+
endAgentSpan(pending.otelSpan, {
|
|
656
|
+
status: "error",
|
|
657
|
+
errorMessage: interruptedMessage,
|
|
658
|
+
attributes: { "tool.name": pending.toolName },
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
else {
|
|
662
|
+
pending.endResult = {
|
|
663
|
+
status: "error",
|
|
664
|
+
errorMessage: interruptedMessage,
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
spans.push({
|
|
668
|
+
id: pending.spanId,
|
|
669
|
+
runId,
|
|
670
|
+
threadId,
|
|
671
|
+
userId,
|
|
672
|
+
parentSpanId,
|
|
673
|
+
spanType: "tool_call",
|
|
674
|
+
name: pending.toolName,
|
|
675
|
+
inputTokens: 0,
|
|
676
|
+
outputTokens: 0,
|
|
677
|
+
cacheReadTokens: 0,
|
|
678
|
+
cacheWriteTokens: 0,
|
|
679
|
+
costCentsX100: 0,
|
|
680
|
+
durationMs: Math.max(0, runEnd - pending.startMs),
|
|
620
681
|
status: "error",
|
|
621
682
|
errorMessage: interruptedMessage,
|
|
622
|
-
|
|
683
|
+
metadata: null,
|
|
684
|
+
createdAt: runEnd,
|
|
623
685
|
});
|
|
624
686
|
}
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
687
|
+
pendingTools.clear();
|
|
688
|
+
toolNameToCounters.clear();
|
|
689
|
+
toolCallIdToCounter.clear();
|
|
690
|
+
}
|
|
691
|
+
let costCentsX100 = 0;
|
|
692
|
+
try {
|
|
693
|
+
const { calculateCost } = await import("../usage/store.js");
|
|
694
|
+
if (usage) {
|
|
695
|
+
costCentsX100 = calculateCost(usage.inputTokens, usage.outputTokens, usage.model, usage.cacheReadTokens, usage.cacheWriteTokens);
|
|
630
696
|
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
697
|
+
// Cost estimation is enrichment: a pricing-table miss leaves the span
|
|
698
|
+
// without a cost rather than failing the trace.
|
|
699
|
+
}
|
|
700
|
+
catch { } // coercion-ok: see above
|
|
701
|
+
// A cut-off run never reaches the loop's outcome classification, so stand in
|
|
702
|
+
// for it here rather than reporting no terminal state at all. `failed` +
|
|
703
|
+
// `retryable` is the honest encoding available in `AgentLoopOutcome`: the
|
|
704
|
+
// turn did not finish, and the continuation machinery is expected to
|
|
705
|
+
// recover it. A real reported outcome always wins.
|
|
706
|
+
const effectiveTerminalOutcome = terminalOutcome ??
|
|
707
|
+
(cutOffReason && !EXPECTED_CONTINUATION_REASONS.has(cutOffReason)
|
|
708
|
+
? {
|
|
709
|
+
state: "failed",
|
|
710
|
+
code: cutOffReason,
|
|
711
|
+
retryable: true,
|
|
712
|
+
message: `Agent run was cut off before finishing (${cutOffReason}).`,
|
|
713
|
+
}
|
|
714
|
+
: undefined);
|
|
715
|
+
let llmCallCount = 0;
|
|
716
|
+
if (usage || runStatus === "error") {
|
|
717
|
+
llmCallCount =
|
|
718
|
+
usage?.llmCalls ??
|
|
719
|
+
// Compatibility for custom loop implementations that predate the
|
|
720
|
+
// attempt counter: a measured run still counts as one call.
|
|
721
|
+
1;
|
|
722
|
+
const generationUsage = usage ?? {
|
|
639
723
|
inputTokens: 0,
|
|
640
724
|
outputTokens: 0,
|
|
641
725
|
cacheReadTokens: 0,
|
|
642
726
|
cacheWriteTokens: 0,
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
727
|
+
model: loopOpts.model,
|
|
728
|
+
};
|
|
729
|
+
// The engine never reported a `usage` event for this run (killed for
|
|
730
|
+
// silence before any provider response, or the loop threw before
|
|
731
|
+
// returning). `generationUsage`'s token fields are placeholder zeros in
|
|
732
|
+
// that case, not measured values — the tracking event below must omit
|
|
733
|
+
// them rather than report a fabricated 0.
|
|
734
|
+
const usageReported = usage?.usageReported === true;
|
|
735
|
+
const firstTokenMs = usage?.firstEngineEventAtMs !== undefined
|
|
736
|
+
? Math.max(0, usage.firstEngineEventAtMs - runStart)
|
|
737
|
+
: undefined;
|
|
738
|
+
const llmSpanId = spanId();
|
|
739
|
+
const generationContent = buildGenerationContent({
|
|
740
|
+
config,
|
|
741
|
+
messages: loopOpts.messages,
|
|
742
|
+
tools: loopOpts.tools,
|
|
743
|
+
assistantText: assistantTextParts.join(""),
|
|
744
|
+
toolSpans: spans.filter((s) => s.spanType === "tool_call"),
|
|
745
|
+
});
|
|
746
|
+
const llmSpan = {
|
|
747
|
+
id: llmSpanId,
|
|
748
|
+
runId,
|
|
749
|
+
threadId,
|
|
750
|
+
userId,
|
|
751
|
+
parentSpanId,
|
|
752
|
+
spanType: "llm_call",
|
|
753
|
+
name: generationUsage.model,
|
|
754
|
+
inputTokens: generationUsage.inputTokens,
|
|
755
|
+
outputTokens: generationUsage.outputTokens,
|
|
756
|
+
cacheReadTokens: generationUsage.cacheReadTokens,
|
|
757
|
+
cacheWriteTokens: generationUsage.cacheWriteTokens,
|
|
758
|
+
costCentsX100,
|
|
759
|
+
durationMs: totalDurationMs,
|
|
760
|
+
status: runStatus,
|
|
761
|
+
errorMessage,
|
|
647
762
|
metadata: null,
|
|
648
|
-
createdAt:
|
|
763
|
+
createdAt: runStart,
|
|
764
|
+
};
|
|
765
|
+
spans.push(llmSpan);
|
|
766
|
+
emitLlmGenerationTrackingEvent({
|
|
767
|
+
runId,
|
|
768
|
+
threadId,
|
|
769
|
+
userId,
|
|
770
|
+
parentSpanId,
|
|
771
|
+
llmSpanId,
|
|
772
|
+
engineName: typeof loopOpts.engine?.name === "string"
|
|
773
|
+
? loopOpts.engine.name
|
|
774
|
+
: undefined,
|
|
775
|
+
model: generationUsage.model,
|
|
776
|
+
inputTokens: usageReported ? generationUsage.inputTokens : undefined,
|
|
777
|
+
outputTokens: usageReported
|
|
778
|
+
? generationUsage.outputTokens
|
|
779
|
+
: undefined,
|
|
780
|
+
cacheReadTokens: usageReported
|
|
781
|
+
? generationUsage.cacheReadTokens
|
|
782
|
+
: undefined,
|
|
783
|
+
cacheWriteTokens: usageReported
|
|
784
|
+
? generationUsage.cacheWriteTokens
|
|
785
|
+
: undefined,
|
|
786
|
+
costCentsX100: usageReported ? costCentsX100 : undefined,
|
|
787
|
+
durationMs: totalDurationMs,
|
|
788
|
+
firstTokenMs,
|
|
789
|
+
status: runStatus,
|
|
790
|
+
errorMessage,
|
|
791
|
+
toolCalls: toolCallCount,
|
|
792
|
+
successfulTools,
|
|
793
|
+
failedTools,
|
|
794
|
+
tools: [...generationToolCalls.entries()]
|
|
795
|
+
.sort(([a], [b]) => a - b)
|
|
796
|
+
.map(([, detail]) => detail),
|
|
797
|
+
toolsTruncated: toolInvocationCounter > MAX_TRACKED_GENERATION_TOOL_CALLS,
|
|
798
|
+
terminalOutcome: effectiveTerminalOutcome,
|
|
799
|
+
delegation: opts.delegation,
|
|
800
|
+
createdAt: runStart,
|
|
801
|
+
experimentAssignments: opts.experimentAssignments,
|
|
802
|
+
modelSelectionSource: opts.modelSelectionSource,
|
|
803
|
+
browserSessionId,
|
|
804
|
+
...generationContent,
|
|
649
805
|
});
|
|
650
806
|
}
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
toolCallIdToCounter.clear();
|
|
654
|
-
}
|
|
655
|
-
let costCentsX100 = 0;
|
|
656
|
-
try {
|
|
657
|
-
const { calculateCost } = await import("../usage/store.js");
|
|
658
|
-
if (usage) {
|
|
659
|
-
costCentsX100 = calculateCost(usage.inputTokens, usage.outputTokens, usage.model, usage.cacheReadTokens, usage.cacheWriteTokens);
|
|
660
|
-
}
|
|
661
|
-
}
|
|
662
|
-
catch { }
|
|
663
|
-
// A cut-off run never reaches the loop's outcome classification, so stand in
|
|
664
|
-
// for it here rather than reporting no terminal state at all. `failed` +
|
|
665
|
-
// `retryable` is the honest encoding available in `AgentLoopOutcome`: the
|
|
666
|
-
// turn did not finish, and the continuation machinery is expected to
|
|
667
|
-
// recover it. A real reported outcome always wins.
|
|
668
|
-
const effectiveTerminalOutcome = terminalOutcome ??
|
|
669
|
-
(cutOffReason && !EXPECTED_CONTINUATION_REASONS.has(cutOffReason)
|
|
670
|
-
? {
|
|
671
|
-
state: "failed",
|
|
672
|
-
code: cutOffReason,
|
|
673
|
-
retryable: true,
|
|
674
|
-
message: `Agent run was cut off before finishing (${cutOffReason}).`,
|
|
675
|
-
}
|
|
676
|
-
: undefined);
|
|
677
|
-
let llmCallCount = 0;
|
|
678
|
-
if (usage || runStatus === "error") {
|
|
679
|
-
llmCallCount =
|
|
680
|
-
usage?.llmCalls ??
|
|
681
|
-
// Compatibility for custom loop implementations that predate the
|
|
682
|
-
// attempt counter: a measured run still counts as one call.
|
|
683
|
-
1;
|
|
684
|
-
const generationUsage = usage ?? {
|
|
685
|
-
inputTokens: 0,
|
|
686
|
-
outputTokens: 0,
|
|
687
|
-
cacheReadTokens: 0,
|
|
688
|
-
cacheWriteTokens: 0,
|
|
689
|
-
model: loopOpts.model,
|
|
690
|
-
};
|
|
691
|
-
// The engine never reported a `usage` event for this run (killed for
|
|
692
|
-
// silence before any provider response, or the loop threw before
|
|
693
|
-
// returning). `generationUsage`'s token fields are placeholder zeros in
|
|
694
|
-
// that case, not measured values — the tracking event below must omit
|
|
695
|
-
// them rather than report a fabricated 0.
|
|
696
|
-
const usageReported = usage?.usageReported === true;
|
|
697
|
-
const firstTokenMs = usage?.firstEngineEventAtMs !== undefined
|
|
698
|
-
? Math.max(0, usage.firstEngineEventAtMs - runStart)
|
|
699
|
-
: undefined;
|
|
700
|
-
const llmSpanId = spanId();
|
|
701
|
-
const generationContent = buildGenerationContent({
|
|
702
|
-
config,
|
|
703
|
-
messages: loopOpts.messages,
|
|
704
|
-
tools: loopOpts.tools,
|
|
705
|
-
assistantText: assistantTextParts.join(""),
|
|
706
|
-
toolSpans: spans.filter((s) => s.spanType === "tool_call"),
|
|
707
|
-
});
|
|
708
|
-
const llmSpan = {
|
|
709
|
-
id: llmSpanId,
|
|
807
|
+
const parentSpan = {
|
|
808
|
+
id: parentSpanId,
|
|
710
809
|
runId,
|
|
711
810
|
threadId,
|
|
712
811
|
userId,
|
|
713
|
-
parentSpanId,
|
|
714
|
-
spanType: "
|
|
715
|
-
name:
|
|
716
|
-
inputTokens:
|
|
717
|
-
outputTokens:
|
|
718
|
-
cacheReadTokens:
|
|
719
|
-
cacheWriteTokens:
|
|
812
|
+
parentSpanId: null,
|
|
813
|
+
spanType: "agent_run",
|
|
814
|
+
name: spanName,
|
|
815
|
+
inputTokens: usage?.inputTokens ?? 0,
|
|
816
|
+
outputTokens: usage?.outputTokens ?? 0,
|
|
817
|
+
cacheReadTokens: usage?.cacheReadTokens ?? 0,
|
|
818
|
+
cacheWriteTokens: usage?.cacheWriteTokens ?? 0,
|
|
720
819
|
costCentsX100,
|
|
721
820
|
durationMs: totalDurationMs,
|
|
722
821
|
status: runStatus,
|
|
723
822
|
errorMessage,
|
|
724
|
-
metadata:
|
|
823
|
+
metadata: runMetadata,
|
|
725
824
|
createdAt: runStart,
|
|
726
825
|
};
|
|
727
|
-
spans.push(
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
? generationUsage.cacheWriteTokens
|
|
745
|
-
: undefined,
|
|
746
|
-
costCentsX100: usageReported ? costCentsX100 : undefined,
|
|
747
|
-
durationMs: totalDurationMs,
|
|
748
|
-
firstTokenMs,
|
|
749
|
-
status: runStatus,
|
|
750
|
-
errorMessage,
|
|
751
|
-
toolCalls: toolCallCount,
|
|
752
|
-
successfulTools,
|
|
753
|
-
failedTools,
|
|
754
|
-
tools: [...generationToolCalls.entries()]
|
|
755
|
-
.sort(([a], [b]) => a - b)
|
|
756
|
-
.map(([, detail]) => detail),
|
|
757
|
-
toolsTruncated: toolInvocationCounter > MAX_TRACKED_GENERATION_TOOL_CALLS,
|
|
758
|
-
terminalOutcome: effectiveTerminalOutcome,
|
|
759
|
-
delegation: opts.delegation,
|
|
760
|
-
createdAt: runStart,
|
|
761
|
-
experimentAssignments: opts.experimentAssignments,
|
|
762
|
-
modelSelectionSource: opts.modelSelectionSource,
|
|
763
|
-
browserSessionId,
|
|
764
|
-
...generationContent,
|
|
765
|
-
});
|
|
766
|
-
}
|
|
767
|
-
const parentSpan = {
|
|
768
|
-
id: parentSpanId,
|
|
769
|
-
runId,
|
|
770
|
-
threadId,
|
|
771
|
-
userId,
|
|
772
|
-
parentSpanId: null,
|
|
773
|
-
spanType: "agent_run",
|
|
774
|
-
name: "agent_run",
|
|
775
|
-
inputTokens: usage?.inputTokens ?? 0,
|
|
776
|
-
outputTokens: usage?.outputTokens ?? 0,
|
|
777
|
-
cacheReadTokens: usage?.cacheReadTokens ?? 0,
|
|
778
|
-
cacheWriteTokens: usage?.cacheWriteTokens ?? 0,
|
|
779
|
-
costCentsX100,
|
|
780
|
-
durationMs: totalDurationMs,
|
|
781
|
-
status: runStatus,
|
|
782
|
-
errorMessage,
|
|
783
|
-
metadata: runMetadata,
|
|
784
|
-
createdAt: runStart,
|
|
785
|
-
};
|
|
786
|
-
spans.push(parentSpan);
|
|
787
|
-
// PostHog LLM analytics: the run is a `$ai_trace`, each tool call an
|
|
788
|
-
// `$ai_span` under it. Emitted from the collected spans rather than from a
|
|
789
|
-
// second instrumentation pass, so the tree PostHog shows and the tree we
|
|
790
|
-
// persist cannot drift apart.
|
|
791
|
-
try {
|
|
792
|
-
const aiError = runStatus === "error"
|
|
793
|
-
? toAiErrorDetail(errorMessage, {
|
|
794
|
-
state: effectiveTerminalOutcome?.state,
|
|
795
|
-
code: effectiveTerminalOutcome?.state === "failed" ||
|
|
796
|
-
effectiveTerminalOutcome?.state === "input_required"
|
|
797
|
-
? effectiveTerminalOutcome.code
|
|
798
|
-
: undefined,
|
|
799
|
-
retryable: effectiveTerminalOutcome?.state === "failed"
|
|
800
|
-
? effectiveTerminalOutcome.retryable
|
|
801
|
-
: undefined,
|
|
802
|
-
})
|
|
803
|
-
: undefined;
|
|
804
|
-
const provider = llmProviderFromEngine(typeof loopOpts.engine?.name === "string"
|
|
805
|
-
? loopOpts.engine.name
|
|
806
|
-
: undefined, usage?.model ?? loopOpts.model);
|
|
807
|
-
const toolSpans = config.captureLlmSpans
|
|
808
|
-
? spans.filter((s) => s.spanType === "tool_call")
|
|
809
|
-
: [];
|
|
810
|
-
const emittedToolSpans = toolSpans.slice(0, MAX_AI_SPANS_PER_RUN);
|
|
811
|
-
const droppedToolSpans = toolSpans.length - emittedToolSpans.length;
|
|
812
|
-
emitAiTraceEvent({
|
|
813
|
-
runId,
|
|
814
|
-
threadId,
|
|
815
|
-
userId,
|
|
816
|
-
spanName: "agent_run",
|
|
817
|
-
model: usage?.model ?? loopOpts.model,
|
|
818
|
-
provider,
|
|
819
|
-
latencySeconds: Math.round(totalDurationMs) / 1000,
|
|
820
|
-
isError: runStatus === "error",
|
|
821
|
-
error: aiError,
|
|
822
|
-
inputTokens: usage?.usageReported ? usage.inputTokens : undefined,
|
|
823
|
-
outputTokens: usage?.usageReported ? usage.outputTokens : undefined,
|
|
824
|
-
costUsd: usage?.usageReported
|
|
825
|
-
? costUsdFromCenticents(costCentsX100)
|
|
826
|
-
: undefined,
|
|
827
|
-
createdAt: runStart,
|
|
828
|
-
browserSessionId,
|
|
829
|
-
extraProperties: {
|
|
830
|
-
...trackingIdentityProperties(),
|
|
831
|
-
source: "agent_observability",
|
|
832
|
-
run_id: runId,
|
|
833
|
-
thread_id: threadId,
|
|
834
|
-
// Present for planned boundaries too, which are not errors: the ratio
|
|
835
|
-
// of run_timeout to no_progress is the signal, and it is unreadable
|
|
836
|
-
// if only one side of it is recorded.
|
|
837
|
-
...(cutOffReason ? { terminal_reason: cutOffReason } : {}),
|
|
838
|
-
// A truncated run must not read as a complete one.
|
|
839
|
-
...(droppedToolSpans > 0
|
|
840
|
-
? {
|
|
841
|
-
$ai_spans_dropped: droppedToolSpans,
|
|
842
|
-
$ai_spans_emitted: emittedToolSpans.length,
|
|
843
|
-
}
|
|
844
|
-
: {}),
|
|
845
|
-
},
|
|
846
|
-
});
|
|
847
|
-
for (const span of emittedToolSpans) {
|
|
848
|
-
// `span.errorMessage` is the raw tool result. It routinely contains
|
|
849
|
-
// upstream response bodies with Authorization headers and standalone
|
|
850
|
-
// API keys, so it gets the same redaction + bounding the generation
|
|
851
|
-
// event's `tools[].error_message` already applies, and the same
|
|
852
|
-
// `captureToolResults` gate — exporting it here otherwise reintroduced
|
|
853
|
-
// the leak that gate exists to prevent. `$ai_is_error` still marks the
|
|
854
|
-
// failure when the content is withheld.
|
|
855
|
-
const toolErrorMessage = span.status === "error" &&
|
|
856
|
-
span.errorMessage &&
|
|
857
|
-
config.captureToolResults
|
|
858
|
-
? truncateToolErrorMessage(redactToolErrorMessage(span.errorMessage))
|
|
826
|
+
spans.push(parentSpan);
|
|
827
|
+
// PostHog LLM analytics: the run is a `$ai_trace`, each tool call an
|
|
828
|
+
// `$ai_span` under it. Emitted from the collected spans rather than from a
|
|
829
|
+
// second instrumentation pass, so the tree PostHog shows and the tree we
|
|
830
|
+
// persist cannot drift apart.
|
|
831
|
+
try {
|
|
832
|
+
const aiError = runStatus === "error"
|
|
833
|
+
? toAiErrorDetail(errorMessage, {
|
|
834
|
+
state: effectiveTerminalOutcome?.state,
|
|
835
|
+
code: effectiveTerminalOutcome?.state === "failed" ||
|
|
836
|
+
effectiveTerminalOutcome?.state === "input_required"
|
|
837
|
+
? effectiveTerminalOutcome.code
|
|
838
|
+
: undefined,
|
|
839
|
+
retryable: effectiveTerminalOutcome?.state === "failed"
|
|
840
|
+
? effectiveTerminalOutcome.retryable
|
|
841
|
+
: undefined,
|
|
842
|
+
})
|
|
859
843
|
: undefined;
|
|
860
|
-
|
|
844
|
+
const provider = llmProviderFromEngine(typeof loopOpts.engine?.name === "string"
|
|
845
|
+
? loopOpts.engine.name
|
|
846
|
+
: undefined, usage?.model ?? loopOpts.model);
|
|
847
|
+
const toolSpans = config.captureLlmSpans
|
|
848
|
+
? spans.filter((s) => s.spanType === "tool_call")
|
|
849
|
+
: [];
|
|
850
|
+
const emittedToolSpans = toolSpans.slice(0, MAX_AI_SPANS_PER_RUN);
|
|
851
|
+
const droppedToolSpans = toolSpans.length - emittedToolSpans.length;
|
|
852
|
+
emitAiTraceEvent({
|
|
861
853
|
runId,
|
|
862
854
|
threadId,
|
|
863
855
|
userId,
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
856
|
+
spanName,
|
|
857
|
+
model: usage?.model ?? loopOpts.model,
|
|
858
|
+
provider,
|
|
859
|
+
latencySeconds: Math.round(totalDurationMs) / 1000,
|
|
860
|
+
isError: runStatus === "error",
|
|
861
|
+
error: aiError,
|
|
862
|
+
inputTokens: usage?.usageReported ? usage.inputTokens : undefined,
|
|
863
|
+
outputTokens: usage?.usageReported ? usage.outputTokens : undefined,
|
|
864
|
+
costUsd: usage?.usageReported
|
|
865
|
+
? costUsdFromCenticents(costCentsX100)
|
|
870
866
|
: undefined,
|
|
871
|
-
createdAt:
|
|
867
|
+
createdAt: runStart,
|
|
872
868
|
browserSessionId,
|
|
873
|
-
// `metadata.input` is already redacted and only present when
|
|
874
|
-
// `captureToolArgs` is on; absent stays absent.
|
|
875
|
-
inputState: span.metadata?.input,
|
|
876
|
-
outputState: toolErrorMessage,
|
|
877
869
|
extraProperties: {
|
|
878
870
|
...trackingIdentityProperties(),
|
|
879
871
|
source: "agent_observability",
|
|
880
|
-
|
|
872
|
+
run_id: runId,
|
|
873
|
+
thread_id: threadId,
|
|
874
|
+
...aiTraceMetadataProperties(runMetadata),
|
|
875
|
+
// Present for planned boundaries too, which are not errors: the ratio
|
|
876
|
+
// of run_timeout to no_progress is the signal, and it is unreadable
|
|
877
|
+
// if only one side of it is recorded.
|
|
878
|
+
...(cutOffReason ? { terminal_reason: cutOffReason } : {}),
|
|
879
|
+
// A truncated run must not read as a complete one.
|
|
880
|
+
...(droppedToolSpans > 0
|
|
881
|
+
? {
|
|
882
|
+
$ai_spans_dropped: droppedToolSpans,
|
|
883
|
+
$ai_spans_emitted: emittedToolSpans.length,
|
|
884
|
+
}
|
|
885
|
+
: {}),
|
|
881
886
|
},
|
|
882
887
|
});
|
|
888
|
+
for (const span of emittedToolSpans) {
|
|
889
|
+
// `span.errorMessage` is the raw tool result. It routinely contains
|
|
890
|
+
// upstream response bodies with Authorization headers and standalone
|
|
891
|
+
// API keys, so it gets the same redaction + bounding the generation
|
|
892
|
+
// event's `tools[].error_message` already applies, and the same
|
|
893
|
+
// `captureToolResults` gate — exporting it here otherwise reintroduced
|
|
894
|
+
// the leak that gate exists to prevent. `$ai_is_error` still marks the
|
|
895
|
+
// failure when the content is withheld.
|
|
896
|
+
const toolErrorMessage = span.status === "error" &&
|
|
897
|
+
span.errorMessage &&
|
|
898
|
+
config.captureToolResults
|
|
899
|
+
? truncateToolErrorMessage(redactToolErrorMessage(span.errorMessage))
|
|
900
|
+
: undefined;
|
|
901
|
+
emitAiSpanEvent({
|
|
902
|
+
runId,
|
|
903
|
+
threadId,
|
|
904
|
+
userId,
|
|
905
|
+
spanId: span.id,
|
|
906
|
+
spanName: span.name,
|
|
907
|
+
latencySeconds: Math.round(span.durationMs) / 1000,
|
|
908
|
+
isError: span.status === "error",
|
|
909
|
+
error: toolErrorMessage
|
|
910
|
+
? toAiErrorDetail(toolErrorMessage)
|
|
911
|
+
: undefined,
|
|
912
|
+
createdAt: span.createdAt,
|
|
913
|
+
browserSessionId,
|
|
914
|
+
// `metadata.input` is already redacted and only present when
|
|
915
|
+
// `captureToolArgs` is on; absent stays absent.
|
|
916
|
+
inputState: span.metadata?.input,
|
|
917
|
+
outputState: toolErrorMessage,
|
|
918
|
+
extraProperties: {
|
|
919
|
+
...trackingIdentityProperties(),
|
|
920
|
+
source: "agent_observability",
|
|
921
|
+
span_type: "tool_call",
|
|
922
|
+
},
|
|
923
|
+
});
|
|
924
|
+
}
|
|
925
|
+
// coercion-ok: a throw here would skip trace persistence below
|
|
883
926
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
927
|
+
catch {
|
|
928
|
+
// LLM analytics must never affect the run or trace persistence.
|
|
929
|
+
}
|
|
930
|
+
const summary = {
|
|
931
|
+
runId,
|
|
932
|
+
threadId,
|
|
933
|
+
userId,
|
|
934
|
+
totalSpans: spans.length,
|
|
935
|
+
llmCalls: llmCallCount,
|
|
936
|
+
toolCalls: toolCallCount,
|
|
937
|
+
successfulTools,
|
|
938
|
+
failedTools,
|
|
939
|
+
totalDurationMs,
|
|
940
|
+
totalCostCentsX100: costCentsX100,
|
|
941
|
+
totalInputTokens: usage?.inputTokens ?? 0,
|
|
942
|
+
totalOutputTokens: usage?.outputTokens ?? 0,
|
|
943
|
+
model: usage?.model ?? loopOpts.model,
|
|
944
|
+
createdAt: runStart,
|
|
945
|
+
};
|
|
946
|
+
writeTraceData(spans, summary, runId, config).catch(() => { });
|
|
947
|
+
// OpenTelemetry export (no-op unless a provider is registered). Emit a
|
|
948
|
+
// self-contained `llm.call` span carrying model + token usage, end any
|
|
949
|
+
// tool spans still open (loop threw mid-tool), and end the run span. Awaited
|
|
950
|
+
// so the spans are emitted before the function returns; cheap when no-op.
|
|
951
|
+
try {
|
|
952
|
+
if (usage) {
|
|
953
|
+
endAgentSpan(await startAgentSpan("llm.call", {}), {
|
|
954
|
+
status: runStatus,
|
|
955
|
+
errorMessage,
|
|
956
|
+
attributes: {
|
|
957
|
+
"llm.model": usage.model,
|
|
958
|
+
"llm.input_tokens": usage.inputTokens,
|
|
959
|
+
"llm.output_tokens": usage.outputTokens,
|
|
960
|
+
"llm.cache_read_tokens": usage.cacheReadTokens,
|
|
961
|
+
"llm.cache_write_tokens": usage.cacheWriteTokens,
|
|
962
|
+
"llm.cost_cents_x100": costCentsX100,
|
|
963
|
+
},
|
|
964
|
+
});
|
|
965
|
+
}
|
|
966
|
+
for (const toolSpan of openOtelToolSpans) {
|
|
967
|
+
endAgentSpan(toolSpan, {
|
|
968
|
+
status: "error",
|
|
969
|
+
errorMessage: "Agent run ended before tool_done.",
|
|
970
|
+
});
|
|
971
|
+
}
|
|
972
|
+
openOtelToolSpans.clear();
|
|
973
|
+
endAgentSpan(await otelRunSpanPromise, {
|
|
913
974
|
status: runStatus,
|
|
914
975
|
errorMessage,
|
|
915
976
|
attributes: {
|
|
916
|
-
"
|
|
917
|
-
"
|
|
918
|
-
"
|
|
919
|
-
"
|
|
920
|
-
"
|
|
921
|
-
"
|
|
977
|
+
"agent.tool_calls": toolCallCount,
|
|
978
|
+
"agent.successful_tools": successfulTools,
|
|
979
|
+
"agent.failed_tools": failedTools,
|
|
980
|
+
"agent.duration_ms": totalDurationMs,
|
|
981
|
+
"agent.input_tokens": usage?.inputTokens ?? 0,
|
|
982
|
+
"agent.output_tokens": usage?.outputTokens ?? 0,
|
|
983
|
+
"agent.cost_cents_x100": costCentsX100,
|
|
984
|
+
"agent.terminal_state": effectiveTerminalOutcome?.state,
|
|
985
|
+
"agent.terminal_code": effectiveTerminalOutcome?.state === "failed" ||
|
|
986
|
+
effectiveTerminalOutcome?.state === "input_required"
|
|
987
|
+
? effectiveTerminalOutcome.code
|
|
988
|
+
: undefined,
|
|
922
989
|
},
|
|
923
990
|
});
|
|
991
|
+
// coercion-ok: OTel export must never break the run.
|
|
924
992
|
}
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
status: "error",
|
|
928
|
-
errorMessage: "Agent run ended before tool_done.",
|
|
929
|
-
});
|
|
993
|
+
catch {
|
|
994
|
+
// OTel export must never break the run.
|
|
930
995
|
}
|
|
931
|
-
openOtelToolSpans.clear();
|
|
932
|
-
endAgentSpan(await otelRunSpanPromise, {
|
|
933
|
-
status: runStatus,
|
|
934
|
-
errorMessage,
|
|
935
|
-
attributes: {
|
|
936
|
-
"agent.tool_calls": toolCallCount,
|
|
937
|
-
"agent.successful_tools": successfulTools,
|
|
938
|
-
"agent.failed_tools": failedTools,
|
|
939
|
-
"agent.duration_ms": totalDurationMs,
|
|
940
|
-
"agent.input_tokens": usage?.inputTokens ?? 0,
|
|
941
|
-
"agent.output_tokens": usage?.outputTokens ?? 0,
|
|
942
|
-
"agent.cost_cents_x100": costCentsX100,
|
|
943
|
-
"agent.terminal_state": effectiveTerminalOutcome?.state,
|
|
944
|
-
"agent.terminal_code": effectiveTerminalOutcome?.state === "failed" ||
|
|
945
|
-
effectiveTerminalOutcome?.state === "input_required"
|
|
946
|
-
? effectiveTerminalOutcome.code
|
|
947
|
-
: undefined,
|
|
948
|
-
},
|
|
949
|
-
});
|
|
950
996
|
}
|
|
951
|
-
catch {
|
|
952
|
-
//
|
|
997
|
+
catch (instrumentationError) {
|
|
998
|
+
// Deliberately not rethrown and deliberately not silent: the run's own
|
|
999
|
+
// outcome stands, and the telemetry failure is reported as its own.
|
|
1000
|
+
captureError(instrumentationError, {
|
|
1001
|
+
tags: { source: "agent-observability", phase: "trace-finalize" },
|
|
1002
|
+
aiTraceId: runId,
|
|
1003
|
+
extra: { runId, threadId },
|
|
1004
|
+
});
|
|
953
1005
|
}
|
|
954
1006
|
}
|
|
955
1007
|
// Classify only after the main loop has finished so the tiny managed Luna
|