@axlsdk/axl 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/dist/index.cjs +726 -145
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +385 -13
- package/dist/index.d.ts +385 -13
- package/dist/index.js +726 -145
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -2112,6 +2112,21 @@ var TimeoutError = class extends AxlError {
|
|
|
2112
2112
|
this.name = "TimeoutError";
|
|
2113
2113
|
}
|
|
2114
2114
|
};
|
|
2115
|
+
function formatBudgetCost(cost) {
|
|
2116
|
+
if (Number.isNaN(cost)) return "$NaN";
|
|
2117
|
+
if (cost === Infinity) return "$Infinity";
|
|
2118
|
+
if (cost === -Infinity) return "-$Infinity";
|
|
2119
|
+
if (cost === 0) return "$0.00";
|
|
2120
|
+
const sign = cost < 0 ? "-" : "";
|
|
2121
|
+
const abs = Math.abs(cost);
|
|
2122
|
+
if (abs < 1e-6) return `${sign}< $0.000001`;
|
|
2123
|
+
if (abs < 1e-4) {
|
|
2124
|
+
const [mantissa, exponent] = abs.toExponential(2).split("e");
|
|
2125
|
+
return `${sign}$${mantissa}e${parseInt(exponent, 10)}`;
|
|
2126
|
+
}
|
|
2127
|
+
if (abs < 0.01) return `${sign}$${abs.toFixed(6)}`;
|
|
2128
|
+
return `${sign}$${abs.toFixed(2)}`;
|
|
2129
|
+
}
|
|
2115
2130
|
var BudgetExceededError = class extends AxlError {
|
|
2116
2131
|
limit;
|
|
2117
2132
|
spent;
|
|
@@ -2119,7 +2134,9 @@ var BudgetExceededError = class extends AxlError {
|
|
|
2119
2134
|
constructor(limit, spent, policy) {
|
|
2120
2135
|
super(
|
|
2121
2136
|
"BUDGET_EXCEEDED",
|
|
2122
|
-
`Budget exceeded: spent
|
|
2137
|
+
`Budget exceeded: spent ${formatBudgetCost(spent)} of ${formatBudgetCost(
|
|
2138
|
+
limit
|
|
2139
|
+
)} limit (policy: ${policy})`
|
|
2123
2140
|
);
|
|
2124
2141
|
this.name = "BudgetExceededError";
|
|
2125
2142
|
this.limit = limit;
|
|
@@ -2307,6 +2324,14 @@ function extractBalanced(str, start, openChar, closeChar) {
|
|
|
2307
2324
|
}
|
|
2308
2325
|
return null;
|
|
2309
2326
|
}
|
|
2327
|
+
function appendRetryMessages(messages, content, feedbackMessage, providerMetadata) {
|
|
2328
|
+
messages.push({
|
|
2329
|
+
role: "assistant",
|
|
2330
|
+
content,
|
|
2331
|
+
...providerMetadata ? { providerMetadata } : {}
|
|
2332
|
+
});
|
|
2333
|
+
messages.push({ role: "system", content: feedbackMessage });
|
|
2334
|
+
}
|
|
2310
2335
|
function estimateMessagesTokens(messages) {
|
|
2311
2336
|
let total = 0;
|
|
2312
2337
|
for (const msg of messages) {
|
|
@@ -2354,6 +2379,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
2354
2379
|
awaitHumanHandler;
|
|
2355
2380
|
onAgentStart;
|
|
2356
2381
|
onAgentCallComplete;
|
|
2382
|
+
parentToolCallId;
|
|
2357
2383
|
constructor(init) {
|
|
2358
2384
|
this.input = init.input;
|
|
2359
2385
|
this.executionId = init.executionId;
|
|
@@ -2377,6 +2403,7 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
2377
2403
|
this.awaitHumanHandler = init.awaitHumanHandler;
|
|
2378
2404
|
this.onAgentStart = init.onAgentStart;
|
|
2379
2405
|
this.onAgentCallComplete = init.onAgentCallComplete;
|
|
2406
|
+
this.parentToolCallId = init.parentToolCallId;
|
|
2380
2407
|
if (init.metadata?.summaryCache) {
|
|
2381
2408
|
this.summaryCache = init.metadata.summaryCache;
|
|
2382
2409
|
}
|
|
@@ -2387,8 +2414,13 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
2387
2414
|
* state store, span manager, memory manager, MCP manager, config,
|
|
2388
2415
|
* awaitHuman handler, pending decisions, tool overrides.
|
|
2389
2416
|
* Isolates: session history, step counter, streaming callbacks (onToken, onAgentStart, onToolCall).
|
|
2417
|
+
*
|
|
2418
|
+
* @param parentToolCallId - The `callId` of the outer `tool_call` that
|
|
2419
|
+
* spawned this child. When set, every trace event emitted by the child
|
|
2420
|
+
* will carry this id as `parentToolCallId`, so consumers can join nested
|
|
2421
|
+
* agent calls back to the outer tool call that invoked them.
|
|
2390
2422
|
*/
|
|
2391
|
-
createChildContext() {
|
|
2423
|
+
createChildContext(parentToolCallId) {
|
|
2392
2424
|
return new _WorkflowContext({
|
|
2393
2425
|
input: this.input,
|
|
2394
2426
|
executionId: this.executionId,
|
|
@@ -2407,7 +2439,11 @@ var WorkflowContext = class _WorkflowContext {
|
|
|
2407
2439
|
pendingDecisions: this.pendingDecisions,
|
|
2408
2440
|
toolOverrides: this.toolOverrides,
|
|
2409
2441
|
signal: this.signal,
|
|
2410
|
-
workflowName: this.workflowName
|
|
2442
|
+
workflowName: this.workflowName,
|
|
2443
|
+
// Join key for nested event correlation. Inherit the parent's
|
|
2444
|
+
// `parentToolCallId` when this child is itself nested inside another
|
|
2445
|
+
// child — so grand-children still point to the outermost tool call.
|
|
2446
|
+
parentToolCallId: parentToolCallId ?? this.parentToolCallId
|
|
2411
2447
|
// Isolated: sessionHistory (empty), stepCounter (0),
|
|
2412
2448
|
// onToken (null), onAgentStart (null), onToolCall (null)
|
|
2413
2449
|
});
|
|
@@ -2577,11 +2613,22 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2577
2613
|
this.emitTrace({
|
|
2578
2614
|
type: "guardrail",
|
|
2579
2615
|
agent: agent2._name,
|
|
2580
|
-
data: {
|
|
2616
|
+
data: {
|
|
2617
|
+
guardrailType: "input",
|
|
2618
|
+
blocked: inputResult.block,
|
|
2619
|
+
...inputResult.reason ? { reason: inputResult.reason } : {},
|
|
2620
|
+
// Input guardrails can't retry (prompt is user-supplied), so attempt
|
|
2621
|
+
// and maxAttempts are always 1 — emit them for shape consistency with
|
|
2622
|
+
// output guardrails so consumers don't need two narrowers.
|
|
2623
|
+
attempt: 1,
|
|
2624
|
+
maxAttempts: 1
|
|
2625
|
+
}
|
|
2581
2626
|
});
|
|
2582
2627
|
this.spanManager?.addEventToActiveSpan("axl.guardrail.check", {
|
|
2583
2628
|
"axl.guardrail.type": "input",
|
|
2584
2629
|
"axl.guardrail.blocked": inputResult.block,
|
|
2630
|
+
"axl.guardrail.attempt": 1,
|
|
2631
|
+
"axl.guardrail.maxAttempts": 1,
|
|
2585
2632
|
...inputResult.reason ? { "axl.guardrail.reason": inputResult.reason } : {}
|
|
2586
2633
|
});
|
|
2587
2634
|
if (inputResult.block) {
|
|
@@ -2608,11 +2655,15 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2608
2655
|
let guardrailOutputRetries = 0;
|
|
2609
2656
|
let schemaRetries = 0;
|
|
2610
2657
|
let validateRetries = 0;
|
|
2658
|
+
const maxGuardrailRetries = guardrails?.maxRetries ?? 2;
|
|
2659
|
+
let pendingRetryReason;
|
|
2660
|
+
const verboseTrace = this.config.trace?.level === "full";
|
|
2611
2661
|
while (turns < maxTurns) {
|
|
2612
2662
|
if (Date.now() - startTime > timeoutMs) {
|
|
2613
2663
|
throw new TimeoutError("ctx.ask()", timeoutMs);
|
|
2614
2664
|
}
|
|
2615
2665
|
turns++;
|
|
2666
|
+
const turnStart = Date.now();
|
|
2616
2667
|
const chatOptions = {
|
|
2617
2668
|
model,
|
|
2618
2669
|
temperature: options?.temperature ?? agent2._config.temperature,
|
|
@@ -2693,16 +2744,22 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2693
2744
|
usageCapture.value = response.usage;
|
|
2694
2745
|
}
|
|
2695
2746
|
if (response.cost) {
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2747
|
+
this._accumulateBudgetCost(response.cost);
|
|
2748
|
+
}
|
|
2749
|
+
let messagesSnapshot;
|
|
2750
|
+
if (verboseTrace) {
|
|
2751
|
+
try {
|
|
2752
|
+
messagesSnapshot = structuredClone(currentMessages);
|
|
2753
|
+
} catch (err) {
|
|
2754
|
+
console.warn(
|
|
2755
|
+
"[axl] verbose trace messages snapshot failed to clone; emitting shallow copy:",
|
|
2756
|
+
err instanceof Error ? err.message : String(err)
|
|
2757
|
+
);
|
|
2758
|
+
messagesSnapshot = [...currentMessages];
|
|
2704
2759
|
}
|
|
2705
2760
|
}
|
|
2761
|
+
const retryReason = pendingRetryReason;
|
|
2762
|
+
pendingRetryReason = void 0;
|
|
2706
2763
|
this.emitTrace({
|
|
2707
2764
|
type: "agent_call",
|
|
2708
2765
|
agent: agent2._name,
|
|
@@ -2714,8 +2771,25 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2714
2771
|
output: response.usage.completion_tokens,
|
|
2715
2772
|
reasoning: response.usage.reasoning_tokens
|
|
2716
2773
|
} : void 0,
|
|
2717
|
-
duration: Date.now() -
|
|
2718
|
-
data: {
|
|
2774
|
+
duration: Date.now() - turnStart,
|
|
2775
|
+
data: {
|
|
2776
|
+
prompt,
|
|
2777
|
+
response: response.content,
|
|
2778
|
+
...systemPrompt ? { system: systemPrompt } : {},
|
|
2779
|
+
...response.thinking_content ? { thinking: response.thinking_content } : {},
|
|
2780
|
+
params: {
|
|
2781
|
+
...chatOptions.temperature !== void 0 ? { temperature: chatOptions.temperature } : {},
|
|
2782
|
+
maxTokens: chatOptions.maxTokens,
|
|
2783
|
+
...chatOptions.effort !== void 0 ? { effort: chatOptions.effort } : {},
|
|
2784
|
+
...chatOptions.thinkingBudget !== void 0 ? { thinkingBudget: chatOptions.thinkingBudget } : {},
|
|
2785
|
+
...chatOptions.includeThoughts !== void 0 ? { includeThoughts: chatOptions.includeThoughts } : {},
|
|
2786
|
+
...chatOptions.toolChoice !== void 0 ? { toolChoice: chatOptions.toolChoice } : {},
|
|
2787
|
+
...chatOptions.stop !== void 0 ? { stop: chatOptions.stop } : {}
|
|
2788
|
+
},
|
|
2789
|
+
turn: turns,
|
|
2790
|
+
...retryReason ? { retryReason } : {},
|
|
2791
|
+
...messagesSnapshot ? { messages: messagesSnapshot } : {}
|
|
2792
|
+
}
|
|
2719
2793
|
});
|
|
2720
2794
|
if (response.tool_calls && response.tool_calls.length > 0) {
|
|
2721
2795
|
currentMessages.push({
|
|
@@ -2780,7 +2854,13 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2780
2854
|
this.emitTrace({
|
|
2781
2855
|
type: "handoff",
|
|
2782
2856
|
agent: agent2._name,
|
|
2783
|
-
data: {
|
|
2857
|
+
data: {
|
|
2858
|
+
target: targetName,
|
|
2859
|
+
mode,
|
|
2860
|
+
duration,
|
|
2861
|
+
source: agent2._name,
|
|
2862
|
+
...handoffPrompt !== prompt ? { message: handoffPrompt } : {}
|
|
2863
|
+
}
|
|
2784
2864
|
});
|
|
2785
2865
|
return result;
|
|
2786
2866
|
}
|
|
@@ -2790,7 +2870,13 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2790
2870
|
this.emitTrace({
|
|
2791
2871
|
type: "handoff",
|
|
2792
2872
|
agent: agent2._name,
|
|
2793
|
-
data: {
|
|
2873
|
+
data: {
|
|
2874
|
+
target: targetName,
|
|
2875
|
+
mode,
|
|
2876
|
+
duration: Date.now() - handoffStart,
|
|
2877
|
+
source: agent2._name,
|
|
2878
|
+
...handoffPrompt !== prompt ? { message: handoffPrompt } : {}
|
|
2879
|
+
}
|
|
2794
2880
|
});
|
|
2795
2881
|
}
|
|
2796
2882
|
continue;
|
|
@@ -2810,7 +2896,7 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2810
2896
|
this.emitTrace({
|
|
2811
2897
|
type: "handoff",
|
|
2812
2898
|
agent: agent2._name,
|
|
2813
|
-
data: { target: targetName, mode, duration }
|
|
2899
|
+
data: { target: targetName, mode, duration, source: agent2._name }
|
|
2814
2900
|
});
|
|
2815
2901
|
return result;
|
|
2816
2902
|
}
|
|
@@ -2820,7 +2906,12 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2820
2906
|
this.emitTrace({
|
|
2821
2907
|
type: "handoff",
|
|
2822
2908
|
agent: agent2._name,
|
|
2823
|
-
data: {
|
|
2909
|
+
data: {
|
|
2910
|
+
target: targetName,
|
|
2911
|
+
mode,
|
|
2912
|
+
duration: Date.now() - handoffStart,
|
|
2913
|
+
source: agent2._name
|
|
2914
|
+
}
|
|
2824
2915
|
});
|
|
2825
2916
|
return onewayResult;
|
|
2826
2917
|
}
|
|
@@ -2913,24 +3004,18 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2913
3004
|
});
|
|
2914
3005
|
if (!decision.approved) {
|
|
2915
3006
|
const reason = decision.reason ?? "Denied by human";
|
|
2916
|
-
this.emitTrace({
|
|
2917
|
-
type: "tool_denied",
|
|
2918
|
-
agent: agent2._name,
|
|
2919
|
-
tool: toolName,
|
|
2920
|
-
data: { denied: true, reason, args: toolArgs }
|
|
2921
|
-
});
|
|
2922
3007
|
currentMessages.push({
|
|
2923
3008
|
role: "tool",
|
|
2924
3009
|
content: JSON.stringify({ error: `Tool denied by human: ${reason}` }),
|
|
2925
3010
|
tool_call_id: toolCall.id
|
|
2926
3011
|
});
|
|
2927
|
-
return false;
|
|
3012
|
+
return { approved: false, reason };
|
|
2928
3013
|
}
|
|
2929
|
-
return true;
|
|
3014
|
+
return { approved: true };
|
|
2930
3015
|
};
|
|
2931
|
-
let
|
|
3016
|
+
let approvalOutcome;
|
|
2932
3017
|
if (this.spanManager) {
|
|
2933
|
-
|
|
3018
|
+
approvalOutcome = await this.spanManager.withSpanAsync(
|
|
2934
3019
|
"axl.tool.approval",
|
|
2935
3020
|
{
|
|
2936
3021
|
"axl.tool.name": toolName,
|
|
@@ -2938,20 +3023,24 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2938
3023
|
},
|
|
2939
3024
|
async (span) => {
|
|
2940
3025
|
const result = await approvalFn();
|
|
2941
|
-
span.setAttribute("axl.tool.approval.approved", result);
|
|
3026
|
+
span.setAttribute("axl.tool.approval.approved", result.approved);
|
|
2942
3027
|
return result;
|
|
2943
3028
|
}
|
|
2944
3029
|
);
|
|
2945
3030
|
} else {
|
|
2946
|
-
|
|
3031
|
+
approvalOutcome = await approvalFn();
|
|
2947
3032
|
}
|
|
2948
|
-
if (!approved) continue;
|
|
2949
3033
|
this.emitTrace({
|
|
2950
|
-
type: "
|
|
3034
|
+
type: "tool_approval",
|
|
2951
3035
|
agent: agent2._name,
|
|
2952
3036
|
tool: toolName,
|
|
2953
|
-
data: {
|
|
3037
|
+
data: {
|
|
3038
|
+
approved: approvalOutcome.approved,
|
|
3039
|
+
args: toolArgs,
|
|
3040
|
+
...approvalOutcome.reason ? { reason: approvalOutcome.reason } : {}
|
|
3041
|
+
}
|
|
2954
3042
|
});
|
|
3043
|
+
if (!approvalOutcome.approved) continue;
|
|
2955
3044
|
}
|
|
2956
3045
|
if (tool2 && tool2.hooks?.before) {
|
|
2957
3046
|
try {
|
|
@@ -2984,7 +3073,7 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
2984
3073
|
resultContent2 = JSON.stringify(toolResult2);
|
|
2985
3074
|
}
|
|
2986
3075
|
} else if (tool2) {
|
|
2987
|
-
const childCtx = this.createChildContext();
|
|
3076
|
+
const childCtx = this.createChildContext(toolCall.id);
|
|
2988
3077
|
try {
|
|
2989
3078
|
toolResult2 = await tool2._execute(toolArgs, childCtx);
|
|
2990
3079
|
} catch (err) {
|
|
@@ -3044,37 +3133,43 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
3044
3133
|
const content = response.content;
|
|
3045
3134
|
if (guardrails?.output) {
|
|
3046
3135
|
const outputResult = await guardrails.output(content, { metadata: this.metadata });
|
|
3136
|
+
const attempt = guardrailOutputRetries + 1;
|
|
3137
|
+
const maxAttempts = maxGuardrailRetries + 1;
|
|
3138
|
+
const onBlock = guardrails.onBlock ?? "throw";
|
|
3139
|
+
let feedbackMessage;
|
|
3140
|
+
if (outputResult.block && onBlock === "retry" && guardrailOutputRetries < maxGuardrailRetries) {
|
|
3141
|
+
feedbackMessage = `Your previous response was blocked by a safety guardrail: ${outputResult.reason ?? "Output blocked"}. Please provide a different response that complies with the guidelines.`;
|
|
3142
|
+
}
|
|
3047
3143
|
this.emitTrace({
|
|
3048
3144
|
type: "guardrail",
|
|
3049
3145
|
agent: agent2._name,
|
|
3050
3146
|
data: {
|
|
3051
3147
|
guardrailType: "output",
|
|
3052
3148
|
blocked: outputResult.block,
|
|
3053
|
-
reason: outputResult.reason
|
|
3149
|
+
...outputResult.reason ? { reason: outputResult.reason } : {},
|
|
3150
|
+
attempt,
|
|
3151
|
+
maxAttempts,
|
|
3152
|
+
...feedbackMessage ? { feedbackMessage } : {}
|
|
3054
3153
|
}
|
|
3055
3154
|
});
|
|
3056
3155
|
this.spanManager?.addEventToActiveSpan("axl.guardrail.check", {
|
|
3057
3156
|
"axl.guardrail.type": "output",
|
|
3058
3157
|
"axl.guardrail.blocked": outputResult.block,
|
|
3158
|
+
"axl.guardrail.attempt": attempt,
|
|
3159
|
+
"axl.guardrail.maxAttempts": maxAttempts,
|
|
3059
3160
|
...outputResult.reason ? { "axl.guardrail.reason": outputResult.reason } : {}
|
|
3060
3161
|
});
|
|
3061
3162
|
if (outputResult.block) {
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
currentMessages.push({
|
|
3073
|
-
role: "system",
|
|
3074
|
-
content: `Your previous response was blocked by a safety guardrail: ${outputResult.reason ?? "Output blocked"}. Please provide a different response that complies with the guidelines.`
|
|
3075
|
-
});
|
|
3076
|
-
continue;
|
|
3077
|
-
}
|
|
3163
|
+
if (feedbackMessage) {
|
|
3164
|
+
guardrailOutputRetries++;
|
|
3165
|
+
appendRetryMessages(
|
|
3166
|
+
currentMessages,
|
|
3167
|
+
content,
|
|
3168
|
+
feedbackMessage,
|
|
3169
|
+
response.providerMetadata
|
|
3170
|
+
);
|
|
3171
|
+
pendingRetryReason = "guardrail";
|
|
3172
|
+
continue;
|
|
3078
3173
|
}
|
|
3079
3174
|
if (typeof onBlock === "function") {
|
|
3080
3175
|
return onBlock(outputResult.reason ?? "Output blocked by guardrail", {
|
|
@@ -3086,30 +3181,58 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
3086
3181
|
}
|
|
3087
3182
|
let validated = void 0;
|
|
3088
3183
|
if (options?.schema) {
|
|
3184
|
+
const maxSchemaRetries = options.retries ?? 3;
|
|
3185
|
+
const schemaAttempt = schemaRetries + 1;
|
|
3186
|
+
const schemaMaxAttempts = maxSchemaRetries + 1;
|
|
3187
|
+
let schemaValid = false;
|
|
3188
|
+
let schemaReason;
|
|
3189
|
+
let schemaFeedback;
|
|
3190
|
+
let schemaErr;
|
|
3089
3191
|
try {
|
|
3090
3192
|
const parsed = JSON.parse(extractJson(content));
|
|
3091
3193
|
validated = options.schema.parse(parsed);
|
|
3194
|
+
schemaValid = true;
|
|
3092
3195
|
} catch (err) {
|
|
3093
|
-
|
|
3196
|
+
schemaErr = err;
|
|
3197
|
+
schemaReason = err instanceof Error ? err.message : String(err);
|
|
3094
3198
|
if (schemaRetries < maxSchemaRetries) {
|
|
3199
|
+
schemaFeedback = `Your response was not valid JSON or did not match the required schema: ${schemaReason}. Please fix and try again.`;
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
this.emitTrace({
|
|
3203
|
+
type: "schema_check",
|
|
3204
|
+
agent: agent2._name,
|
|
3205
|
+
data: {
|
|
3206
|
+
valid: schemaValid,
|
|
3207
|
+
...schemaReason ? { reason: schemaReason } : {},
|
|
3208
|
+
attempt: schemaAttempt,
|
|
3209
|
+
maxAttempts: schemaMaxAttempts,
|
|
3210
|
+
...schemaFeedback ? { feedbackMessage: schemaFeedback } : {}
|
|
3211
|
+
}
|
|
3212
|
+
});
|
|
3213
|
+
this.spanManager?.addEventToActiveSpan("axl.schema.check", {
|
|
3214
|
+
"axl.schema.valid": schemaValid,
|
|
3215
|
+
"axl.schema.attempt": schemaAttempt,
|
|
3216
|
+
"axl.schema.maxAttempts": schemaMaxAttempts,
|
|
3217
|
+
...schemaReason ? { "axl.schema.reason": schemaReason } : {}
|
|
3218
|
+
});
|
|
3219
|
+
if (!schemaValid) {
|
|
3220
|
+
if (schemaFeedback) {
|
|
3095
3221
|
schemaRetries++;
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
role: "assistant",
|
|
3222
|
+
appendRetryMessages(
|
|
3223
|
+
currentMessages,
|
|
3099
3224
|
content,
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
content: `Your response was not valid JSON or did not match the required schema: ${errorMsg}. Please fix and try again.`
|
|
3105
|
-
});
|
|
3225
|
+
schemaFeedback,
|
|
3226
|
+
response.providerMetadata
|
|
3227
|
+
);
|
|
3228
|
+
pendingRetryReason = "schema";
|
|
3106
3229
|
continue;
|
|
3107
3230
|
}
|
|
3108
|
-
const zodErr =
|
|
3231
|
+
const zodErr = schemaErr instanceof import_zod.ZodError ? schemaErr : new import_zod.ZodError([
|
|
3109
3232
|
{
|
|
3110
3233
|
code: "custom",
|
|
3111
3234
|
path: [],
|
|
3112
|
-
message:
|
|
3235
|
+
message: schemaReason ?? "Schema parse failed"
|
|
3113
3236
|
}
|
|
3114
3237
|
]);
|
|
3115
3238
|
throw new VerifyError(content, zodErr, maxSchemaRetries);
|
|
@@ -3125,31 +3248,40 @@ ${JSON.stringify(jsonSchema, null, 2)}`;
|
|
|
3125
3248
|
const reason = err instanceof Error ? err.message : String(err);
|
|
3126
3249
|
validateResult = { valid: false, reason: `Validator error: ${reason}` };
|
|
3127
3250
|
}
|
|
3251
|
+
const maxValidateRetries = options.validateRetries ?? 2;
|
|
3252
|
+
const validateAttempt = validateRetries + 1;
|
|
3253
|
+
const validateMaxAttempts = maxValidateRetries + 1;
|
|
3254
|
+
let validateFeedback;
|
|
3255
|
+
if (!validateResult.valid && validateRetries < maxValidateRetries) {
|
|
3256
|
+
validateFeedback = `Your response parsed correctly but failed validation: ${validateResult.reason ?? "Validation failed"}. Previous attempts are visible above. Please fix and try again.`;
|
|
3257
|
+
}
|
|
3128
3258
|
this.emitTrace({
|
|
3129
3259
|
type: "validate",
|
|
3130
3260
|
agent: agent2._name,
|
|
3131
3261
|
data: {
|
|
3132
3262
|
valid: validateResult.valid,
|
|
3133
|
-
...validateResult.reason ? { reason: validateResult.reason } : {}
|
|
3263
|
+
...validateResult.reason ? { reason: validateResult.reason } : {},
|
|
3264
|
+
attempt: validateAttempt,
|
|
3265
|
+
maxAttempts: validateMaxAttempts,
|
|
3266
|
+
...validateFeedback ? { feedbackMessage: validateFeedback } : {}
|
|
3134
3267
|
}
|
|
3135
3268
|
});
|
|
3136
3269
|
this.spanManager?.addEventToActiveSpan("axl.validate.check", {
|
|
3137
3270
|
"axl.validate.valid": validateResult.valid,
|
|
3271
|
+
"axl.validate.attempt": validateAttempt,
|
|
3272
|
+
"axl.validate.maxAttempts": validateMaxAttempts,
|
|
3138
3273
|
...validateResult.reason ? { "axl.validate.reason": validateResult.reason } : {}
|
|
3139
3274
|
});
|
|
3140
3275
|
if (!validateResult.valid) {
|
|
3141
|
-
|
|
3142
|
-
if (validateRetries < maxValidateRetries) {
|
|
3276
|
+
if (validateFeedback) {
|
|
3143
3277
|
validateRetries++;
|
|
3144
|
-
|
|
3145
|
-
|
|
3278
|
+
appendRetryMessages(
|
|
3279
|
+
currentMessages,
|
|
3146
3280
|
content,
|
|
3147
|
-
|
|
3148
|
-
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
content: `Your response parsed correctly but failed validation: ${validateResult.reason ?? "Validation failed"}. Previous attempts are visible above. Please fix and try again.`
|
|
3152
|
-
});
|
|
3281
|
+
validateFeedback,
|
|
3282
|
+
response.providerMetadata
|
|
3283
|
+
);
|
|
3284
|
+
pendingRetryReason = "validate";
|
|
3153
3285
|
continue;
|
|
3154
3286
|
}
|
|
3155
3287
|
throw new ValidationError(
|
|
@@ -3518,6 +3650,17 @@ ${summaryResponse.content}`
|
|
|
3518
3650
|
async verify(fn, schema, options) {
|
|
3519
3651
|
const maxRetries = options?.retries ?? 3;
|
|
3520
3652
|
let lastRetry = void 0;
|
|
3653
|
+
const emitVerifyOutcome = (passed, attempts, lastError) => {
|
|
3654
|
+
this.emitTrace({
|
|
3655
|
+
type: "verify",
|
|
3656
|
+
agent: void 0,
|
|
3657
|
+
data: {
|
|
3658
|
+
passed,
|
|
3659
|
+
attempts,
|
|
3660
|
+
...lastError ? { lastError } : {}
|
|
3661
|
+
}
|
|
3662
|
+
});
|
|
3663
|
+
};
|
|
3521
3664
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
3522
3665
|
let rawOutput;
|
|
3523
3666
|
try {
|
|
@@ -3536,12 +3679,14 @@ ${summaryResponse.content}`
|
|
|
3536
3679
|
const errorMsg = validateResult.reason ?? "Validation failed";
|
|
3537
3680
|
lastRetry = { error: errorMsg, output: rawOutput, parsed };
|
|
3538
3681
|
if (attempt === maxRetries) {
|
|
3682
|
+
emitVerifyOutcome(false, attempt + 1, errorMsg);
|
|
3539
3683
|
if (options?.fallback !== void 0) return options.fallback;
|
|
3540
3684
|
throw new ValidationError(parsed, errorMsg, maxRetries);
|
|
3541
3685
|
}
|
|
3542
3686
|
continue;
|
|
3543
3687
|
}
|
|
3544
3688
|
}
|
|
3689
|
+
emitVerifyOutcome(true, attempt + 1);
|
|
3545
3690
|
return parsed;
|
|
3546
3691
|
} catch (err) {
|
|
3547
3692
|
if (err instanceof ValidationError) {
|
|
@@ -3551,6 +3696,7 @@ ${summaryResponse.content}`
|
|
|
3551
3696
|
parsed: err.lastOutput
|
|
3552
3697
|
};
|
|
3553
3698
|
if (attempt === maxRetries) {
|
|
3699
|
+
emitVerifyOutcome(false, attempt + 1, err.reason);
|
|
3554
3700
|
if (options?.fallback !== void 0) return options.fallback;
|
|
3555
3701
|
throw err;
|
|
3556
3702
|
}
|
|
@@ -3559,6 +3705,7 @@ ${summaryResponse.content}`
|
|
|
3559
3705
|
if (err instanceof VerifyError) {
|
|
3560
3706
|
lastRetry = { error: err.message, output: rawOutput ?? err.lastOutput };
|
|
3561
3707
|
if (attempt === maxRetries) {
|
|
3708
|
+
emitVerifyOutcome(false, attempt + 1, err.message);
|
|
3562
3709
|
if (options?.fallback !== void 0) return options.fallback;
|
|
3563
3710
|
throw err;
|
|
3564
3711
|
}
|
|
@@ -3567,12 +3714,14 @@ ${summaryResponse.content}`
|
|
|
3567
3714
|
const errorMsg = err instanceof import_zod.ZodError ? err.message : err instanceof Error ? err.message : String(err);
|
|
3568
3715
|
lastRetry = { error: errorMsg, output: rawOutput };
|
|
3569
3716
|
if (attempt === maxRetries) {
|
|
3717
|
+
emitVerifyOutcome(false, attempt + 1, errorMsg);
|
|
3570
3718
|
if (options?.fallback !== void 0) return options.fallback;
|
|
3571
3719
|
const zodErr = err instanceof import_zod.ZodError ? err : new import_zod.ZodError([{ code: "custom", path: [], message: errorMsg }]);
|
|
3572
3720
|
throw new VerifyError(rawOutput, zodErr, maxRetries);
|
|
3573
3721
|
}
|
|
3574
3722
|
}
|
|
3575
3723
|
}
|
|
3724
|
+
emitVerifyOutcome(false, maxRetries + 1, lastRetry?.error ?? "Verify failed");
|
|
3576
3725
|
if (options?.fallback !== void 0) return options.fallback;
|
|
3577
3726
|
throw new VerifyError(
|
|
3578
3727
|
lastRetry?.output,
|
|
@@ -3878,6 +4027,76 @@ ${summaryResponse.content}`
|
|
|
3878
4027
|
}
|
|
3879
4028
|
return decision;
|
|
3880
4029
|
}
|
|
4030
|
+
/**
|
|
4031
|
+
* Compose an `AbortSignal` for a memory operation's underlying embedder
|
|
4032
|
+
* fetch. Combines the parent context signal (user cancellation,
|
|
4033
|
+
* `runtime.execute({ signal })`) with the budget hard_stop abort so
|
|
4034
|
+
* that either cause correctly aborts an in-flight embed call.
|
|
4035
|
+
*
|
|
4036
|
+
* Returns `undefined` if there's nothing to cancel on — the embedder
|
|
4037
|
+
* runs without a signal in that case, matching its prior behavior.
|
|
4038
|
+
*
|
|
4039
|
+
* @internal
|
|
4040
|
+
*/
|
|
4041
|
+
_composeMemorySignal() {
|
|
4042
|
+
const budgetSignal = this.budgetContext?.abortController?.signal;
|
|
4043
|
+
if (this.signal && budgetSignal) return AbortSignal.any([this.signal, budgetSignal]);
|
|
4044
|
+
return this.signal ?? budgetSignal;
|
|
4045
|
+
}
|
|
4046
|
+
/**
|
|
4047
|
+
* Accumulate a cost amount into the active `budgetContext` and trip the
|
|
4048
|
+
* `exceeded` flag if we've crossed the limit. On `hard_stop` policy,
|
|
4049
|
+
* also fires the abort controller so in-flight operations cancel.
|
|
4050
|
+
*
|
|
4051
|
+
* Called from every code path that spends money: the `agent_call` loop,
|
|
4052
|
+
* semantic memory operations (`ctx.remember({embed:true})`, `ctx.recall({query})`),
|
|
4053
|
+
* and any future cost-emitting primitive. Centralizing the logic here
|
|
4054
|
+
* means `ctx.budget({ limit, policy })` accurately enforces the limit
|
|
4055
|
+
* across ALL cost sources — not just agent calls.
|
|
4056
|
+
*
|
|
4057
|
+
* @internal
|
|
4058
|
+
*/
|
|
4059
|
+
_accumulateBudgetCost(amount) {
|
|
4060
|
+
if (!this.budgetContext) return;
|
|
4061
|
+
if (!Number.isFinite(amount) || amount <= 0) return;
|
|
4062
|
+
this.budgetContext.totalCost += amount;
|
|
4063
|
+
if (this.budgetContext.totalCost >= this.budgetContext.limit) {
|
|
4064
|
+
this.budgetContext.exceeded = true;
|
|
4065
|
+
if (this.budgetContext.policy === "hard_stop" && this.budgetContext.abortController) {
|
|
4066
|
+
this.budgetContext.abortController.abort();
|
|
4067
|
+
}
|
|
4068
|
+
}
|
|
4069
|
+
}
|
|
4070
|
+
// ── workflow lifecycle trace emission ──────────────────────────────────
|
|
4071
|
+
//
|
|
4072
|
+
// These are called by the runtime at execution boundaries. They emit
|
|
4073
|
+
// first-class `workflow_start` / `workflow_end` trace events instead of
|
|
4074
|
+
// the previous `ctx.log('workflow_start', ...)` indirection — so consumers
|
|
4075
|
+
// that narrow via `event.type === 'workflow_start'` actually see them.
|
|
4076
|
+
// Internal: the runtime is the only caller, user workflows never call these.
|
|
4077
|
+
/** @internal */
|
|
4078
|
+
_emitWorkflowStart(input) {
|
|
4079
|
+
this.emitTrace({
|
|
4080
|
+
type: "workflow_start",
|
|
4081
|
+
workflow: this.workflowName,
|
|
4082
|
+
data: { input }
|
|
4083
|
+
});
|
|
4084
|
+
}
|
|
4085
|
+
/** @internal */
|
|
4086
|
+
_emitWorkflowEnd(info) {
|
|
4087
|
+
this.emitTrace({
|
|
4088
|
+
type: "workflow_end",
|
|
4089
|
+
workflow: this.workflowName,
|
|
4090
|
+
duration: info.duration,
|
|
4091
|
+
data: {
|
|
4092
|
+
status: info.status,
|
|
4093
|
+
duration: info.duration,
|
|
4094
|
+
...info.result !== void 0 ? { result: info.result } : {},
|
|
4095
|
+
...info.error !== void 0 ? { error: info.error } : {},
|
|
4096
|
+
...info.aborted ? { aborted: true } : {}
|
|
4097
|
+
}
|
|
4098
|
+
});
|
|
4099
|
+
}
|
|
3881
4100
|
// ── ctx.log() ─────────────────────────────────────────────────────────
|
|
3882
4101
|
log(event, data) {
|
|
3883
4102
|
this.emitTrace({
|
|
@@ -3913,8 +4132,59 @@ ${summaryResponse.content}`
|
|
|
3913
4132
|
if (!this.stateStore) {
|
|
3914
4133
|
throw new Error("A state store is required for memory operations.");
|
|
3915
4134
|
}
|
|
4135
|
+
if (this.budgetContext?.exceeded) {
|
|
4136
|
+
const { limit, totalCost: spent, policy } = this.budgetContext;
|
|
4137
|
+
if (policy !== "warn") {
|
|
4138
|
+
throw new BudgetExceededError(limit, spent, policy);
|
|
4139
|
+
}
|
|
4140
|
+
}
|
|
3916
4141
|
const sessionId = this.metadata?.sessionId;
|
|
3917
|
-
|
|
4142
|
+
const scope = options?.scope ?? (sessionId ? "session" : "global");
|
|
4143
|
+
try {
|
|
4144
|
+
const memorySignal = this._composeMemorySignal();
|
|
4145
|
+
const { usage } = await this.memoryManager.remember(
|
|
4146
|
+
key,
|
|
4147
|
+
value,
|
|
4148
|
+
this.stateStore,
|
|
4149
|
+
sessionId,
|
|
4150
|
+
options,
|
|
4151
|
+
memorySignal
|
|
4152
|
+
);
|
|
4153
|
+
if (usage?.cost != null) {
|
|
4154
|
+
this._accumulateBudgetCost(usage.cost);
|
|
4155
|
+
}
|
|
4156
|
+
this.emitTrace({
|
|
4157
|
+
type: "log",
|
|
4158
|
+
...usage?.cost != null ? { cost: usage.cost } : {},
|
|
4159
|
+
...usage?.tokens != null ? { tokens: { input: usage.tokens } } : {},
|
|
4160
|
+
data: {
|
|
4161
|
+
event: "memory_remember",
|
|
4162
|
+
key,
|
|
4163
|
+
scope,
|
|
4164
|
+
embed: options?.embed === true,
|
|
4165
|
+
...usage ? { usage } : {}
|
|
4166
|
+
}
|
|
4167
|
+
});
|
|
4168
|
+
} catch (err) {
|
|
4169
|
+
const partialUsage = err.axlEmbedUsage;
|
|
4170
|
+
if (partialUsage?.cost != null) {
|
|
4171
|
+
this._accumulateBudgetCost(partialUsage.cost);
|
|
4172
|
+
}
|
|
4173
|
+
this.emitTrace({
|
|
4174
|
+
type: "log",
|
|
4175
|
+
...partialUsage?.cost != null ? { cost: partialUsage.cost } : {},
|
|
4176
|
+
...partialUsage?.tokens != null ? { tokens: { input: partialUsage.tokens } } : {},
|
|
4177
|
+
data: {
|
|
4178
|
+
event: "memory_remember",
|
|
4179
|
+
key,
|
|
4180
|
+
scope,
|
|
4181
|
+
embed: options?.embed === true,
|
|
4182
|
+
...partialUsage ? { usage: partialUsage } : {},
|
|
4183
|
+
error: err instanceof Error ? err.message : String(err)
|
|
4184
|
+
}
|
|
4185
|
+
});
|
|
4186
|
+
throw err;
|
|
4187
|
+
}
|
|
3918
4188
|
}
|
|
3919
4189
|
/**
|
|
3920
4190
|
* Recall a value from memory by key, or perform semantic search if query option is provided.
|
|
@@ -3928,8 +4198,64 @@ ${summaryResponse.content}`
|
|
|
3928
4198
|
if (!this.stateStore) {
|
|
3929
4199
|
throw new Error("A state store is required for memory operations.");
|
|
3930
4200
|
}
|
|
4201
|
+
if (this.budgetContext?.exceeded) {
|
|
4202
|
+
const { limit, totalCost: spent, policy } = this.budgetContext;
|
|
4203
|
+
if (policy !== "warn") {
|
|
4204
|
+
throw new BudgetExceededError(limit, spent, policy);
|
|
4205
|
+
}
|
|
4206
|
+
}
|
|
3931
4207
|
const sessionId = this.metadata?.sessionId;
|
|
3932
|
-
|
|
4208
|
+
const scope = options?.scope ?? (sessionId ? "session" : "global");
|
|
4209
|
+
const semantic = options?.query !== void 0;
|
|
4210
|
+
try {
|
|
4211
|
+
const memorySignal = this._composeMemorySignal();
|
|
4212
|
+
const { data, usage } = await this.memoryManager.recall(
|
|
4213
|
+
key,
|
|
4214
|
+
this.stateStore,
|
|
4215
|
+
sessionId,
|
|
4216
|
+
options,
|
|
4217
|
+
memorySignal
|
|
4218
|
+
);
|
|
4219
|
+
let hit;
|
|
4220
|
+
let resultCount;
|
|
4221
|
+
if (semantic) {
|
|
4222
|
+
resultCount = Array.isArray(data) ? data.length : 0;
|
|
4223
|
+
hit = resultCount > 0;
|
|
4224
|
+
} else {
|
|
4225
|
+
hit = data !== null && data !== void 0;
|
|
4226
|
+
}
|
|
4227
|
+
if (usage?.cost != null) {
|
|
4228
|
+
this._accumulateBudgetCost(usage.cost);
|
|
4229
|
+
}
|
|
4230
|
+
this.emitTrace({
|
|
4231
|
+
type: "log",
|
|
4232
|
+
...usage?.cost != null ? { cost: usage.cost } : {},
|
|
4233
|
+
...usage?.tokens != null ? { tokens: { input: usage.tokens } } : {},
|
|
4234
|
+
data: {
|
|
4235
|
+
event: "memory_recall",
|
|
4236
|
+
key,
|
|
4237
|
+
scope,
|
|
4238
|
+
semantic,
|
|
4239
|
+
hit,
|
|
4240
|
+
...resultCount !== void 0 ? { resultCount } : {},
|
|
4241
|
+
...usage ? { usage } : {}
|
|
4242
|
+
}
|
|
4243
|
+
});
|
|
4244
|
+
return data;
|
|
4245
|
+
} catch (err) {
|
|
4246
|
+
this.emitTrace({
|
|
4247
|
+
type: "log",
|
|
4248
|
+
data: {
|
|
4249
|
+
event: "memory_recall",
|
|
4250
|
+
key,
|
|
4251
|
+
scope,
|
|
4252
|
+
semantic,
|
|
4253
|
+
hit: false,
|
|
4254
|
+
error: err instanceof Error ? err.message : String(err)
|
|
4255
|
+
}
|
|
4256
|
+
});
|
|
4257
|
+
throw err;
|
|
4258
|
+
}
|
|
3933
4259
|
}
|
|
3934
4260
|
/** Delete a memory entry by key. */
|
|
3935
4261
|
async forget(key, options) {
|
|
@@ -3942,7 +4268,25 @@ ${summaryResponse.content}`
|
|
|
3942
4268
|
throw new Error("A state store is required for memory operations.");
|
|
3943
4269
|
}
|
|
3944
4270
|
const sessionId = this.metadata?.sessionId;
|
|
3945
|
-
|
|
4271
|
+
const scope = options?.scope ?? (sessionId ? "session" : "global");
|
|
4272
|
+
try {
|
|
4273
|
+
await this.memoryManager.forget(key, this.stateStore, sessionId, options);
|
|
4274
|
+
this.emitTrace({
|
|
4275
|
+
type: "log",
|
|
4276
|
+
data: { event: "memory_forget", key, scope }
|
|
4277
|
+
});
|
|
4278
|
+
} catch (err) {
|
|
4279
|
+
this.emitTrace({
|
|
4280
|
+
type: "log",
|
|
4281
|
+
data: {
|
|
4282
|
+
event: "memory_forget",
|
|
4283
|
+
key,
|
|
4284
|
+
scope,
|
|
4285
|
+
error: err instanceof Error ? err.message : String(err)
|
|
4286
|
+
}
|
|
4287
|
+
});
|
|
4288
|
+
throw err;
|
|
4289
|
+
}
|
|
3946
4290
|
}
|
|
3947
4291
|
// ── ctx.delegate() ──────────────────────────────────────────────────
|
|
3948
4292
|
/**
|
|
@@ -3970,6 +4314,15 @@ ${summaryResponse.content}`
|
|
|
3970
4314
|
names.add(a._name);
|
|
3971
4315
|
}
|
|
3972
4316
|
if (agents.length === 1) {
|
|
4317
|
+
this.emitTrace({
|
|
4318
|
+
type: "delegate",
|
|
4319
|
+
agent: agents[0]._name,
|
|
4320
|
+
data: {
|
|
4321
|
+
candidates: [agents[0]._name],
|
|
4322
|
+
selected: agents[0]._name,
|
|
4323
|
+
reason: "single_candidate"
|
|
4324
|
+
}
|
|
4325
|
+
});
|
|
3973
4326
|
return this.ask(agents[0], prompt, {
|
|
3974
4327
|
schema: options?.schema,
|
|
3975
4328
|
retries: options?.retries,
|
|
@@ -4010,7 +4363,8 @@ ${summaryResponse.content}`
|
|
|
4010
4363
|
agent: "_delegate_router",
|
|
4011
4364
|
data: {
|
|
4012
4365
|
candidates: agents.map((a) => a._name),
|
|
4013
|
-
routerModel: routerModelUri
|
|
4366
|
+
routerModel: routerModelUri,
|
|
4367
|
+
reason: "routed"
|
|
4014
4368
|
}
|
|
4015
4369
|
});
|
|
4016
4370
|
return this.ask(routerAgent, prompt, {
|
|
@@ -4022,19 +4376,117 @@ ${summaryResponse.content}`
|
|
|
4022
4376
|
});
|
|
4023
4377
|
}
|
|
4024
4378
|
// ── Private ───────────────────────────────────────────────────────────
|
|
4379
|
+
/**
|
|
4380
|
+
* Internal emitter input — intentionally loose so call sites don't need to
|
|
4381
|
+
* build a perfectly-narrowed discriminated-union member. The resulting
|
|
4382
|
+
* `TraceEvent` (exported type) remains strict, and TypeScript narrows it at
|
|
4383
|
+
* consumer sites via the `type` discriminator.
|
|
4384
|
+
*/
|
|
4025
4385
|
emitTrace(partial) {
|
|
4026
4386
|
let data = partial.data;
|
|
4027
|
-
if (this.config.trace?.redact &&
|
|
4028
|
-
|
|
4387
|
+
if (this.config.trace?.redact && data) {
|
|
4388
|
+
if (partial.type === "agent_call") {
|
|
4389
|
+
const d = data;
|
|
4390
|
+
const redacted = {
|
|
4391
|
+
...d,
|
|
4392
|
+
prompt: "[redacted]",
|
|
4393
|
+
response: "[redacted]"
|
|
4394
|
+
};
|
|
4395
|
+
if (d.system !== void 0) redacted.system = "[redacted]";
|
|
4396
|
+
if (d.thinking !== void 0) redacted.thinking = "[redacted]";
|
|
4397
|
+
if (Array.isArray(d.messages)) {
|
|
4398
|
+
redacted.messages = [
|
|
4399
|
+
{ role: "system", content: `[${d.messages.length} messages redacted]` }
|
|
4400
|
+
];
|
|
4401
|
+
}
|
|
4402
|
+
data = redacted;
|
|
4403
|
+
} else if (partial.type === "guardrail" || partial.type === "schema_check" || partial.type === "validate") {
|
|
4404
|
+
const d = data;
|
|
4405
|
+
const needsRedact = d.feedbackMessage !== void 0 || d.reason !== void 0;
|
|
4406
|
+
if (needsRedact) {
|
|
4407
|
+
data = {
|
|
4408
|
+
...d,
|
|
4409
|
+
...d.reason !== void 0 ? { reason: "[redacted]" } : {},
|
|
4410
|
+
...d.feedbackMessage !== void 0 ? { feedbackMessage: "[redacted]" } : {}
|
|
4411
|
+
};
|
|
4412
|
+
}
|
|
4413
|
+
} else if (partial.type === "tool_call") {
|
|
4414
|
+
const d = data;
|
|
4415
|
+
data = {
|
|
4416
|
+
...d,
|
|
4417
|
+
args: "[redacted]",
|
|
4418
|
+
result: "[redacted]"
|
|
4419
|
+
};
|
|
4420
|
+
} else if (partial.type === "tool_approval") {
|
|
4421
|
+
const d = data;
|
|
4422
|
+
data = {
|
|
4423
|
+
...d,
|
|
4424
|
+
args: "[redacted]",
|
|
4425
|
+
...d.reason !== void 0 ? { reason: "[redacted]" } : {}
|
|
4426
|
+
};
|
|
4427
|
+
} else if (partial.type === "handoff") {
|
|
4428
|
+
const d = data;
|
|
4429
|
+
if (d.message !== void 0) {
|
|
4430
|
+
data = { ...d, message: "[redacted]" };
|
|
4431
|
+
}
|
|
4432
|
+
} else if (partial.type === "workflow_start") {
|
|
4433
|
+
const d = data;
|
|
4434
|
+
if (d.input !== void 0) {
|
|
4435
|
+
data = { ...d, input: "[redacted]" };
|
|
4436
|
+
}
|
|
4437
|
+
} else if (partial.type === "workflow_end") {
|
|
4438
|
+
const d = data;
|
|
4439
|
+
const redacted = { ...d };
|
|
4440
|
+
if (d.result !== void 0) redacted.result = "[redacted]";
|
|
4441
|
+
if (d.error !== void 0) redacted.error = "[redacted]";
|
|
4442
|
+
data = redacted;
|
|
4443
|
+
} else if (partial.type === "log") {
|
|
4444
|
+
const d = data;
|
|
4445
|
+
const redacted = {};
|
|
4446
|
+
for (const [k, v] of Object.entries(d)) {
|
|
4447
|
+
if (k === "event" || typeof v === "number" || typeof v === "boolean") {
|
|
4448
|
+
redacted[k] = v;
|
|
4449
|
+
} else if (typeof v === "string") {
|
|
4450
|
+
redacted[k] = "[redacted]";
|
|
4451
|
+
} else if (v !== null && typeof v === "object" && !Array.isArray(v)) {
|
|
4452
|
+
const inner = v;
|
|
4453
|
+
const innerRedacted = {};
|
|
4454
|
+
for (const [ik, iv] of Object.entries(inner)) {
|
|
4455
|
+
if (typeof iv === "number" || typeof iv === "boolean") {
|
|
4456
|
+
innerRedacted[ik] = iv;
|
|
4457
|
+
} else if (typeof iv === "string") {
|
|
4458
|
+
innerRedacted[ik] = "[redacted]";
|
|
4459
|
+
} else {
|
|
4460
|
+
innerRedacted[ik] = "[redacted]";
|
|
4461
|
+
}
|
|
4462
|
+
}
|
|
4463
|
+
redacted[k] = innerRedacted;
|
|
4464
|
+
} else {
|
|
4465
|
+
redacted[k] = "[redacted]";
|
|
4466
|
+
}
|
|
4467
|
+
}
|
|
4468
|
+
data = redacted;
|
|
4469
|
+
}
|
|
4029
4470
|
}
|
|
4030
4471
|
const event = {
|
|
4031
4472
|
executionId: this.executionId,
|
|
4032
4473
|
step: this.stepCounter++,
|
|
4033
4474
|
timestamp: Date.now(),
|
|
4475
|
+
...this.workflowName ? { workflow: this.workflowName } : {},
|
|
4476
|
+
...this.parentToolCallId ? { parentToolCallId: this.parentToolCallId } : {},
|
|
4034
4477
|
...partial,
|
|
4035
4478
|
data
|
|
4036
4479
|
};
|
|
4037
|
-
this.onTrace
|
|
4480
|
+
if (this.onTrace) {
|
|
4481
|
+
try {
|
|
4482
|
+
this.onTrace(event);
|
|
4483
|
+
} catch (err) {
|
|
4484
|
+
console.error(
|
|
4485
|
+
"[axl] onTrace handler threw; trace event dropped:",
|
|
4486
|
+
err instanceof Error ? err.message : String(err)
|
|
4487
|
+
);
|
|
4488
|
+
}
|
|
4489
|
+
}
|
|
4038
4490
|
}
|
|
4039
4491
|
};
|
|
4040
4492
|
|
|
@@ -5276,6 +5728,18 @@ var McpManager = class {
|
|
|
5276
5728
|
};
|
|
5277
5729
|
|
|
5278
5730
|
// src/memory/manager.ts
|
|
5731
|
+
function assertEmbedResult(result) {
|
|
5732
|
+
if (Array.isArray(result)) {
|
|
5733
|
+
throw new Error(
|
|
5734
|
+
"Embedder.embed() returned a bare number[][] (legacy shape). As of @axlsdk/axl 0.15, Embedder.embed() must return { vectors: number[][]; usage?: { tokens?, cost?, model? } }. Wrap your existing return value: `return { vectors: await old() }`."
|
|
5735
|
+
);
|
|
5736
|
+
}
|
|
5737
|
+
if (!result || typeof result !== "object" || !Array.isArray(result.vectors)) {
|
|
5738
|
+
throw new Error(
|
|
5739
|
+
`Embedder.embed() must return { vectors: number[][]; usage?: EmbedUsage }. Got ${result === null ? "null" : typeof result}.`
|
|
5740
|
+
);
|
|
5741
|
+
}
|
|
5742
|
+
}
|
|
5279
5743
|
var MemoryManager = class {
|
|
5280
5744
|
vectorStore;
|
|
5281
5745
|
embedder;
|
|
@@ -5290,8 +5754,11 @@ var MemoryManager = class {
|
|
|
5290
5754
|
* Store a key-value pair in memory.
|
|
5291
5755
|
* If a vector store and embedder are configured and embed is true,
|
|
5292
5756
|
* also embeds the value for semantic search.
|
|
5757
|
+
*
|
|
5758
|
+
* Returns `{ usage }` when embedding happened, carrying the embedder's
|
|
5759
|
+
* cost/token reporting out to the caller for cost attribution.
|
|
5293
5760
|
*/
|
|
5294
|
-
async remember(key, value, stateStore, sessionId, options) {
|
|
5761
|
+
async remember(key, value, stateStore, sessionId, options, signal) {
|
|
5295
5762
|
const scope = options?.scope ?? "session";
|
|
5296
5763
|
if (scope === "session" && !sessionId) {
|
|
5297
5764
|
throw new Error(
|
|
@@ -5310,23 +5777,44 @@ var MemoryManager = class {
|
|
|
5310
5777
|
}
|
|
5311
5778
|
if (options?.embed === true && this.vectorStore && this.embedder) {
|
|
5312
5779
|
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
5313
|
-
const
|
|
5314
|
-
|
|
5315
|
-
|
|
5316
|
-
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5780
|
+
const raw = await this.embedder.embed([text], signal);
|
|
5781
|
+
assertEmbedResult(raw);
|
|
5782
|
+
const { vectors, usage } = raw;
|
|
5783
|
+
try {
|
|
5784
|
+
await this.vectorStore.upsert([
|
|
5785
|
+
{
|
|
5786
|
+
id: `${storeScope}:${key}`,
|
|
5787
|
+
content: text,
|
|
5788
|
+
embedding: vectors[0],
|
|
5789
|
+
metadata: { key, scope, ...options?.metadata }
|
|
5790
|
+
}
|
|
5791
|
+
]);
|
|
5792
|
+
} catch (err) {
|
|
5793
|
+
if (usage && typeof err === "object" && err !== null) {
|
|
5794
|
+
Object.defineProperty(err, "axlEmbedUsage", {
|
|
5795
|
+
value: usage,
|
|
5796
|
+
enumerable: false,
|
|
5797
|
+
writable: true,
|
|
5798
|
+
configurable: true
|
|
5799
|
+
});
|
|
5320
5800
|
}
|
|
5321
|
-
|
|
5801
|
+
throw err;
|
|
5802
|
+
}
|
|
5803
|
+
return usage ? { usage } : {};
|
|
5322
5804
|
}
|
|
5805
|
+
return {};
|
|
5323
5806
|
}
|
|
5324
5807
|
/**
|
|
5325
5808
|
* Recall a value from memory by key, or perform semantic search if query is provided.
|
|
5809
|
+
*
|
|
5810
|
+
* Always returns `{ data }`; semantic recalls additionally include
|
|
5811
|
+
* `usage` from the embedder (when the embedder reported it).
|
|
5326
5812
|
*/
|
|
5327
|
-
async recall(key, stateStore, sessionId, options) {
|
|
5813
|
+
async recall(key, stateStore, sessionId, options, signal) {
|
|
5328
5814
|
if (options?.query && this.vectorStore && this.embedder) {
|
|
5329
|
-
const
|
|
5815
|
+
const raw = await this.embedder.embed([options.query], signal);
|
|
5816
|
+
assertEmbedResult(raw);
|
|
5817
|
+
const { vectors, usage } = raw;
|
|
5330
5818
|
const topK = options?.topK ?? 5;
|
|
5331
5819
|
const searchScope = options?.scope ?? "session";
|
|
5332
5820
|
if (searchScope === "session" && !sessionId) {
|
|
@@ -5334,13 +5822,14 @@ var MemoryManager = class {
|
|
|
5334
5822
|
'sessionId is required for session-scoped memory. Use { scope: "global" } or provide a sessionId in metadata.'
|
|
5335
5823
|
);
|
|
5336
5824
|
}
|
|
5337
|
-
const rawResults = await this.vectorStore.search(
|
|
5825
|
+
const rawResults = await this.vectorStore.search(vectors[0], topK * 3);
|
|
5338
5826
|
const filtered = rawResults.filter((r) => {
|
|
5339
5827
|
if (r.id.startsWith("global:")) return true;
|
|
5340
5828
|
if (searchScope === "session") return r.id.startsWith(`session:${sessionId}:`);
|
|
5341
5829
|
return true;
|
|
5342
5830
|
});
|
|
5343
|
-
|
|
5831
|
+
const data = filtered.slice(0, topK);
|
|
5832
|
+
return usage ? { data, usage } : { data };
|
|
5344
5833
|
}
|
|
5345
5834
|
const scope = options?.scope ?? "session";
|
|
5346
5835
|
if (scope === "session" && !sessionId) {
|
|
@@ -5350,10 +5839,10 @@ var MemoryManager = class {
|
|
|
5350
5839
|
}
|
|
5351
5840
|
const storeScope = scope === "session" ? `session:${sessionId}` : "global";
|
|
5352
5841
|
if (stateStore.getMemory) {
|
|
5353
|
-
return stateStore.getMemory(storeScope, key);
|
|
5842
|
+
return { data: await stateStore.getMemory(storeScope, key) };
|
|
5354
5843
|
}
|
|
5355
5844
|
const storeKey = `memory:${storeScope}:${key}`;
|
|
5356
|
-
return stateStore.getSessionMeta(storeKey, "value");
|
|
5845
|
+
return { data: await stateStore.getSessionMeta(storeKey, "value") };
|
|
5357
5846
|
}
|
|
5358
5847
|
/** Delete a memory entry. If a vector store is configured, also removes the embedding. */
|
|
5359
5848
|
async forget(key, stateStore, sessionId, options) {
|
|
@@ -5454,6 +5943,22 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5454
5943
|
});
|
|
5455
5944
|
}
|
|
5456
5945
|
}
|
|
5946
|
+
/**
|
|
5947
|
+
* Whether `config.trace.redact` is enabled on this runtime. A narrow
|
|
5948
|
+
* boolean getter is preferred over exposing the full config because:
|
|
5949
|
+
* (a) `Readonly<AxlConfig>` is shallow so the config would be mutable
|
|
5950
|
+
* at runtime through sub-objects, subverting any compliance guarantee;
|
|
5951
|
+
* (b) observability consumers like Studio only need the boolean, not
|
|
5952
|
+
* the whole config tree; (c) if future observability config needs to
|
|
5953
|
+
* expand, each new flag gets its own narrow getter.
|
|
5954
|
+
*
|
|
5955
|
+
* Studio's server-side redaction helpers (executions, memory, sessions,
|
|
5956
|
+
* evals, decisions, tools, playground, workflows) all route through
|
|
5957
|
+
* this accessor to decide whether to scrub response payloads.
|
|
5958
|
+
*/
|
|
5959
|
+
isRedactEnabled() {
|
|
5960
|
+
return this.config.trace?.redact === true;
|
|
5961
|
+
}
|
|
5457
5962
|
/**
|
|
5458
5963
|
* Initialize MCP servers configured in the config.
|
|
5459
5964
|
* Call this before executing workflows that use MCP tools.
|
|
@@ -5603,10 +6108,19 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5603
6108
|
metadata: result2.metadata ?? metadata
|
|
5604
6109
|
};
|
|
5605
6110
|
};
|
|
5606
|
-
result = await runEvalFn(entry.config, wrappedExecuteFn, this
|
|
6111
|
+
result = await runEvalFn(entry.config, wrappedExecuteFn, this, {
|
|
6112
|
+
onProgress: options?.onProgress,
|
|
6113
|
+
signal: options?.signal,
|
|
6114
|
+
captureTraces: options?.captureTraces
|
|
6115
|
+
});
|
|
5607
6116
|
} else {
|
|
5608
6117
|
result = await this.eval(
|
|
5609
|
-
entry.config
|
|
6118
|
+
entry.config,
|
|
6119
|
+
{
|
|
6120
|
+
onProgress: options?.onProgress,
|
|
6121
|
+
signal: options?.signal,
|
|
6122
|
+
captureTraces: options?.captureTraces
|
|
6123
|
+
}
|
|
5610
6124
|
);
|
|
5611
6125
|
}
|
|
5612
6126
|
const resultObj = result;
|
|
@@ -5744,6 +6258,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5744
6258
|
}
|
|
5745
6259
|
},
|
|
5746
6260
|
pendingDecisions: this.pendingDecisionResolvers,
|
|
6261
|
+
awaitHumanHandler: options?.awaitHumanHandler,
|
|
5747
6262
|
stateStore: this.stateStore,
|
|
5748
6263
|
workflowName: name,
|
|
5749
6264
|
mcpManager: this.mcpManager,
|
|
@@ -5757,7 +6272,6 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5757
6272
|
policy: "finish_and_stop"
|
|
5758
6273
|
}
|
|
5759
6274
|
});
|
|
5760
|
-
ctx.log("workflow_start", { workflow: name, executionId });
|
|
5761
6275
|
return this.spanManager.withSpanAsync(
|
|
5762
6276
|
"axl.workflow.execute",
|
|
5763
6277
|
{
|
|
@@ -5767,17 +6281,17 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5767
6281
|
},
|
|
5768
6282
|
async (span) => {
|
|
5769
6283
|
try {
|
|
6284
|
+
ctx._emitWorkflowStart(validated);
|
|
5770
6285
|
const result = await workflow2.handler(ctx);
|
|
5771
6286
|
const output = workflow2.outputSchema ? workflow2.outputSchema.parse(result) : result;
|
|
5772
6287
|
execInfo.status = "completed";
|
|
5773
6288
|
execInfo.completedAt = Date.now();
|
|
5774
6289
|
execInfo.duration = execInfo.completedAt - execInfo.startedAt;
|
|
5775
6290
|
execInfo.result = output;
|
|
5776
|
-
ctx.
|
|
5777
|
-
workflow: name,
|
|
6291
|
+
ctx._emitWorkflowEnd({
|
|
5778
6292
|
status: "completed",
|
|
5779
6293
|
duration: execInfo.duration,
|
|
5780
|
-
|
|
6294
|
+
result: output
|
|
5781
6295
|
});
|
|
5782
6296
|
if (this.stateStore.deleteCheckpoints) {
|
|
5783
6297
|
await this.stateStore.deleteCheckpoints(executionId);
|
|
@@ -5787,11 +6301,17 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5787
6301
|
this.persistExecution(execInfo);
|
|
5788
6302
|
return output;
|
|
5789
6303
|
} catch (err) {
|
|
6304
|
+
const aborted = typeof err === "object" && err !== null && err.name === "AbortError";
|
|
5790
6305
|
execInfo.status = "failed";
|
|
5791
6306
|
execInfo.completedAt = Date.now();
|
|
5792
6307
|
execInfo.duration = execInfo.completedAt - execInfo.startedAt;
|
|
5793
6308
|
execInfo.error = err instanceof Error ? err.message : String(err);
|
|
5794
|
-
ctx.
|
|
6309
|
+
ctx._emitWorkflowEnd({
|
|
6310
|
+
status: "failed",
|
|
6311
|
+
duration: execInfo.duration,
|
|
6312
|
+
error: execInfo.error,
|
|
6313
|
+
...aborted ? { aborted: true } : {}
|
|
6314
|
+
});
|
|
5795
6315
|
this.persistExecution(execInfo);
|
|
5796
6316
|
throw err;
|
|
5797
6317
|
} finally {
|
|
@@ -5856,24 +6376,16 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5856
6376
|
duration: data?.duration ?? void 0
|
|
5857
6377
|
});
|
|
5858
6378
|
}
|
|
5859
|
-
} else if (event.type === "
|
|
6379
|
+
} else if (event.type === "tool_approval") {
|
|
5860
6380
|
const data = event.data;
|
|
5861
|
-
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5865
|
-
|
|
5866
|
-
|
|
5867
|
-
|
|
5868
|
-
|
|
5869
|
-
axlStream._push({
|
|
5870
|
-
type: "tool_approval",
|
|
5871
|
-
name: event.tool ?? "",
|
|
5872
|
-
args: data?.args,
|
|
5873
|
-
approved: false,
|
|
5874
|
-
reason: data?.reason ?? void 0
|
|
5875
|
-
});
|
|
5876
|
-
}
|
|
6381
|
+
axlStream._push({
|
|
6382
|
+
type: "tool_approval",
|
|
6383
|
+
name: event.tool ?? "",
|
|
6384
|
+
args: data?.args,
|
|
6385
|
+
approved: data?.approved === true,
|
|
6386
|
+
...data?.reason ? { reason: data.reason } : {}
|
|
6387
|
+
});
|
|
6388
|
+
} else if (event.type === "tool_denied") {
|
|
5877
6389
|
} else if (event.type === "agent_call") {
|
|
5878
6390
|
axlStream._push({
|
|
5879
6391
|
type: "agent_end",
|
|
@@ -5906,6 +6418,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5906
6418
|
axlStream._push({ type: "agent_start", agent: info.agent, model: info.model });
|
|
5907
6419
|
},
|
|
5908
6420
|
pendingDecisions: this.pendingDecisionResolvers,
|
|
6421
|
+
awaitHumanHandler: options?.awaitHumanHandler,
|
|
5909
6422
|
stateStore: this.stateStore,
|
|
5910
6423
|
workflowName: name,
|
|
5911
6424
|
mcpManager: this.mcpManager,
|
|
@@ -5929,17 +6442,17 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5929
6442
|
},
|
|
5930
6443
|
async (span) => {
|
|
5931
6444
|
try {
|
|
6445
|
+
wfCtx._emitWorkflowStart(validated);
|
|
5932
6446
|
const rawResult = await workflow2.handler(wfCtx);
|
|
5933
6447
|
const result = workflow2.outputSchema ? workflow2.outputSchema.parse(rawResult) : rawResult;
|
|
5934
6448
|
execInfo.status = "completed";
|
|
5935
6449
|
execInfo.completedAt = Date.now();
|
|
5936
6450
|
execInfo.duration = execInfo.completedAt - execInfo.startedAt;
|
|
5937
6451
|
execInfo.result = result;
|
|
5938
|
-
wfCtx.
|
|
5939
|
-
workflow: name,
|
|
6452
|
+
wfCtx._emitWorkflowEnd({
|
|
5940
6453
|
status: "completed",
|
|
5941
6454
|
duration: execInfo.duration,
|
|
5942
|
-
|
|
6455
|
+
result
|
|
5943
6456
|
});
|
|
5944
6457
|
if (this.stateStore.deleteCheckpoints) {
|
|
5945
6458
|
await this.stateStore.deleteCheckpoints(executionId);
|
|
@@ -5956,14 +6469,16 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
5956
6469
|
};
|
|
5957
6470
|
run().then((result) => axlStream._done(result)).catch((err) => {
|
|
5958
6471
|
if (execInfo) {
|
|
6472
|
+
const aborted = typeof err === "object" && err !== null && err.name === "AbortError";
|
|
5959
6473
|
execInfo.status = "failed";
|
|
5960
6474
|
execInfo.completedAt = Date.now();
|
|
5961
6475
|
execInfo.duration = execInfo.completedAt - execInfo.startedAt;
|
|
5962
6476
|
execInfo.error = err instanceof Error ? err.message : String(err);
|
|
5963
|
-
ctx?.
|
|
5964
|
-
workflow: name,
|
|
6477
|
+
ctx?._emitWorkflowEnd({
|
|
5965
6478
|
status: "failed",
|
|
5966
|
-
|
|
6479
|
+
duration: execInfo.duration,
|
|
6480
|
+
error: execInfo.error,
|
|
6481
|
+
...aborted ? { aborted: true } : {}
|
|
5967
6482
|
});
|
|
5968
6483
|
this.persistExecution(execInfo);
|
|
5969
6484
|
}
|
|
@@ -6028,6 +6543,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6028
6543
|
if (this.stateStore.saveEvalResult) {
|
|
6029
6544
|
await this.stateStore.saveEvalResult(entry);
|
|
6030
6545
|
}
|
|
6546
|
+
this.emit("eval_result", entry);
|
|
6031
6547
|
}
|
|
6032
6548
|
/**
|
|
6033
6549
|
* Delete an eval history entry by id. Removes from in-memory cache and
|
|
@@ -6159,7 +6675,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6159
6675
|
*
|
|
6160
6676
|
* @see Spec Section 13.5
|
|
6161
6677
|
*/
|
|
6162
|
-
async eval(config) {
|
|
6678
|
+
async eval(config, options) {
|
|
6163
6679
|
let runEvalFn;
|
|
6164
6680
|
try {
|
|
6165
6681
|
({ runEval: runEvalFn } = await import("@axlsdk/eval"));
|
|
@@ -6174,7 +6690,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6174
6690
|
});
|
|
6175
6691
|
return { output: result, cost, metadata };
|
|
6176
6692
|
};
|
|
6177
|
-
return runEvalFn(config, executeWorkflow, this);
|
|
6693
|
+
return runEvalFn(config, executeWorkflow, this, options);
|
|
6178
6694
|
}
|
|
6179
6695
|
/**
|
|
6180
6696
|
* Compare two evaluation results to detect regressions and improvements.
|
|
@@ -6212,9 +6728,31 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6212
6728
|
* Returns cost (same as `trackCost`) plus metadata extracted from trace events:
|
|
6213
6729
|
* models (unique URIs), tokens (input/output/reasoning sums), and agent call count.
|
|
6214
6730
|
*
|
|
6731
|
+
* ## Cost vs tokens semantics
|
|
6732
|
+
*
|
|
6733
|
+
* - `cost` is the full aggregate across EVERY event with a top-level
|
|
6734
|
+
* `event.cost` set: agent calls, tool calls, semantic memory ops, etc.
|
|
6735
|
+
* This is the number to reconcile against your provider bill.
|
|
6736
|
+
*
|
|
6737
|
+
* - `metadata.tokens` is narrowly scoped to **agent** prompt/completion/
|
|
6738
|
+
* reasoning tokens. Embedder tokens from semantic `ctx.remember({embed:true})`
|
|
6739
|
+
* / `ctx.recall({query})` are deliberately NOT summed here — they're a
|
|
6740
|
+
* different category (input-only, different pricing, different model).
|
|
6741
|
+
* Conflating them would make "prompt tokens" misleading in the UI. If you
|
|
6742
|
+
* need embedder token counts, subscribe to `runtime.on('trace', ...)` and
|
|
6743
|
+
* read `data.usage.tokens` on `memory_remember` / `memory_recall` events.
|
|
6744
|
+
*
|
|
6745
|
+
* Pass `{ captureTraces: true }` to also collect the raw `TraceEvent[]` observed
|
|
6746
|
+
* during `fn()`. This is opt-in because it keeps every event in memory for the
|
|
6747
|
+
* duration of the call — useful for eval per-item capture, debugging, and test
|
|
6748
|
+
* assertions, but overhead grows with trace volume. When enabled, verbose-mode
|
|
6749
|
+
* `agent_call.data.messages` snapshots are omitted from captured events (still
|
|
6750
|
+
* broadcast via onTrace) to keep memory bounded — callers who need the full
|
|
6751
|
+
* verbose snapshot should subscribe to `runtime.on('trace', ...)` directly.
|
|
6752
|
+
*
|
|
6215
6753
|
* Works with both `createContext()` and `execute()` calls inside `fn`.
|
|
6216
6754
|
*/
|
|
6217
|
-
async trackExecution(fn) {
|
|
6755
|
+
async trackExecution(fn, options) {
|
|
6218
6756
|
const parentScope = costScopeStorage.getStore();
|
|
6219
6757
|
const scope = {
|
|
6220
6758
|
totalCost: 0,
|
|
@@ -6225,6 +6763,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6225
6763
|
const workflowCalls = /* @__PURE__ */ new Map();
|
|
6226
6764
|
const tokens = { input: 0, output: 0, reasoning: 0 };
|
|
6227
6765
|
let agentCalls = 0;
|
|
6766
|
+
const capturedTraces = options?.captureTraces ? [] : void 0;
|
|
6228
6767
|
const listener = (event) => {
|
|
6229
6768
|
if (!scope.trackedIds.has(event.executionId)) return;
|
|
6230
6769
|
if (event.cost) scope.totalCost += event.cost;
|
|
@@ -6237,17 +6776,22 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6237
6776
|
tokens.reasoning += event.tokens.reasoning ?? 0;
|
|
6238
6777
|
}
|
|
6239
6778
|
}
|
|
6240
|
-
let workflowName;
|
|
6241
6779
|
if (event.type === "workflow_start" && event.workflow) {
|
|
6242
|
-
|
|
6243
|
-
} else if (event.type === "log" && event.data && typeof event.data === "object") {
|
|
6244
|
-
const d = event.data;
|
|
6245
|
-
if (d.event === "workflow_start" && typeof d.workflow === "string") {
|
|
6246
|
-
workflowName = d.workflow;
|
|
6247
|
-
}
|
|
6780
|
+
workflowCalls.set(event.workflow, (workflowCalls.get(event.workflow) ?? 0) + 1);
|
|
6248
6781
|
}
|
|
6249
|
-
if (
|
|
6250
|
-
|
|
6782
|
+
if (capturedTraces) {
|
|
6783
|
+
if (event.type === "agent_call" && event.data) {
|
|
6784
|
+
const d = event.data;
|
|
6785
|
+
if ("messages" in d) {
|
|
6786
|
+
const rest = {};
|
|
6787
|
+
for (const k of Object.keys(d)) {
|
|
6788
|
+
if (k !== "messages") rest[k] = d[k];
|
|
6789
|
+
}
|
|
6790
|
+
capturedTraces.push({ ...event, data: rest });
|
|
6791
|
+
return;
|
|
6792
|
+
}
|
|
6793
|
+
}
|
|
6794
|
+
capturedTraces.push(event);
|
|
6251
6795
|
}
|
|
6252
6796
|
};
|
|
6253
6797
|
this.setMaxListeners(this.getMaxListeners() + 1);
|
|
@@ -6257,6 +6801,7 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6257
6801
|
return {
|
|
6258
6802
|
result,
|
|
6259
6803
|
cost: scope.totalCost,
|
|
6804
|
+
...capturedTraces ? { traces: capturedTraces } : {},
|
|
6260
6805
|
metadata: {
|
|
6261
6806
|
models: [...modelCalls.keys()],
|
|
6262
6807
|
modelCallCounts: modelCalls.size > 0 ? Object.fromEntries(modelCalls) : void 0,
|
|
@@ -6266,6 +6811,16 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6266
6811
|
workflowCallCounts: workflowCalls.size > 0 ? Object.fromEntries(workflowCalls) : void 0
|
|
6267
6812
|
}
|
|
6268
6813
|
};
|
|
6814
|
+
} catch (err) {
|
|
6815
|
+
if (capturedTraces && typeof err === "object" && err !== null) {
|
|
6816
|
+
Object.defineProperty(err, "axlCapturedTraces", {
|
|
6817
|
+
value: capturedTraces,
|
|
6818
|
+
enumerable: false,
|
|
6819
|
+
writable: true,
|
|
6820
|
+
configurable: true
|
|
6821
|
+
});
|
|
6822
|
+
}
|
|
6823
|
+
throw err;
|
|
6269
6824
|
} finally {
|
|
6270
6825
|
this.off("trace", listener);
|
|
6271
6826
|
this.setMaxListeners(this.getMaxListeners() - 1);
|
|
@@ -6320,11 +6875,13 @@ var AxlRuntime = class extends import_node_events2.EventEmitter {
|
|
|
6320
6875
|
const level = this.config.trace?.level ?? "steps";
|
|
6321
6876
|
const workflowPrefix = event.workflow ? `workflow:${event.workflow} | ` : "";
|
|
6322
6877
|
const parts = [`[axl] execution:${event.executionId}`];
|
|
6323
|
-
const data = event.data;
|
|
6324
6878
|
if (event.type === "workflow_start") {
|
|
6325
|
-
parts.push(`${workflowPrefix}
|
|
6879
|
+
parts.push(`${workflowPrefix}started`);
|
|
6326
6880
|
} else if (event.type === "workflow_end") {
|
|
6327
|
-
|
|
6881
|
+
const d = event.data;
|
|
6882
|
+
const status = d?.aborted ? "aborted" : d?.status ?? "completed";
|
|
6883
|
+
parts.push(`${workflowPrefix}${status}`);
|
|
6884
|
+
if (event.duration != null) parts.push(`${(event.duration / 1e3).toFixed(1)}s`);
|
|
6328
6885
|
} else if (event.type === "agent_call") {
|
|
6329
6886
|
parts.push(`${workflowPrefix}step:${event.step} agent_call`);
|
|
6330
6887
|
if (event.agent) parts.push(`agent:${event.agent}`);
|
|
@@ -6560,6 +7117,16 @@ var RedisStore = class _RedisStore {
|
|
|
6560
7117
|
};
|
|
6561
7118
|
|
|
6562
7119
|
// src/memory/embedder-openai.ts
|
|
7120
|
+
var EMBEDDING_PRICE_PER_1M_TOKENS = {
|
|
7121
|
+
"text-embedding-3-small": 0.02,
|
|
7122
|
+
"text-embedding-3-large": 0.13,
|
|
7123
|
+
"text-embedding-ada-002": 0.1
|
|
7124
|
+
};
|
|
7125
|
+
function computeCost(model, tokens) {
|
|
7126
|
+
const pricePerMillion = EMBEDDING_PRICE_PER_1M_TOKENS[model];
|
|
7127
|
+
if (pricePerMillion == null) return void 0;
|
|
7128
|
+
return tokens / 1e6 * pricePerMillion;
|
|
7129
|
+
}
|
|
6563
7130
|
var OpenAIEmbedder = class {
|
|
6564
7131
|
apiKey;
|
|
6565
7132
|
model;
|
|
@@ -6574,8 +7141,8 @@ var OpenAIEmbedder = class {
|
|
|
6574
7141
|
this.baseUrl = options.baseUrl ?? "https://api.openai.com";
|
|
6575
7142
|
this.dimensions = options.dimensions ?? 1536;
|
|
6576
7143
|
}
|
|
6577
|
-
async embed(texts) {
|
|
6578
|
-
const response = await
|
|
7144
|
+
async embed(texts, signal) {
|
|
7145
|
+
const response = await fetchWithRetry(`${this.baseUrl}/v1/embeddings`, {
|
|
6579
7146
|
method: "POST",
|
|
6580
7147
|
headers: {
|
|
6581
7148
|
"Content-Type": "application/json",
|
|
@@ -6585,14 +7152,28 @@ var OpenAIEmbedder = class {
|
|
|
6585
7152
|
model: this.model,
|
|
6586
7153
|
input: texts,
|
|
6587
7154
|
dimensions: this.dimensions
|
|
6588
|
-
})
|
|
7155
|
+
}),
|
|
7156
|
+
signal
|
|
6589
7157
|
});
|
|
6590
7158
|
if (!response.ok) {
|
|
6591
7159
|
const body = await response.text();
|
|
6592
7160
|
throw new Error(`OpenAI embeddings API error (${response.status}): ${body}`);
|
|
6593
7161
|
}
|
|
6594
7162
|
const json = await response.json();
|
|
6595
|
-
|
|
7163
|
+
const vectors = json.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
|
|
7164
|
+
const tokens = json.usage?.prompt_tokens ?? json.usage?.total_tokens;
|
|
7165
|
+
const cost = tokens != null ? computeCost(this.model, tokens) : void 0;
|
|
7166
|
+
if (tokens == null && cost == null) {
|
|
7167
|
+
return { vectors };
|
|
7168
|
+
}
|
|
7169
|
+
return {
|
|
7170
|
+
vectors,
|
|
7171
|
+
usage: {
|
|
7172
|
+
...tokens != null ? { tokens } : {},
|
|
7173
|
+
...cost != null ? { cost } : {},
|
|
7174
|
+
model: this.model
|
|
7175
|
+
}
|
|
7176
|
+
};
|
|
6596
7177
|
}
|
|
6597
7178
|
};
|
|
6598
7179
|
|