@amitdeshmukh/ax-crew 3.11.3 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +31 -62
- package/dist/agents/agentConfig.d.ts +6 -6
- package/dist/agents/agentConfig.js +39 -32
- package/dist/agents/index.d.ts +13 -10
- package/dist/agents/index.js +154 -40
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +2 -0
- package/dist/metrics/registry.d.ts +9 -0
- package/dist/metrics/registry.js +138 -0
- package/dist/metrics/types.d.ts +40 -0
- package/dist/metrics/types.js +1 -0
- package/dist/types.d.ts +6 -6
- package/examples/basic-researcher-writer.ts +18 -13
- package/examples/mcp-agent.ts +12 -8
- package/examples/solve-math-problem.ts +7 -5
- package/examples/streaming.ts +7 -6
- package/package.json +5 -3
- package/src/agents/agentConfig.ts +44 -38
- package/src/agents/index.ts +168 -59
- package/src/index.ts +2 -0
- package/src/metrics/index.ts +4 -0
- package/src/metrics/registry.ts +167 -0
- package/src/metrics/types.ts +48 -0
- package/src/types.ts +23 -7
- package/tests/AxCrew.test.ts +22 -18
package/src/agents/index.ts
CHANGED
|
@@ -20,7 +20,7 @@ import type {
|
|
|
20
20
|
|
|
21
21
|
import { createState } from "../state/index.js";
|
|
22
22
|
import { parseCrewConfig, parseAgentConfig } from "./agentConfig.js";
|
|
23
|
-
import {
|
|
23
|
+
import { MetricsRegistry } from "../metrics/index.js";
|
|
24
24
|
|
|
25
25
|
// Define the interface for the agent configuration
|
|
26
26
|
interface ParsedAgentConfig {
|
|
@@ -36,6 +36,7 @@ interface ParsedAgentConfig {
|
|
|
36
36
|
mcpServers?: Record<string, MCPTransportConfig>;
|
|
37
37
|
subAgentNames: string[];
|
|
38
38
|
examples?: Array<Record<string, any>>;
|
|
39
|
+
tracker?: any;
|
|
39
40
|
}
|
|
40
41
|
|
|
41
42
|
// Extend the AxAgent class from ax-llm
|
|
@@ -43,6 +44,14 @@ class StatefulAxAgent extends AxAgent<any, any> {
|
|
|
43
44
|
state: StateInstance;
|
|
44
45
|
axai: any;
|
|
45
46
|
private agentName: string;
|
|
47
|
+
private costTracker?: any;
|
|
48
|
+
private lastRecordedCostUSD: number = 0;
|
|
49
|
+
private isAxAIService(obj: any): obj is AxAI {
|
|
50
|
+
return !!obj && typeof obj.getName === 'function' && typeof obj.chat === 'function';
|
|
51
|
+
}
|
|
52
|
+
private isAxAIInstance(obj: any): obj is AxAI {
|
|
53
|
+
return !!obj && typeof obj === 'object' && ('defaults' in obj || 'modelInfo' in obj);
|
|
54
|
+
}
|
|
46
55
|
|
|
47
56
|
constructor(
|
|
48
57
|
ai: AxAI,
|
|
@@ -76,52 +85,92 @@ class StatefulAxAgent extends AxAgent<any, any> {
|
|
|
76
85
|
}
|
|
77
86
|
|
|
78
87
|
// Function overloads for forward method
|
|
79
|
-
async forward(values: Record<string, any>, options?: Readonly<AxProgramForwardOptions
|
|
80
|
-
async forward(ai: AxAI, values: Record<string, any>, options?: Readonly<AxProgramForwardOptions
|
|
88
|
+
async forward(values: Record<string, any>, options?: Readonly<AxProgramForwardOptions<any>>): Promise<Record<string, any>>;
|
|
89
|
+
async forward(ai: AxAI, values: Record<string, any>, options?: Readonly<AxProgramForwardOptions<any>>): Promise<Record<string, any>>;
|
|
81
90
|
|
|
82
91
|
// Implementation
|
|
83
92
|
async forward(
|
|
84
93
|
first: Record<string, any> | AxAI,
|
|
85
|
-
second?: Record<string, any> | Readonly<AxProgramForwardOptions
|
|
86
|
-
third?: Readonly<AxProgramForwardOptions
|
|
94
|
+
second?: Record<string, any> | Readonly<AxProgramForwardOptions<any>>,
|
|
95
|
+
third?: Readonly<AxProgramForwardOptions<any>>
|
|
87
96
|
): Promise<Record<string, any>> {
|
|
88
97
|
let result;
|
|
89
98
|
|
|
99
|
+
const start = performance.now();
|
|
100
|
+
const crewId = (this.state as any)?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
101
|
+
const labels = { crewId, agent: this.agentName } as any;
|
|
102
|
+
|
|
90
103
|
// Track costs regardless of whether it's a direct or sub-agent call
|
|
91
104
|
// This ensures we capture multiple legitimate calls to the same agent
|
|
92
|
-
if (
|
|
105
|
+
if (this.isAxAIService(first)) {
|
|
93
106
|
// Sub-agent case (called with AI service)
|
|
94
107
|
result = await super.forward(this.axai, second as Record<string, any>, third);
|
|
95
108
|
} else {
|
|
96
109
|
// Direct call case
|
|
97
|
-
result = await super.forward(this.axai, first, second as Readonly<AxProgramForwardOptions
|
|
110
|
+
result = await super.forward(this.axai, first, second as Readonly<AxProgramForwardOptions<any>>);
|
|
98
111
|
}
|
|
99
112
|
|
|
100
|
-
// Track costs after the call
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
113
|
+
// Track metrics and costs after the call using built-in usage
|
|
114
|
+
const durationMs = performance.now() - start;
|
|
115
|
+
MetricsRegistry.recordRequest(labels, false, durationMs);
|
|
116
|
+
// Always record tokens from built-in usage array if present
|
|
117
|
+
const builtIn = (this as any).getUsage?.();
|
|
118
|
+
if (Array.isArray(builtIn)) {
|
|
119
|
+
const totals = builtIn.reduce(
|
|
120
|
+
(acc: any, u: any) => {
|
|
121
|
+
const pt = u.tokens?.promptTokens ?? u.promptTokens ?? 0;
|
|
122
|
+
const ct = u.tokens?.completionTokens ?? u.completionTokens ?? 0;
|
|
123
|
+
acc.promptTokens += typeof pt === 'number' ? pt : 0;
|
|
124
|
+
acc.completionTokens += typeof ct === 'number' ? ct : 0;
|
|
125
|
+
// also aggregate per-model to feed Ax tracker
|
|
126
|
+
const model = u.model || (this.axai as any)?.getLastUsedChatModel?.() || (this.axai as any)?.defaults?.model;
|
|
127
|
+
if (model) {
|
|
128
|
+
acc.byModel[model] = (acc.byModel[model] || 0) + (pt + ct);
|
|
129
|
+
}
|
|
130
|
+
return acc;
|
|
131
|
+
},
|
|
132
|
+
{ promptTokens: 0, completionTokens: 0, byModel: {} as Record<string, number> }
|
|
133
|
+
);
|
|
134
|
+
MetricsRegistry.recordTokens(labels, {
|
|
135
|
+
promptTokens: totals.promptTokens,
|
|
136
|
+
completionTokens: totals.completionTokens,
|
|
137
|
+
totalTokens: totals.promptTokens + totals.completionTokens,
|
|
138
|
+
});
|
|
139
|
+
// Feed Ax's cost tracker with token totals per model; Ax owns pricing
|
|
140
|
+
const costTracker = (this as any).costTracker;
|
|
141
|
+
try {
|
|
142
|
+
for (const [m, count] of Object.entries(totals.byModel)) {
|
|
143
|
+
costTracker?.trackTokens?.(count, m);
|
|
144
|
+
}
|
|
145
|
+
const totalUSD = Number(costTracker?.getCurrentCost?.() ?? 0);
|
|
146
|
+
if (!Number.isNaN(totalUSD) && totalUSD > 0) {
|
|
147
|
+
MetricsRegistry.recordEstimatedCost(labels, totalUSD);
|
|
148
|
+
}
|
|
149
|
+
} catch {}
|
|
104
150
|
}
|
|
105
151
|
|
|
106
152
|
return result;
|
|
107
153
|
}
|
|
108
154
|
|
|
109
155
|
// Add streaming forward method overloads
|
|
110
|
-
streamingForward(values: Record<string, any>, options?: Readonly<AxProgramStreamingForwardOptions
|
|
111
|
-
streamingForward(ai: AxAI, values: Record<string, any>, options?: Readonly<AxProgramStreamingForwardOptions
|
|
156
|
+
streamingForward(values: Record<string, any>, options?: Readonly<AxProgramStreamingForwardOptions<any>>): AxGenStreamingOut<any>;
|
|
157
|
+
streamingForward(ai: AxAI, values: Record<string, any>, options?: Readonly<AxProgramStreamingForwardOptions<any>>): AxGenStreamingOut<any>;
|
|
112
158
|
|
|
113
159
|
// Implementation
|
|
114
160
|
streamingForward(
|
|
115
161
|
first: Record<string, any> | AxAI,
|
|
116
|
-
second?: Record<string, any> | Readonly<AxProgramStreamingForwardOptions
|
|
117
|
-
third?: Readonly<AxProgramStreamingForwardOptions
|
|
162
|
+
second?: Record<string, any> | Readonly<AxProgramStreamingForwardOptions<any>>,
|
|
163
|
+
third?: Readonly<AxProgramStreamingForwardOptions<any>>
|
|
118
164
|
): AxGenStreamingOut<any> {
|
|
165
|
+
const start = performance.now();
|
|
166
|
+
const crewId = (this.state as any)?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
167
|
+
const labels = { crewId, agent: this.agentName } as any;
|
|
119
168
|
let streamingResult: AxGenStreamingOut<any>;
|
|
120
169
|
|
|
121
|
-
if (
|
|
170
|
+
if (this.isAxAIService(first)) {
|
|
122
171
|
streamingResult = super.streamingForward(this.axai, second as Record<string, any>, third);
|
|
123
172
|
} else {
|
|
124
|
-
streamingResult = super.streamingForward(this.axai, first, second as Readonly<AxProgramStreamingForwardOptions
|
|
173
|
+
streamingResult = super.streamingForward(this.axai, first, second as Readonly<AxProgramStreamingForwardOptions<any>>);
|
|
125
174
|
}
|
|
126
175
|
|
|
127
176
|
// Create a new async generator that tracks costs after completion
|
|
@@ -131,43 +180,71 @@ class StatefulAxAgent extends AxAgent<any, any> {
|
|
|
131
180
|
yield chunk;
|
|
132
181
|
}
|
|
133
182
|
} finally {
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
183
|
+
const durationMs = performance.now() - start;
|
|
184
|
+
MetricsRegistry.recordRequest(labels, true, durationMs);
|
|
185
|
+
// Record tokens from built-in usage array if present
|
|
186
|
+
const builtIn = (this as any).getUsage?.();
|
|
187
|
+
if (Array.isArray(builtIn)) {
|
|
188
|
+
const totals = builtIn.reduce(
|
|
189
|
+
(acc: any, u: any) => {
|
|
190
|
+
const pt = u.tokens?.promptTokens ?? u.promptTokens ?? 0;
|
|
191
|
+
const ct = u.tokens?.completionTokens ?? u.completionTokens ?? 0;
|
|
192
|
+
acc.promptTokens += typeof pt === 'number' ? pt : 0;
|
|
193
|
+
acc.completionTokens += typeof ct === 'number' ? ct : 0;
|
|
194
|
+
const model = u.model || (this.axai as any)?.getLastUsedChatModel?.() || (this.axai as any)?.defaults?.model;
|
|
195
|
+
if (model) {
|
|
196
|
+
acc.byModel[model] = (acc.byModel[model] || 0) + (pt + ct);
|
|
197
|
+
}
|
|
198
|
+
return acc;
|
|
199
|
+
},
|
|
200
|
+
{ promptTokens: 0, completionTokens: 0, byModel: {} as Record<string, number> }
|
|
201
|
+
);
|
|
202
|
+
MetricsRegistry.recordTokens(labels, {
|
|
203
|
+
promptTokens: totals.promptTokens,
|
|
204
|
+
completionTokens: totals.completionTokens,
|
|
205
|
+
totalTokens: totals.promptTokens + totals.completionTokens,
|
|
206
|
+
});
|
|
207
|
+
const costTracker = (this as any).costTracker;
|
|
208
|
+
try {
|
|
209
|
+
for (const [m, count] of Object.entries(totals.byModel)) {
|
|
210
|
+
costTracker?.trackTokens?.(count, m);
|
|
211
|
+
}
|
|
212
|
+
const totalUSD = Number(costTracker?.getCurrentCost?.() ?? 0);
|
|
213
|
+
if (!Number.isNaN(totalUSD) && totalUSD > 0) {
|
|
214
|
+
MetricsRegistry.recordEstimatedCost(labels, totalUSD);
|
|
215
|
+
}
|
|
216
|
+
} catch {}
|
|
137
217
|
}
|
|
218
|
+
// Record estimated cost (USD) via attached tracker if available
|
|
219
|
+
const costTracker = (this as any).costTracker;
|
|
220
|
+
try {
|
|
221
|
+
const totalUSD = Number(costTracker?.getCurrentCost?.() ?? 0);
|
|
222
|
+
if (!Number.isNaN(totalUSD) && totalUSD > 0) {
|
|
223
|
+
MetricsRegistry.recordEstimatedCost(labels, totalUSD);
|
|
224
|
+
}
|
|
225
|
+
} catch {}
|
|
138
226
|
}
|
|
139
227
|
}).bind(this)();
|
|
140
228
|
|
|
141
229
|
return wrappedGenerator as AxGenStreamingOut<any>;
|
|
142
230
|
}
|
|
143
231
|
|
|
144
|
-
//
|
|
145
|
-
getLastUsageCost(): UsageCost | null {
|
|
146
|
-
const { modelUsage, modelInfo, defaults } = this.axai;
|
|
147
|
-
|
|
148
|
-
// Check if all required properties exist
|
|
149
|
-
if (!modelUsage?.promptTokens || !modelUsage?.completionTokens) {
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
232
|
+
// Legacy cost API removed: rely on Ax trackers for cost reporting
|
|
233
|
+
getLastUsageCost(): UsageCost | null { return null; }
|
|
152
234
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
const currentModelInfo = modelInfo.find((m: { name: string }) => m.name === defaults.model);
|
|
158
|
-
|
|
159
|
-
if (!currentModelInfo?.promptTokenCostPer1M || !currentModelInfo?.completionTokenCostPer1M) {
|
|
160
|
-
return null;
|
|
161
|
-
}
|
|
235
|
+
// Get the accumulated costs for all runs of this agent
|
|
236
|
+
getAccumulatedCosts(): UsageCost | null { return null; }
|
|
162
237
|
|
|
163
|
-
|
|
238
|
+
// Metrics API for this agent
|
|
239
|
+
getMetrics() {
|
|
240
|
+
const crewId = (this.state as any)?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
241
|
+
return MetricsRegistry.snapshot({ crewId, agent: this.agentName } as any);
|
|
164
242
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
const stateKey = `${StateFulAxAgentUsage.STATE_KEY_PREFIX}${this.agentName}`;
|
|
169
|
-
return this.state.get(stateKey) as UsageCost | null;
|
|
243
|
+
resetMetrics(): void {
|
|
244
|
+
const crewId = (this.state as any)?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
245
|
+
MetricsRegistry.reset({ crewId, agent: this.agentName } as any);
|
|
170
246
|
}
|
|
247
|
+
|
|
171
248
|
}
|
|
172
249
|
|
|
173
250
|
/**
|
|
@@ -208,6 +285,8 @@ class AxCrew {
|
|
|
208
285
|
this.crewId = crewId;
|
|
209
286
|
this.agents = new Map<string, StatefulAxAgent>();
|
|
210
287
|
this.state = createState(crewId);
|
|
288
|
+
// Make crewId discoverable to metrics
|
|
289
|
+
this.state.set('crewId', crewId);
|
|
211
290
|
}
|
|
212
291
|
|
|
213
292
|
/**
|
|
@@ -226,7 +305,7 @@ class AxCrew {
|
|
|
226
305
|
);
|
|
227
306
|
|
|
228
307
|
// Destructure with type assertion
|
|
229
|
-
const { ai, name, description, signature, functions, subAgentNames, examples } = agentConfig;
|
|
308
|
+
const { ai, name, description, signature, functions, subAgentNames, examples, tracker } = agentConfig;
|
|
230
309
|
|
|
231
310
|
// Get subagents for the AI agent
|
|
232
311
|
const subAgents = subAgentNames.map((subAgentName: string) => {
|
|
@@ -238,6 +317,30 @@ class AxCrew {
|
|
|
238
317
|
return this.agents?.get(subAgentName);
|
|
239
318
|
});
|
|
240
319
|
|
|
320
|
+
// Dedupe sub-agents by name (defensive)
|
|
321
|
+
const subAgentSet = new Map<string, StatefulAxAgent>();
|
|
322
|
+
for (const sa of subAgents.filter((agent): agent is StatefulAxAgent => agent !== undefined)) {
|
|
323
|
+
const n = (sa as any)?.agentName ?? (sa as any)?.name ?? '';
|
|
324
|
+
if (!subAgentSet.has(n)) subAgentSet.set(n, sa);
|
|
325
|
+
}
|
|
326
|
+
const uniqueSubAgents = Array.from(subAgentSet.values());
|
|
327
|
+
|
|
328
|
+
// Dedupe functions by name and avoid collision with sub-agent names
|
|
329
|
+
const subAgentNameSet = new Set(uniqueSubAgents.map((sa: any) => sa?.agentName ?? sa?.name).filter(Boolean));
|
|
330
|
+
const uniqueFunctions: AxFunction[] = [];
|
|
331
|
+
const seenFn = new Set<string>();
|
|
332
|
+
for (const fn of functions.filter((fn): fn is AxFunction => fn !== undefined)) {
|
|
333
|
+
const fnName = fn.name;
|
|
334
|
+
if (subAgentNameSet.has(fnName)) {
|
|
335
|
+
// Skip function that collides with a sub-agent name
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
if (!seenFn.has(fnName)) {
|
|
339
|
+
seenFn.add(fnName);
|
|
340
|
+
uniqueFunctions.push(fn);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
241
344
|
// Create an instance of StatefulAxAgent
|
|
242
345
|
const agent = new StatefulAxAgent(
|
|
243
346
|
ai,
|
|
@@ -245,16 +348,13 @@ class AxCrew {
|
|
|
245
348
|
name,
|
|
246
349
|
description,
|
|
247
350
|
signature,
|
|
248
|
-
functions:
|
|
249
|
-
|
|
250
|
-
),
|
|
251
|
-
agents: subAgents.filter(
|
|
252
|
-
(agent): agent is StatefulAxAgent => agent !== undefined
|
|
253
|
-
),
|
|
351
|
+
functions: uniqueFunctions,
|
|
352
|
+
agents: uniqueSubAgents,
|
|
254
353
|
examples,
|
|
255
354
|
},
|
|
256
355
|
this.state
|
|
257
356
|
);
|
|
357
|
+
(agent as any).costTracker = tracker;
|
|
258
358
|
|
|
259
359
|
return agent;
|
|
260
360
|
} catch (error) {
|
|
@@ -394,20 +494,29 @@ class AxCrew {
|
|
|
394
494
|
this.state.reset();
|
|
395
495
|
}
|
|
396
496
|
|
|
397
|
-
/**
|
|
398
|
-
* Gets aggregated costs for all agents in the crew
|
|
399
|
-
* @returns Aggregated cost information for all agents
|
|
400
|
-
*/
|
|
401
|
-
getAggregatedCosts(): ReturnType<typeof StateFulAxAgentUsage.getAggregatedCosts> {
|
|
402
|
-
return StateFulAxAgentUsage.getAggregatedCosts(this.state);
|
|
403
|
-
}
|
|
404
497
|
|
|
405
498
|
/**
|
|
406
499
|
* Resets all cost tracking for the crew
|
|
407
500
|
*/
|
|
408
501
|
resetCosts(): void {
|
|
409
|
-
|
|
502
|
+
// Reset AxAgent built-in usage and our metrics registry
|
|
503
|
+
if (this.agents) {
|
|
504
|
+
for (const [, agent] of this.agents) {
|
|
505
|
+
try { (agent as any).resetUsage?.(); } catch {}
|
|
506
|
+
try { (agent as any).resetMetrics?.(); } catch {}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
MetricsRegistry.reset({ crewId: this.crewId });
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
// Metrics API
|
|
513
|
+
getCrewMetrics() {
|
|
514
|
+
return MetricsRegistry.snapshotCrew(this.crewId);
|
|
515
|
+
}
|
|
516
|
+
resetCrewMetrics(): void {
|
|
517
|
+
MetricsRegistry.reset({ crewId: this.crewId });
|
|
410
518
|
}
|
|
411
519
|
}
|
|
412
520
|
|
|
413
|
-
export { AxCrew };
|
|
521
|
+
export { AxCrew };
|
|
522
|
+
export type { StatefulAxAgent };
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import type { LabelKeys, MetricsSnapshot, TokenUsage } from './types.js';
|
|
2
|
+
import Big from 'big.js';
|
|
3
|
+
|
|
4
|
+
type Key = string;
|
|
5
|
+
|
|
6
|
+
type Counters = {
|
|
7
|
+
requests: number;
|
|
8
|
+
errors: number;
|
|
9
|
+
streaming: number;
|
|
10
|
+
durationMsSum: number;
|
|
11
|
+
durationCount: number;
|
|
12
|
+
inputTokens: number;
|
|
13
|
+
outputTokens: number;
|
|
14
|
+
estimatedCostUSD: number;
|
|
15
|
+
functionCalls: number;
|
|
16
|
+
functionLatencyMs: number;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const store = new Map<Key, Counters>();
|
|
20
|
+
|
|
21
|
+
function keyOf(labels: LabelKeys): Key {
|
|
22
|
+
const { crewId, agent = '', provider = '', model = '' } = labels;
|
|
23
|
+
return [crewId, agent, provider, model].join('|');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function getOrInit(labels: LabelKeys): Counters {
|
|
27
|
+
const k = keyOf(labels);
|
|
28
|
+
let c = store.get(k);
|
|
29
|
+
if (!c) {
|
|
30
|
+
c = {
|
|
31
|
+
requests: 0,
|
|
32
|
+
errors: 0,
|
|
33
|
+
streaming: 0,
|
|
34
|
+
durationMsSum: 0,
|
|
35
|
+
durationCount: 0,
|
|
36
|
+
inputTokens: 0,
|
|
37
|
+
outputTokens: 0,
|
|
38
|
+
estimatedCostUSD: 0,
|
|
39
|
+
functionCalls: 0,
|
|
40
|
+
functionLatencyMs: 0,
|
|
41
|
+
};
|
|
42
|
+
store.set(k, c);
|
|
43
|
+
}
|
|
44
|
+
return c;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function recordRequest(labels: LabelKeys, streaming: boolean, durationMs: number) {
|
|
48
|
+
const c = getOrInit(labels);
|
|
49
|
+
c.requests += 1;
|
|
50
|
+
if (streaming) c.streaming += 1;
|
|
51
|
+
c.durationMsSum += durationMs;
|
|
52
|
+
c.durationCount += 1;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function recordError(labels: LabelKeys) {
|
|
56
|
+
const c = getOrInit(labels);
|
|
57
|
+
c.errors += 1;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function recordTokens(labels: LabelKeys, usage: TokenUsage) {
|
|
61
|
+
const c = getOrInit(labels);
|
|
62
|
+
c.inputTokens += usage.promptTokens || 0;
|
|
63
|
+
c.outputTokens += usage.completionTokens || 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function recordEstimatedCost(labels: LabelKeys, usd: number) {
|
|
67
|
+
const c = getOrInit(labels);
|
|
68
|
+
const current = new Big(c.estimatedCostUSD || 0);
|
|
69
|
+
const addition = new Big(usd || 0);
|
|
70
|
+
c.estimatedCostUSD = Number(current.plus(addition));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function recordFunctionCall(labels: LabelKeys, latencyMs: number) {
|
|
74
|
+
const c = getOrInit(labels);
|
|
75
|
+
c.functionCalls += 1;
|
|
76
|
+
c.functionLatencyMs += latencyMs || 0;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function snapshot(labels: LabelKeys): MetricsSnapshot {
|
|
80
|
+
const c = getOrInit(labels);
|
|
81
|
+
const totalTokens = c.inputTokens + c.outputTokens;
|
|
82
|
+
return {
|
|
83
|
+
provider: labels.provider,
|
|
84
|
+
model: labels.model,
|
|
85
|
+
requests: {
|
|
86
|
+
totalRequests: c.requests,
|
|
87
|
+
totalErrors: c.errors,
|
|
88
|
+
errorRate: c.requests > 0 ? c.errors / c.requests : 0,
|
|
89
|
+
totalStreamingRequests: c.streaming,
|
|
90
|
+
durationMsSum: c.durationMsSum,
|
|
91
|
+
durationCount: c.durationCount,
|
|
92
|
+
},
|
|
93
|
+
tokens: {
|
|
94
|
+
promptTokens: c.inputTokens,
|
|
95
|
+
completionTokens: c.outputTokens,
|
|
96
|
+
totalTokens,
|
|
97
|
+
},
|
|
98
|
+
estimatedCostUSD: Number(new Big(c.estimatedCostUSD || 0).round(5)),
|
|
99
|
+
functions: {
|
|
100
|
+
totalFunctionCalls: c.functionCalls,
|
|
101
|
+
totalFunctionLatencyMs: c.functionLatencyMs,
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function reset(labels?: LabelKeys) {
|
|
107
|
+
if (!labels) {
|
|
108
|
+
store.clear();
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const k = keyOf(labels);
|
|
112
|
+
store.delete(k);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function snapshotCrew(crewId: string): MetricsSnapshot {
|
|
116
|
+
const empty: Counters = {
|
|
117
|
+
requests: 0,
|
|
118
|
+
errors: 0,
|
|
119
|
+
streaming: 0,
|
|
120
|
+
durationMsSum: 0,
|
|
121
|
+
durationCount: 0,
|
|
122
|
+
inputTokens: 0,
|
|
123
|
+
outputTokens: 0,
|
|
124
|
+
estimatedCostUSD: 0,
|
|
125
|
+
functionCalls: 0,
|
|
126
|
+
functionLatencyMs: 0,
|
|
127
|
+
};
|
|
128
|
+
const agg = Array.from(store.entries()).reduce((acc, [k, v]) => {
|
|
129
|
+
if (k.startsWith(crewId + '|')) {
|
|
130
|
+
acc.requests += v.requests;
|
|
131
|
+
acc.errors += v.errors;
|
|
132
|
+
acc.streaming += v.streaming;
|
|
133
|
+
acc.durationMsSum += v.durationMsSum;
|
|
134
|
+
acc.durationCount += v.durationCount;
|
|
135
|
+
acc.inputTokens += v.inputTokens;
|
|
136
|
+
acc.outputTokens += v.outputTokens;
|
|
137
|
+
acc.estimatedCostUSD = Number(new Big(acc.estimatedCostUSD || 0).plus(v.estimatedCostUSD || 0));
|
|
138
|
+
acc.functionCalls += v.functionCalls;
|
|
139
|
+
acc.functionLatencyMs += v.functionLatencyMs;
|
|
140
|
+
}
|
|
141
|
+
return acc;
|
|
142
|
+
}, { ...empty });
|
|
143
|
+
|
|
144
|
+
const totalTokens = agg.inputTokens + agg.outputTokens;
|
|
145
|
+
return {
|
|
146
|
+
requests: {
|
|
147
|
+
totalRequests: agg.requests,
|
|
148
|
+
totalErrors: agg.errors,
|
|
149
|
+
errorRate: agg.requests > 0 ? agg.errors / agg.requests : 0,
|
|
150
|
+
totalStreamingRequests: agg.streaming,
|
|
151
|
+
durationMsSum: agg.durationMsSum,
|
|
152
|
+
durationCount: agg.durationCount,
|
|
153
|
+
},
|
|
154
|
+
tokens: {
|
|
155
|
+
promptTokens: agg.inputTokens,
|
|
156
|
+
completionTokens: agg.outputTokens,
|
|
157
|
+
totalTokens,
|
|
158
|
+
},
|
|
159
|
+
estimatedCostUSD: Number(new Big(agg.estimatedCostUSD || 0).round(5)),
|
|
160
|
+
functions: {
|
|
161
|
+
totalFunctionCalls: agg.functionCalls,
|
|
162
|
+
totalFunctionLatencyMs: agg.functionLatencyMs,
|
|
163
|
+
},
|
|
164
|
+
} as MetricsSnapshot;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export interface TokenUsage {
|
|
2
|
+
promptTokens: number;
|
|
3
|
+
completionTokens: number;
|
|
4
|
+
totalTokens?: number;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export interface CostSnapshot {
|
|
8
|
+
usdTotal: number;
|
|
9
|
+
tokenUsage: TokenUsage;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface RequestStats {
|
|
13
|
+
totalRequests: number;
|
|
14
|
+
totalErrors: number;
|
|
15
|
+
errorRate: number;
|
|
16
|
+
totalStreamingRequests: number;
|
|
17
|
+
durationMsSum: number;
|
|
18
|
+
durationCount: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface FunctionStats {
|
|
22
|
+
totalFunctionCalls: number;
|
|
23
|
+
totalFunctionLatencyMs: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface MetricsSnapshot {
|
|
27
|
+
provider?: string;
|
|
28
|
+
model?: string;
|
|
29
|
+
requests: RequestStats;
|
|
30
|
+
tokens: TokenUsage;
|
|
31
|
+
estimatedCostUSD: number;
|
|
32
|
+
functions: FunctionStats;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface LabelKeys {
|
|
36
|
+
crewId: string;
|
|
37
|
+
agent?: string;
|
|
38
|
+
provider?: string;
|
|
39
|
+
model?: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface BudgetConfig {
|
|
43
|
+
maxTokens?: number;
|
|
44
|
+
maxCost?: number;
|
|
45
|
+
costPerModel?: Record<string, number>; // USD per 1K tokens override
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
package/src/types.ts
CHANGED
|
@@ -6,7 +6,23 @@ import type {
|
|
|
6
6
|
AxProgramForwardOptions
|
|
7
7
|
} from '@ax-llm/ax';
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
// Canonical provider slugs supported by ai() factory
|
|
10
|
+
export type Provider =
|
|
11
|
+
// Canonical slugs per docs
|
|
12
|
+
| 'openai'
|
|
13
|
+
| 'anthropic'
|
|
14
|
+
| 'google-gemini'
|
|
15
|
+
| 'mistral'
|
|
16
|
+
| 'groq'
|
|
17
|
+
| 'cohere'
|
|
18
|
+
| 'together'
|
|
19
|
+
| 'deepseek'
|
|
20
|
+
| 'ollama'
|
|
21
|
+
| 'huggingface'
|
|
22
|
+
| 'openrouter'
|
|
23
|
+
| 'azure-openai'
|
|
24
|
+
| 'reka'
|
|
25
|
+
| 'x-grok'
|
|
10
26
|
|
|
11
27
|
/**
|
|
12
28
|
* A state instance that is shared between agents.
|
|
@@ -132,7 +148,7 @@ interface MCPHTTPSSETransportConfig {
|
|
|
132
148
|
* @property {string} mcpEndpoint - The HTTP endpoint URL for the MCP server.
|
|
133
149
|
* @property {AxMCPStreamableHTTPTransportOptions} options - Optional transport options.
|
|
134
150
|
*/
|
|
135
|
-
interface
|
|
151
|
+
interface MCPStreamableHTTPTransportConfig {
|
|
136
152
|
mcpEndpoint: string
|
|
137
153
|
options?: AxMCPStreamableHTTPTransportOptions
|
|
138
154
|
}
|
|
@@ -142,7 +158,7 @@ interface MCPStreambleHTTPTransportConfig {
|
|
|
142
158
|
*
|
|
143
159
|
* @property {MCPStdioTransportConfig | MCPHTTPSSETransportConfig | MCPStreambleHTTPTransportConfig} config - The config for the MCP server. Config can be either stdio, http-sse, or streamable http transport.
|
|
144
160
|
*/
|
|
145
|
-
type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig |
|
|
161
|
+
type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig | MCPStreamableHTTPTransportConfig
|
|
146
162
|
|
|
147
163
|
/**
|
|
148
164
|
* The configuration for an agent.
|
|
@@ -155,7 +171,7 @@ type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig |
|
|
|
155
171
|
* @property {AxModelConfig & { model: string }} ai - The AI model configuration to be passed to the agent.
|
|
156
172
|
* @property {boolean} debug - Whether to enable debug mode.
|
|
157
173
|
* @property {string} apiURL - Set this if you are using a custom API URL e.g. ollama on localhost.
|
|
158
|
-
* @property {Partial<AxProgramForwardOptions>} options - Agent options including thinkingTokenBudget, showThoughts, etc.
|
|
174
|
+
* @property {Partial<AxProgramForwardOptions<any>> & Record<string, any>} options - Agent options including thinkingTokenBudget, showThoughts, etc. Also allows arbitrary provider-specific keys.
|
|
159
175
|
* @property {string[]} functions - Function names to be used by the agent.
|
|
160
176
|
* @property {string[]} agents - Sub-agent available to the agent.
|
|
161
177
|
* @property {Record<string, any>[]} examples - DSPy examples for the agent to learn from.
|
|
@@ -170,7 +186,7 @@ interface AgentConfig {
|
|
|
170
186
|
ai: AxModelConfig & { model: string };
|
|
171
187
|
debug?: boolean;
|
|
172
188
|
apiURL?: string;
|
|
173
|
-
options?: Partial<AxProgramForwardOptions>;
|
|
189
|
+
options?: Partial<AxProgramForwardOptions<any>> & Record<string, any>;
|
|
174
190
|
functions?: string[];
|
|
175
191
|
agents?: string[];
|
|
176
192
|
examples?: Array<Record<string, any>>;
|
|
@@ -190,10 +206,10 @@ export {
|
|
|
190
206
|
type FunctionRegistryType,
|
|
191
207
|
type MCPStdioTransportConfig,
|
|
192
208
|
type MCPHTTPSSETransportConfig,
|
|
193
|
-
type
|
|
209
|
+
type MCPStreamableHTTPTransportConfig,
|
|
194
210
|
type MCPTransportConfig,
|
|
195
211
|
type ModelUsage,
|
|
196
212
|
type ModelInfo,
|
|
197
213
|
type UsageCost,
|
|
198
214
|
type AggregatedCosts
|
|
199
|
-
}
|
|
215
|
+
}
|