@amitdeshmukh/ax-crew 3.11.3 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +83 -82
- package/dist/agents/agentConfig.d.ts +7 -6
- package/dist/agents/agentConfig.js +40 -32
- package/dist/agents/index.d.ts +14 -10
- package/dist/agents/index.js +155 -40
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/metrics/index.d.ts +2 -0
- package/dist/metrics/index.js +2 -0
- package/dist/metrics/registry.d.ts +9 -0
- package/dist/metrics/registry.js +138 -0
- package/dist/metrics/types.d.ts +40 -0
- package/dist/metrics/types.js +1 -0
- package/dist/types.d.ts +16 -6
- package/examples/basic-researcher-writer.ts +18 -13
- package/examples/mcp-agent.ts +14 -9
- package/examples/solve-math-problem.ts +7 -5
- package/examples/streaming.ts +7 -6
- package/package.json +5 -3
- package/src/agents/agentConfig.ts +45 -38
- package/src/agents/index.ts +171 -59
- package/src/index.ts +2 -0
- package/src/metrics/index.ts +4 -0
- package/src/metrics/registry.ts +167 -0
- package/src/metrics/types.ts +48 -0
- package/src/types.ts +33 -7
- package/tests/AxCrew.test.ts +22 -18
package/dist/agents/index.js
CHANGED
|
@@ -2,12 +2,20 @@ import { v4 as uuidv4 } from "uuid";
|
|
|
2
2
|
import { AxAgent } from "@ax-llm/ax";
|
|
3
3
|
import { createState } from "../state/index.js";
|
|
4
4
|
import { parseCrewConfig, parseAgentConfig } from "./agentConfig.js";
|
|
5
|
-
import {
|
|
5
|
+
import { MetricsRegistry } from "../metrics/index.js";
|
|
6
6
|
// Extend the AxAgent class from ax-llm
|
|
7
7
|
class StatefulAxAgent extends AxAgent {
|
|
8
8
|
state;
|
|
9
9
|
axai;
|
|
10
10
|
agentName;
|
|
11
|
+
costTracker;
|
|
12
|
+
lastRecordedCostUSD = 0;
|
|
13
|
+
isAxAIService(obj) {
|
|
14
|
+
return !!obj && typeof obj.getName === 'function' && typeof obj.chat === 'function';
|
|
15
|
+
}
|
|
16
|
+
isAxAIInstance(obj) {
|
|
17
|
+
return !!obj && typeof obj === 'object' && ('defaults' in obj || 'modelInfo' in obj);
|
|
18
|
+
}
|
|
11
19
|
constructor(ai, options, state) {
|
|
12
20
|
const { examples, ...restOptions } = options;
|
|
13
21
|
const formattedOptions = {
|
|
@@ -26,9 +34,12 @@ class StatefulAxAgent extends AxAgent {
|
|
|
26
34
|
// Implementation
|
|
27
35
|
async forward(first, second, third) {
|
|
28
36
|
let result;
|
|
37
|
+
const start = performance.now();
|
|
38
|
+
const crewId = this.state?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
39
|
+
const labels = { crewId, agent: this.agentName };
|
|
29
40
|
// Track costs regardless of whether it's a direct or sub-agent call
|
|
30
41
|
// This ensures we capture multiple legitimate calls to the same agent
|
|
31
|
-
if (
|
|
42
|
+
if (this.isAxAIService(first)) {
|
|
32
43
|
// Sub-agent case (called with AI service)
|
|
33
44
|
result = await super.forward(this.axai, second, third);
|
|
34
45
|
}
|
|
@@ -36,17 +47,51 @@ class StatefulAxAgent extends AxAgent {
|
|
|
36
47
|
// Direct call case
|
|
37
48
|
result = await super.forward(this.axai, first, second);
|
|
38
49
|
}
|
|
39
|
-
// Track costs after the call
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
50
|
+
// Track metrics and costs after the call using built-in usage
|
|
51
|
+
const durationMs = performance.now() - start;
|
|
52
|
+
MetricsRegistry.recordRequest(labels, false, durationMs);
|
|
53
|
+
// Always record tokens from built-in usage array if present
|
|
54
|
+
const builtIn = this.getUsage?.();
|
|
55
|
+
if (Array.isArray(builtIn)) {
|
|
56
|
+
const totals = builtIn.reduce((acc, u) => {
|
|
57
|
+
const pt = u.tokens?.promptTokens ?? u.promptTokens ?? 0;
|
|
58
|
+
const ct = u.tokens?.completionTokens ?? u.completionTokens ?? 0;
|
|
59
|
+
acc.promptTokens += typeof pt === 'number' ? pt : 0;
|
|
60
|
+
acc.completionTokens += typeof ct === 'number' ? ct : 0;
|
|
61
|
+
// also aggregate per-model to feed Ax tracker
|
|
62
|
+
const model = u.model || this.axai?.getLastUsedChatModel?.() || this.axai?.defaults?.model;
|
|
63
|
+
if (model) {
|
|
64
|
+
acc.byModel[model] = (acc.byModel[model] || 0) + (pt + ct);
|
|
65
|
+
}
|
|
66
|
+
return acc;
|
|
67
|
+
}, { promptTokens: 0, completionTokens: 0, byModel: {} });
|
|
68
|
+
MetricsRegistry.recordTokens(labels, {
|
|
69
|
+
promptTokens: totals.promptTokens,
|
|
70
|
+
completionTokens: totals.completionTokens,
|
|
71
|
+
totalTokens: totals.promptTokens + totals.completionTokens,
|
|
72
|
+
});
|
|
73
|
+
// Feed Ax's cost tracker with token totals per model; Ax owns pricing
|
|
74
|
+
const costTracker = this.costTracker;
|
|
75
|
+
try {
|
|
76
|
+
for (const [m, count] of Object.entries(totals.byModel)) {
|
|
77
|
+
costTracker?.trackTokens?.(count, m);
|
|
78
|
+
}
|
|
79
|
+
const totalUSD = Number(costTracker?.getCurrentCost?.() ?? 0);
|
|
80
|
+
if (!Number.isNaN(totalUSD) && totalUSD > 0) {
|
|
81
|
+
MetricsRegistry.recordEstimatedCost(labels, totalUSD);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch { }
|
|
43
85
|
}
|
|
44
86
|
return result;
|
|
45
87
|
}
|
|
46
88
|
// Implementation
|
|
47
89
|
streamingForward(first, second, third) {
|
|
90
|
+
const start = performance.now();
|
|
91
|
+
const crewId = this.state?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
92
|
+
const labels = { crewId, agent: this.agentName };
|
|
48
93
|
let streamingResult;
|
|
49
|
-
if (
|
|
94
|
+
if (this.isAxAIService(first)) {
|
|
50
95
|
streamingResult = super.streamingForward(this.axai, second, third);
|
|
51
96
|
}
|
|
52
97
|
else {
|
|
@@ -60,34 +105,64 @@ class StatefulAxAgent extends AxAgent {
|
|
|
60
105
|
}
|
|
61
106
|
}
|
|
62
107
|
finally {
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
108
|
+
const durationMs = performance.now() - start;
|
|
109
|
+
MetricsRegistry.recordRequest(labels, true, durationMs);
|
|
110
|
+
// Record tokens from built-in usage array if present
|
|
111
|
+
const builtIn = this.getUsage?.();
|
|
112
|
+
if (Array.isArray(builtIn)) {
|
|
113
|
+
const totals = builtIn.reduce((acc, u) => {
|
|
114
|
+
const pt = u.tokens?.promptTokens ?? u.promptTokens ?? 0;
|
|
115
|
+
const ct = u.tokens?.completionTokens ?? u.completionTokens ?? 0;
|
|
116
|
+
acc.promptTokens += typeof pt === 'number' ? pt : 0;
|
|
117
|
+
acc.completionTokens += typeof ct === 'number' ? ct : 0;
|
|
118
|
+
const model = u.model || this.axai?.getLastUsedChatModel?.() || this.axai?.defaults?.model;
|
|
119
|
+
if (model) {
|
|
120
|
+
acc.byModel[model] = (acc.byModel[model] || 0) + (pt + ct);
|
|
121
|
+
}
|
|
122
|
+
return acc;
|
|
123
|
+
}, { promptTokens: 0, completionTokens: 0, byModel: {} });
|
|
124
|
+
MetricsRegistry.recordTokens(labels, {
|
|
125
|
+
promptTokens: totals.promptTokens,
|
|
126
|
+
completionTokens: totals.completionTokens,
|
|
127
|
+
totalTokens: totals.promptTokens + totals.completionTokens,
|
|
128
|
+
});
|
|
129
|
+
const costTracker = this.costTracker;
|
|
130
|
+
try {
|
|
131
|
+
for (const [m, count] of Object.entries(totals.byModel)) {
|
|
132
|
+
costTracker?.trackTokens?.(count, m);
|
|
133
|
+
}
|
|
134
|
+
const totalUSD = Number(costTracker?.getCurrentCost?.() ?? 0);
|
|
135
|
+
if (!Number.isNaN(totalUSD) && totalUSD > 0) {
|
|
136
|
+
MetricsRegistry.recordEstimatedCost(labels, totalUSD);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
catch { }
|
|
140
|
+
}
|
|
141
|
+
// Record estimated cost (USD) via attached tracker if available
|
|
142
|
+
const costTracker = this.costTracker;
|
|
143
|
+
try {
|
|
144
|
+
const totalUSD = Number(costTracker?.getCurrentCost?.() ?? 0);
|
|
145
|
+
if (!Number.isNaN(totalUSD) && totalUSD > 0) {
|
|
146
|
+
MetricsRegistry.recordEstimatedCost(labels, totalUSD);
|
|
147
|
+
}
|
|
66
148
|
}
|
|
149
|
+
catch { }
|
|
67
150
|
}
|
|
68
151
|
}).bind(this)();
|
|
69
152
|
return wrappedGenerator;
|
|
70
153
|
}
|
|
71
|
-
//
|
|
72
|
-
getLastUsageCost() {
|
|
73
|
-
const { modelUsage, modelInfo, defaults } = this.axai;
|
|
74
|
-
// Check if all required properties exist
|
|
75
|
-
if (!modelUsage?.promptTokens || !modelUsage?.completionTokens) {
|
|
76
|
-
return null;
|
|
77
|
-
}
|
|
78
|
-
if (!modelInfo || !defaults?.model) {
|
|
79
|
-
return null;
|
|
80
|
-
}
|
|
81
|
-
const currentModelInfo = modelInfo.find((m) => m.name === defaults.model);
|
|
82
|
-
if (!currentModelInfo?.promptTokenCostPer1M || !currentModelInfo?.completionTokenCostPer1M) {
|
|
83
|
-
return null;
|
|
84
|
-
}
|
|
85
|
-
return StateFulAxAgentUsage.calculateCost(modelUsage, currentModelInfo);
|
|
86
|
-
}
|
|
154
|
+
// Legacy cost API removed: rely on Ax trackers for cost reporting
|
|
155
|
+
getLastUsageCost() { return null; }
|
|
87
156
|
// Get the accumulated costs for all runs of this agent
|
|
88
|
-
getAccumulatedCosts() {
|
|
89
|
-
|
|
90
|
-
|
|
157
|
+
getAccumulatedCosts() { return null; }
|
|
158
|
+
// Metrics API for this agent
|
|
159
|
+
getMetrics() {
|
|
160
|
+
const crewId = this.state?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
161
|
+
return MetricsRegistry.snapshot({ crewId, agent: this.agentName });
|
|
162
|
+
}
|
|
163
|
+
resetMetrics() {
|
|
164
|
+
const crewId = this.state?.crewId || (this.state.get?.('crewId')) || 'default';
|
|
165
|
+
MetricsRegistry.reset({ crewId, agent: this.agentName });
|
|
91
166
|
}
|
|
92
167
|
}
|
|
93
168
|
/**
|
|
@@ -121,6 +196,8 @@ class AxCrew {
|
|
|
121
196
|
this.crewId = crewId;
|
|
122
197
|
this.agents = new Map();
|
|
123
198
|
this.state = createState(crewId);
|
|
199
|
+
// Make crewId discoverable to metrics
|
|
200
|
+
this.state.set('crewId', crewId);
|
|
124
201
|
}
|
|
125
202
|
/**
|
|
126
203
|
* Factory function for creating an agent.
|
|
@@ -132,7 +209,7 @@ class AxCrew {
|
|
|
132
209
|
try {
|
|
133
210
|
const agentConfig = await parseAgentConfig(agentName, this.crewConfig, this.functionsRegistry, this.state);
|
|
134
211
|
// Destructure with type assertion
|
|
135
|
-
const { ai, name, description, signature, functions, subAgentNames, examples } = agentConfig;
|
|
212
|
+
const { ai, name, description, signature, functions, subAgentNames, examples, tracker } = agentConfig;
|
|
136
213
|
// Get subagents for the AI agent
|
|
137
214
|
const subAgents = subAgentNames.map((subAgentName) => {
|
|
138
215
|
if (!this.agents?.get(subAgentName)) {
|
|
@@ -140,15 +217,40 @@ class AxCrew {
|
|
|
140
217
|
}
|
|
141
218
|
return this.agents?.get(subAgentName);
|
|
142
219
|
});
|
|
220
|
+
// Dedupe sub-agents by name (defensive)
|
|
221
|
+
const subAgentSet = new Map();
|
|
222
|
+
for (const sa of subAgents.filter((agent) => agent !== undefined)) {
|
|
223
|
+
const n = sa?.agentName ?? sa?.name ?? '';
|
|
224
|
+
if (!subAgentSet.has(n))
|
|
225
|
+
subAgentSet.set(n, sa);
|
|
226
|
+
}
|
|
227
|
+
const uniqueSubAgents = Array.from(subAgentSet.values());
|
|
228
|
+
// Dedupe functions by name and avoid collision with sub-agent names
|
|
229
|
+
const subAgentNameSet = new Set(uniqueSubAgents.map((sa) => sa?.agentName ?? sa?.name).filter(Boolean));
|
|
230
|
+
const uniqueFunctions = [];
|
|
231
|
+
const seenFn = new Set();
|
|
232
|
+
for (const fn of functions.filter((fn) => fn !== undefined)) {
|
|
233
|
+
const fnName = fn.name;
|
|
234
|
+
if (subAgentNameSet.has(fnName)) {
|
|
235
|
+
// Skip function that collides with a sub-agent name
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (!seenFn.has(fnName)) {
|
|
239
|
+
seenFn.add(fnName);
|
|
240
|
+
uniqueFunctions.push(fn);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
143
243
|
// Create an instance of StatefulAxAgent
|
|
144
244
|
const agent = new StatefulAxAgent(ai, {
|
|
145
245
|
name,
|
|
146
246
|
description,
|
|
247
|
+
definition: agentConfig.definition,
|
|
147
248
|
signature,
|
|
148
|
-
functions:
|
|
149
|
-
agents:
|
|
249
|
+
functions: uniqueFunctions,
|
|
250
|
+
agents: uniqueSubAgents,
|
|
150
251
|
examples,
|
|
151
252
|
}, this.state);
|
|
253
|
+
agent.costTracker = tracker;
|
|
152
254
|
return agent;
|
|
153
255
|
}
|
|
154
256
|
catch (error) {
|
|
@@ -273,18 +375,31 @@ class AxCrew {
|
|
|
273
375
|
this.agents = null;
|
|
274
376
|
this.state.reset();
|
|
275
377
|
}
|
|
276
|
-
/**
|
|
277
|
-
* Gets aggregated costs for all agents in the crew
|
|
278
|
-
* @returns Aggregated cost information for all agents
|
|
279
|
-
*/
|
|
280
|
-
getAggregatedCosts() {
|
|
281
|
-
return StateFulAxAgentUsage.getAggregatedCosts(this.state);
|
|
282
|
-
}
|
|
283
378
|
/**
|
|
284
379
|
* Resets all cost tracking for the crew
|
|
285
380
|
*/
|
|
286
381
|
resetCosts() {
|
|
287
|
-
|
|
382
|
+
// Reset AxAgent built-in usage and our metrics registry
|
|
383
|
+
if (this.agents) {
|
|
384
|
+
for (const [, agent] of this.agents) {
|
|
385
|
+
try {
|
|
386
|
+
agent.resetUsage?.();
|
|
387
|
+
}
|
|
388
|
+
catch { }
|
|
389
|
+
try {
|
|
390
|
+
agent.resetMetrics?.();
|
|
391
|
+
}
|
|
392
|
+
catch { }
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
MetricsRegistry.reset({ crewId: this.crewId });
|
|
396
|
+
}
|
|
397
|
+
// Metrics API
|
|
398
|
+
getCrewMetrics() {
|
|
399
|
+
return MetricsRegistry.snapshotCrew(this.crewId);
|
|
400
|
+
}
|
|
401
|
+
resetCrewMetrics() {
|
|
402
|
+
MetricsRegistry.reset({ crewId: this.crewId });
|
|
288
403
|
}
|
|
289
404
|
}
|
|
290
405
|
export { AxCrew };
|
package/dist/index.d.ts
CHANGED
|
@@ -2,6 +2,8 @@ import { AxCrew } from './agents/index.js';
|
|
|
2
2
|
import { AxCrewFunctions } from './functions/index.js';
|
|
3
3
|
import type { CrewConfigInput, AgentConfig } from './types.js';
|
|
4
4
|
import type { UsageCost, AggregatedMetrics, AggregatedCosts, StateInstance, FunctionRegistryType } from './types.js';
|
|
5
|
+
export * from './metrics/index.js';
|
|
6
|
+
export { MetricsRegistry } from './metrics/index.js';
|
|
5
7
|
/**
|
|
6
8
|
* The configuration for an AxCrew.
|
|
7
9
|
*
|
package/dist/index.js
CHANGED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { LabelKeys, MetricsSnapshot, TokenUsage } from './types.js';
|
|
2
|
+
export declare function recordRequest(labels: LabelKeys, streaming: boolean, durationMs: number): void;
|
|
3
|
+
export declare function recordError(labels: LabelKeys): void;
|
|
4
|
+
export declare function recordTokens(labels: LabelKeys, usage: TokenUsage): void;
|
|
5
|
+
export declare function recordEstimatedCost(labels: LabelKeys, usd: number): void;
|
|
6
|
+
export declare function recordFunctionCall(labels: LabelKeys, latencyMs: number): void;
|
|
7
|
+
export declare function snapshot(labels: LabelKeys): MetricsSnapshot;
|
|
8
|
+
export declare function reset(labels?: LabelKeys): void;
|
|
9
|
+
export declare function snapshotCrew(crewId: string): MetricsSnapshot;
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import Big from 'big.js';
|
|
2
|
+
const store = new Map();
|
|
3
|
+
function keyOf(labels) {
|
|
4
|
+
const { crewId, agent = '', provider = '', model = '' } = labels;
|
|
5
|
+
return [crewId, agent, provider, model].join('|');
|
|
6
|
+
}
|
|
7
|
+
function getOrInit(labels) {
|
|
8
|
+
const k = keyOf(labels);
|
|
9
|
+
let c = store.get(k);
|
|
10
|
+
if (!c) {
|
|
11
|
+
c = {
|
|
12
|
+
requests: 0,
|
|
13
|
+
errors: 0,
|
|
14
|
+
streaming: 0,
|
|
15
|
+
durationMsSum: 0,
|
|
16
|
+
durationCount: 0,
|
|
17
|
+
inputTokens: 0,
|
|
18
|
+
outputTokens: 0,
|
|
19
|
+
estimatedCostUSD: 0,
|
|
20
|
+
functionCalls: 0,
|
|
21
|
+
functionLatencyMs: 0,
|
|
22
|
+
};
|
|
23
|
+
store.set(k, c);
|
|
24
|
+
}
|
|
25
|
+
return c;
|
|
26
|
+
}
|
|
27
|
+
export function recordRequest(labels, streaming, durationMs) {
|
|
28
|
+
const c = getOrInit(labels);
|
|
29
|
+
c.requests += 1;
|
|
30
|
+
if (streaming)
|
|
31
|
+
c.streaming += 1;
|
|
32
|
+
c.durationMsSum += durationMs;
|
|
33
|
+
c.durationCount += 1;
|
|
34
|
+
}
|
|
35
|
+
export function recordError(labels) {
|
|
36
|
+
const c = getOrInit(labels);
|
|
37
|
+
c.errors += 1;
|
|
38
|
+
}
|
|
39
|
+
export function recordTokens(labels, usage) {
|
|
40
|
+
const c = getOrInit(labels);
|
|
41
|
+
c.inputTokens += usage.promptTokens || 0;
|
|
42
|
+
c.outputTokens += usage.completionTokens || 0;
|
|
43
|
+
}
|
|
44
|
+
export function recordEstimatedCost(labels, usd) {
|
|
45
|
+
const c = getOrInit(labels);
|
|
46
|
+
const current = new Big(c.estimatedCostUSD || 0);
|
|
47
|
+
const addition = new Big(usd || 0);
|
|
48
|
+
c.estimatedCostUSD = Number(current.plus(addition));
|
|
49
|
+
}
|
|
50
|
+
export function recordFunctionCall(labels, latencyMs) {
|
|
51
|
+
const c = getOrInit(labels);
|
|
52
|
+
c.functionCalls += 1;
|
|
53
|
+
c.functionLatencyMs += latencyMs || 0;
|
|
54
|
+
}
|
|
55
|
+
export function snapshot(labels) {
|
|
56
|
+
const c = getOrInit(labels);
|
|
57
|
+
const totalTokens = c.inputTokens + c.outputTokens;
|
|
58
|
+
return {
|
|
59
|
+
provider: labels.provider,
|
|
60
|
+
model: labels.model,
|
|
61
|
+
requests: {
|
|
62
|
+
totalRequests: c.requests,
|
|
63
|
+
totalErrors: c.errors,
|
|
64
|
+
errorRate: c.requests > 0 ? c.errors / c.requests : 0,
|
|
65
|
+
totalStreamingRequests: c.streaming,
|
|
66
|
+
durationMsSum: c.durationMsSum,
|
|
67
|
+
durationCount: c.durationCount,
|
|
68
|
+
},
|
|
69
|
+
tokens: {
|
|
70
|
+
promptTokens: c.inputTokens,
|
|
71
|
+
completionTokens: c.outputTokens,
|
|
72
|
+
totalTokens,
|
|
73
|
+
},
|
|
74
|
+
estimatedCostUSD: Number(new Big(c.estimatedCostUSD || 0).round(5)),
|
|
75
|
+
functions: {
|
|
76
|
+
totalFunctionCalls: c.functionCalls,
|
|
77
|
+
totalFunctionLatencyMs: c.functionLatencyMs,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
export function reset(labels) {
|
|
82
|
+
if (!labels) {
|
|
83
|
+
store.clear();
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const k = keyOf(labels);
|
|
87
|
+
store.delete(k);
|
|
88
|
+
}
|
|
89
|
+
export function snapshotCrew(crewId) {
|
|
90
|
+
const empty = {
|
|
91
|
+
requests: 0,
|
|
92
|
+
errors: 0,
|
|
93
|
+
streaming: 0,
|
|
94
|
+
durationMsSum: 0,
|
|
95
|
+
durationCount: 0,
|
|
96
|
+
inputTokens: 0,
|
|
97
|
+
outputTokens: 0,
|
|
98
|
+
estimatedCostUSD: 0,
|
|
99
|
+
functionCalls: 0,
|
|
100
|
+
functionLatencyMs: 0,
|
|
101
|
+
};
|
|
102
|
+
const agg = Array.from(store.entries()).reduce((acc, [k, v]) => {
|
|
103
|
+
if (k.startsWith(crewId + '|')) {
|
|
104
|
+
acc.requests += v.requests;
|
|
105
|
+
acc.errors += v.errors;
|
|
106
|
+
acc.streaming += v.streaming;
|
|
107
|
+
acc.durationMsSum += v.durationMsSum;
|
|
108
|
+
acc.durationCount += v.durationCount;
|
|
109
|
+
acc.inputTokens += v.inputTokens;
|
|
110
|
+
acc.outputTokens += v.outputTokens;
|
|
111
|
+
acc.estimatedCostUSD = Number(new Big(acc.estimatedCostUSD || 0).plus(v.estimatedCostUSD || 0));
|
|
112
|
+
acc.functionCalls += v.functionCalls;
|
|
113
|
+
acc.functionLatencyMs += v.functionLatencyMs;
|
|
114
|
+
}
|
|
115
|
+
return acc;
|
|
116
|
+
}, { ...empty });
|
|
117
|
+
const totalTokens = agg.inputTokens + agg.outputTokens;
|
|
118
|
+
return {
|
|
119
|
+
requests: {
|
|
120
|
+
totalRequests: agg.requests,
|
|
121
|
+
totalErrors: agg.errors,
|
|
122
|
+
errorRate: agg.requests > 0 ? agg.errors / agg.requests : 0,
|
|
123
|
+
totalStreamingRequests: agg.streaming,
|
|
124
|
+
durationMsSum: agg.durationMsSum,
|
|
125
|
+
durationCount: agg.durationCount,
|
|
126
|
+
},
|
|
127
|
+
tokens: {
|
|
128
|
+
promptTokens: agg.inputTokens,
|
|
129
|
+
completionTokens: agg.outputTokens,
|
|
130
|
+
totalTokens,
|
|
131
|
+
},
|
|
132
|
+
estimatedCostUSD: Number(new Big(agg.estimatedCostUSD || 0).round(5)),
|
|
133
|
+
functions: {
|
|
134
|
+
totalFunctionCalls: agg.functionCalls,
|
|
135
|
+
totalFunctionLatencyMs: agg.functionLatencyMs,
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export interface TokenUsage {
|
|
2
|
+
promptTokens: number;
|
|
3
|
+
completionTokens: number;
|
|
4
|
+
totalTokens?: number;
|
|
5
|
+
}
|
|
6
|
+
export interface CostSnapshot {
|
|
7
|
+
usdTotal: number;
|
|
8
|
+
tokenUsage: TokenUsage;
|
|
9
|
+
}
|
|
10
|
+
export interface RequestStats {
|
|
11
|
+
totalRequests: number;
|
|
12
|
+
totalErrors: number;
|
|
13
|
+
errorRate: number;
|
|
14
|
+
totalStreamingRequests: number;
|
|
15
|
+
durationMsSum: number;
|
|
16
|
+
durationCount: number;
|
|
17
|
+
}
|
|
18
|
+
export interface FunctionStats {
|
|
19
|
+
totalFunctionCalls: number;
|
|
20
|
+
totalFunctionLatencyMs: number;
|
|
21
|
+
}
|
|
22
|
+
export interface MetricsSnapshot {
|
|
23
|
+
provider?: string;
|
|
24
|
+
model?: string;
|
|
25
|
+
requests: RequestStats;
|
|
26
|
+
tokens: TokenUsage;
|
|
27
|
+
estimatedCostUSD: number;
|
|
28
|
+
functions: FunctionStats;
|
|
29
|
+
}
|
|
30
|
+
export interface LabelKeys {
|
|
31
|
+
crewId: string;
|
|
32
|
+
agent?: string;
|
|
33
|
+
provider?: string;
|
|
34
|
+
model?: string;
|
|
35
|
+
}
|
|
36
|
+
export interface BudgetConfig {
|
|
37
|
+
maxTokens?: number;
|
|
38
|
+
maxCost?: number;
|
|
39
|
+
costPerModel?: Record<string, number>;
|
|
40
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AxFunction, AxSignature, AxModelConfig, AxMCPStreamableHTTPTransportOptions, AxProgramForwardOptions } from '@ax-llm/ax';
|
|
2
|
-
|
|
2
|
+
export type Provider = 'openai' | 'anthropic' | 'google-gemini' | 'mistral' | 'groq' | 'cohere' | 'together' | 'deepseek' | 'ollama' | 'huggingface' | 'openrouter' | 'azure-openai' | 'reka' | 'x-grok';
|
|
3
3
|
/**
|
|
4
4
|
* A state instance that is shared between agents.
|
|
5
5
|
* This can be used to store data that becomes available to all agents and functions in an out-of-band manner.
|
|
@@ -117,7 +117,7 @@ interface MCPHTTPSSETransportConfig {
|
|
|
117
117
|
* @property {string} mcpEndpoint - The HTTP endpoint URL for the MCP server.
|
|
118
118
|
* @property {AxMCPStreamableHTTPTransportOptions} options - Optional transport options.
|
|
119
119
|
*/
|
|
120
|
-
interface
|
|
120
|
+
interface MCPStreamableHTTPTransportConfig {
|
|
121
121
|
mcpEndpoint: string;
|
|
122
122
|
options?: AxMCPStreamableHTTPTransportOptions;
|
|
123
123
|
}
|
|
@@ -126,7 +126,7 @@ interface MCPStreambleHTTPTransportConfig {
|
|
|
126
126
|
*
|
|
127
127
|
* @property {MCPStdioTransportConfig | MCPHTTPSSETransportConfig | MCPStreambleHTTPTransportConfig} config - The config for the MCP server. Config can be either stdio, http-sse, or streamable http transport.
|
|
128
128
|
*/
|
|
129
|
-
type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig |
|
|
129
|
+
type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig | MCPStreamableHTTPTransportConfig;
|
|
130
130
|
/**
|
|
131
131
|
* The configuration for an agent.
|
|
132
132
|
*
|
|
@@ -138,7 +138,7 @@ type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig |
|
|
|
138
138
|
* @property {AxModelConfig & { model: string }} ai - The AI model configuration to be passed to the agent.
|
|
139
139
|
* @property {boolean} debug - Whether to enable debug mode.
|
|
140
140
|
* @property {string} apiURL - Set this if you are using a custom API URL e.g. ollama on localhost.
|
|
141
|
-
* @property {Partial<AxProgramForwardOptions>} options - Agent options including thinkingTokenBudget, showThoughts, etc.
|
|
141
|
+
* @property {Partial<AxProgramForwardOptions<any>> & Record<string, any>} options - Agent options including thinkingTokenBudget, showThoughts, etc. Also allows arbitrary provider-specific keys.
|
|
142
142
|
* @property {string[]} functions - Function names to be used by the agent.
|
|
143
143
|
* @property {string[]} agents - Sub-agent available to the agent.
|
|
144
144
|
* @property {Record<string, any>[]} examples - DSPy examples for the agent to learn from.
|
|
@@ -147,6 +147,16 @@ type MCPTransportConfig = MCPStdioTransportConfig | MCPHTTPSSETransportConfig |
|
|
|
147
147
|
interface AgentConfig {
|
|
148
148
|
name: string;
|
|
149
149
|
description: string;
|
|
150
|
+
/**
|
|
151
|
+
* Optional detailed persona/program definition. If provided, becomes the system prompt.
|
|
152
|
+
* Must be at least 100 characters per Ax semantics.
|
|
153
|
+
*/
|
|
154
|
+
definition?: string;
|
|
155
|
+
/**
|
|
156
|
+
* Optional alias for definition for clarity. If provided and definition is omitted,
|
|
157
|
+
* this will be used as the program definition/system prompt.
|
|
158
|
+
*/
|
|
159
|
+
prompt?: string;
|
|
150
160
|
signature: string | AxSignature;
|
|
151
161
|
provider: Provider;
|
|
152
162
|
providerKeyName?: string;
|
|
@@ -155,7 +165,7 @@ interface AgentConfig {
|
|
|
155
165
|
};
|
|
156
166
|
debug?: boolean;
|
|
157
167
|
apiURL?: string;
|
|
158
|
-
options?: Partial<AxProgramForwardOptions>;
|
|
168
|
+
options?: Partial<AxProgramForwardOptions<any>> & Record<string, any>;
|
|
159
169
|
functions?: string[];
|
|
160
170
|
agents?: string[];
|
|
161
171
|
examples?: Array<Record<string, any>>;
|
|
@@ -167,4 +177,4 @@ interface AgentConfig {
|
|
|
167
177
|
type CrewConfigInput = string | {
|
|
168
178
|
crew: AgentConfig[];
|
|
169
179
|
};
|
|
170
|
-
export { type AgentConfig, type CrewConfigInput, type AggregatedMetrics, type StateInstance, type FunctionRegistryType, type MCPStdioTransportConfig, type MCPHTTPSSETransportConfig, type
|
|
180
|
+
export { type AgentConfig, type CrewConfigInput, type AggregatedMetrics, type StateInstance, type FunctionRegistryType, type MCPStdioTransportConfig, type MCPHTTPSSETransportConfig, type MCPStreamableHTTPTransportConfig, type MCPTransportConfig, type ModelUsage, type ModelInfo, type UsageCost, type AggregatedCosts };
|
|
@@ -1,17 +1,21 @@
|
|
|
1
1
|
import { AxCrew } from "../dist/index.js";
|
|
2
|
-
import { AxCrewFunctions } from "../
|
|
2
|
+
import { AxCrewFunctions } from "../dist/functions/index.js";
|
|
3
|
+
import type { AxCrewConfig } from "../dist/index.js";
|
|
4
|
+
import type { Provider } from "../dist/types.js";
|
|
3
5
|
|
|
4
6
|
// Example agent configuration
|
|
5
|
-
const agentConfig = {
|
|
7
|
+
const agentConfig: AxCrewConfig = {
|
|
6
8
|
crew: [
|
|
7
9
|
{
|
|
8
10
|
name: "researcher",
|
|
9
11
|
description: "A research agent that finds information",
|
|
10
12
|
signature: "query:string -> research:string",
|
|
11
|
-
provider: "
|
|
12
|
-
providerKeyName: "
|
|
13
|
+
provider: "google-gemini" as Provider,
|
|
14
|
+
providerKeyName: "GEMINI_API_KEY",
|
|
13
15
|
ai: {
|
|
14
|
-
model: "
|
|
16
|
+
model: "gemini-2.5-flash-lite",
|
|
17
|
+
maxTokens: 4000,
|
|
18
|
+
stream: true
|
|
15
19
|
},
|
|
16
20
|
options: {
|
|
17
21
|
debug: true,
|
|
@@ -22,10 +26,12 @@ const agentConfig = {
|
|
|
22
26
|
name: "writer",
|
|
23
27
|
description: "A writing agent that creates content",
|
|
24
28
|
signature: "topic:string -> article:string",
|
|
25
|
-
provider: "anthropic",
|
|
29
|
+
provider: "anthropic" as Provider,
|
|
26
30
|
providerKeyName: "ANTHROPIC_API_KEY",
|
|
27
31
|
ai: {
|
|
28
|
-
model: "claude-3-haiku-20240307"
|
|
32
|
+
model: "claude-3-haiku-20240307",
|
|
33
|
+
maxTokens: 4000,
|
|
34
|
+
stream: true
|
|
29
35
|
},
|
|
30
36
|
options: {
|
|
31
37
|
debug: true,
|
|
@@ -61,12 +67,11 @@ async function main() {
|
|
|
61
67
|
// Print the article
|
|
62
68
|
console.log("Article:", article);
|
|
63
69
|
|
|
64
|
-
// Print
|
|
65
|
-
console.log("\
|
|
66
|
-
console.log("Writer
|
|
67
|
-
console.log("Researcher
|
|
68
|
-
console.log("
|
|
69
|
-
console.log("Total Cost:", JSON.stringify(crew.getAggregatedCosts(), null, 2));
|
|
70
|
+
// Print metrics snapshots (new mechanism)
|
|
71
|
+
console.log("\nMetrics:\n+++++++++++++++++++++++++++++++++");
|
|
72
|
+
console.log("Writer Metrics:", JSON.stringify((writer as any).getMetrics?.(), null, 2));
|
|
73
|
+
console.log("Researcher Metrics:", JSON.stringify((researcher as any).getMetrics?.(), null, 2));
|
|
74
|
+
console.log("Crew Metrics:", JSON.stringify((crew as any).getCrewMetrics?.(), null, 2));
|
|
70
75
|
|
|
71
76
|
// If you want to start fresh with cost tracking
|
|
72
77
|
crew.resetCosts();
|