@sentrial/sdk 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +210 -137
- package/dist/index.cjs +1908 -46
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1043 -2
- package/dist/index.d.ts +1043 -2
- package/dist/index.js +1875 -45
- package/dist/index.js.map +1 -1
- package/package.json +15 -2
package/dist/index.js
CHANGED
|
@@ -67,26 +67,126 @@ var ValidationError = class extends SentrialError {
|
|
|
67
67
|
}
|
|
68
68
|
};
|
|
69
69
|
|
|
70
|
+
// src/redact.ts
|
|
71
|
+
import { createHash } from "crypto";
|
|
72
|
+
var DEFAULT_FIELDS = [
|
|
73
|
+
"userInput",
|
|
74
|
+
"assistantOutput",
|
|
75
|
+
"toolInput",
|
|
76
|
+
"toolOutput"
|
|
77
|
+
];
|
|
78
|
+
var DEFAULT_BUILTIN = {
|
|
79
|
+
emails: true,
|
|
80
|
+
phones: true,
|
|
81
|
+
ssns: true,
|
|
82
|
+
creditCards: true,
|
|
83
|
+
ipAddresses: true
|
|
84
|
+
};
|
|
85
|
+
var EMAIL_PATTERN = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
|
|
86
|
+
var PHONE_PATTERN = /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g;
|
|
87
|
+
var SSN_PATTERN = /\b\d{3}-\d{2}-\d{4}\b/g;
|
|
88
|
+
var CREDIT_CARD_PATTERN = /\b(?:\d[-\s]?){13,19}\b/g;
|
|
89
|
+
var IP_ADDRESS_PATTERN = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g;
|
|
90
|
+
var BUILTIN_PATTERNS = {
|
|
91
|
+
emails: { pattern: EMAIL_PATTERN, label: "EMAIL" },
|
|
92
|
+
phones: { pattern: PHONE_PATTERN, label: "PHONE" },
|
|
93
|
+
ssns: { pattern: SSN_PATTERN, label: "SSN" },
|
|
94
|
+
creditCards: { pattern: CREDIT_CARD_PATTERN, label: "CREDIT_CARD" },
|
|
95
|
+
ipAddresses: { pattern: IP_ADDRESS_PATTERN, label: "IP_ADDRESS" }
|
|
96
|
+
};
|
|
97
|
+
function hashValue(value) {
|
|
98
|
+
return createHash("sha256").update(value).digest("hex").slice(0, 6);
|
|
99
|
+
}
|
|
100
|
+
function replaceMatch(match, label, mode) {
|
|
101
|
+
switch (mode) {
|
|
102
|
+
case "label":
|
|
103
|
+
return `[${label}]`;
|
|
104
|
+
case "hash":
|
|
105
|
+
return `[PII:${hashValue(match)}]`;
|
|
106
|
+
case "remove":
|
|
107
|
+
return "";
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function redactString(value, mode, builtinPatterns, customPatterns) {
|
|
111
|
+
let result = value;
|
|
112
|
+
for (const [key, config] of Object.entries(BUILTIN_PATTERNS)) {
|
|
113
|
+
if (builtinPatterns[key]) {
|
|
114
|
+
const regex = new RegExp(config.pattern.source, config.pattern.flags);
|
|
115
|
+
result = result.replace(regex, (match) => replaceMatch(match, config.label, mode));
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
for (const custom of customPatterns) {
|
|
119
|
+
const regex = new RegExp(custom.pattern.source, custom.pattern.flags);
|
|
120
|
+
result = result.replace(regex, (match) => replaceMatch(match, custom.label, mode));
|
|
121
|
+
}
|
|
122
|
+
return result;
|
|
123
|
+
}
|
|
124
|
+
function redactValue(value, mode, builtinPatterns, customPatterns) {
|
|
125
|
+
if (typeof value === "string") {
|
|
126
|
+
return redactString(value, mode, builtinPatterns, customPatterns);
|
|
127
|
+
}
|
|
128
|
+
if (Array.isArray(value)) {
|
|
129
|
+
return value.map((item) => redactValue(item, mode, builtinPatterns, customPatterns));
|
|
130
|
+
}
|
|
131
|
+
if (value !== null && typeof value === "object") {
|
|
132
|
+
const result = {};
|
|
133
|
+
for (const [key, val] of Object.entries(value)) {
|
|
134
|
+
result[key] = redactValue(val, mode, builtinPatterns, customPatterns);
|
|
135
|
+
}
|
|
136
|
+
return result;
|
|
137
|
+
}
|
|
138
|
+
return value;
|
|
139
|
+
}
|
|
140
|
+
function redactPayload(payload, config) {
|
|
141
|
+
if (!config.enabled) {
|
|
142
|
+
return payload;
|
|
143
|
+
}
|
|
144
|
+
const mode = config.mode ?? "label";
|
|
145
|
+
const fields = config.fields ?? DEFAULT_FIELDS;
|
|
146
|
+
const builtinPatterns = {
|
|
147
|
+
...DEFAULT_BUILTIN,
|
|
148
|
+
...config.builtinPatterns
|
|
149
|
+
};
|
|
150
|
+
const customPatterns = config.customPatterns ?? [];
|
|
151
|
+
const result = { ...payload };
|
|
152
|
+
for (const field of fields) {
|
|
153
|
+
if (field in result && result[field] !== void 0 && result[field] !== null) {
|
|
154
|
+
result[field] = redactValue(result[field], mode, builtinPatterns, customPatterns);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return result;
|
|
158
|
+
}
|
|
159
|
+
|
|
70
160
|
// src/cost.ts
|
|
71
161
|
var OPENAI_PRICING = {
|
|
72
162
|
"gpt-5.2": { input: 5, output: 15 },
|
|
73
163
|
"gpt-5": { input: 4, output: 12 },
|
|
164
|
+
"gpt-4.1": { input: 2, output: 8 },
|
|
165
|
+
"gpt-4.1-mini": { input: 0.4, output: 1.6 },
|
|
166
|
+
"gpt-4.1-nano": { input: 0.1, output: 0.4 },
|
|
74
167
|
"gpt-4o": { input: 2.5, output: 10 },
|
|
75
168
|
"gpt-4o-mini": { input: 0.15, output: 0.6 },
|
|
76
169
|
"gpt-4-turbo": { input: 10, output: 30 },
|
|
77
170
|
"gpt-4": { input: 30, output: 60 },
|
|
78
171
|
"gpt-3.5-turbo": { input: 0.5, output: 1.5 },
|
|
172
|
+
"o4-mini": { input: 1.1, output: 4.4 },
|
|
79
173
|
"o3": { input: 10, output: 40 },
|
|
80
|
-
"o3-mini": { input:
|
|
174
|
+
"o3-mini": { input: 1.1, output: 4.4 },
|
|
175
|
+
"o3-pro": { input: 20, output: 80 },
|
|
176
|
+
"o1": { input: 15, output: 60 },
|
|
81
177
|
"o1-preview": { input: 15, output: 60 },
|
|
82
178
|
"o1-mini": { input: 3, output: 12 }
|
|
83
179
|
};
|
|
84
180
|
var ANTHROPIC_PRICING = {
|
|
181
|
+
"claude-opus-4": { input: 15, output: 75 },
|
|
182
|
+
"claude-sonnet-4": { input: 3, output: 15 },
|
|
85
183
|
"claude-4.5-opus": { input: 20, output: 100 },
|
|
86
184
|
"claude-4.5-sonnet": { input: 4, output: 20 },
|
|
87
185
|
"claude-4-opus": { input: 18, output: 90 },
|
|
88
186
|
"claude-4-sonnet": { input: 3.5, output: 17.5 },
|
|
187
|
+
"claude-3-7-sonnet": { input: 3, output: 15 },
|
|
89
188
|
"claude-3-5-sonnet": { input: 3, output: 15 },
|
|
189
|
+
"claude-3-5-haiku": { input: 0.8, output: 4 },
|
|
90
190
|
"claude-3-opus": { input: 15, output: 75 },
|
|
91
191
|
"claude-3-sonnet": { input: 3, output: 15 },
|
|
92
192
|
"claude-3-haiku": { input: 0.25, output: 1.25 }
|
|
@@ -147,65 +247,168 @@ var EventType = /* @__PURE__ */ ((EventType2) => {
|
|
|
147
247
|
|
|
148
248
|
// src/client.ts
|
|
149
249
|
var DEFAULT_API_URL = "https://api.sentrial.com";
|
|
250
|
+
var MAX_RETRIES = 3;
|
|
251
|
+
var INITIAL_BACKOFF_MS = 500;
|
|
252
|
+
var MAX_BACKOFF_MS = 8e3;
|
|
253
|
+
var BACKOFF_MULTIPLIER = 2;
|
|
254
|
+
var RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([408, 429, 500, 502, 503, 504]);
|
|
255
|
+
var REQUEST_TIMEOUT_MS = 1e4;
|
|
150
256
|
var SentrialClient = class {
|
|
151
257
|
apiUrl;
|
|
152
258
|
apiKey;
|
|
153
259
|
failSilently;
|
|
260
|
+
piiConfig;
|
|
261
|
+
piiConfigNeedsHydration = false;
|
|
262
|
+
piiHydrationPromise;
|
|
154
263
|
currentState = {};
|
|
155
264
|
constructor(config = {}) {
|
|
156
265
|
this.apiUrl = (config.apiUrl ?? (typeof process !== "undefined" ? process.env?.SENTRIAL_API_URL : void 0) ?? DEFAULT_API_URL).replace(/\/$/, "");
|
|
157
266
|
this.apiKey = config.apiKey ?? (typeof process !== "undefined" ? process.env?.SENTRIAL_API_KEY : void 0);
|
|
158
267
|
this.failSilently = config.failSilently ?? true;
|
|
268
|
+
if (config.pii === true) {
|
|
269
|
+
this.piiConfig = { enabled: true };
|
|
270
|
+
this.piiConfigNeedsHydration = true;
|
|
271
|
+
} else if (config.pii && typeof config.pii === "object") {
|
|
272
|
+
this.piiConfig = config.pii;
|
|
273
|
+
this.piiConfigNeedsHydration = false;
|
|
274
|
+
}
|
|
159
275
|
}
|
|
160
276
|
/**
|
|
161
|
-
*
|
|
277
|
+
* Fetch the organization's PII config from the server.
|
|
278
|
+
*
|
|
279
|
+
* Called lazily on the first request when `pii: true` was passed to the constructor.
|
|
280
|
+
* Uses a single shared promise so concurrent requests don't trigger duplicate fetches.
|
|
162
281
|
*/
|
|
163
|
-
async
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
let errorData = {};
|
|
282
|
+
async hydratePiiConfig() {
|
|
283
|
+
if (!this.piiConfigNeedsHydration) return;
|
|
284
|
+
if (this.piiHydrationPromise) {
|
|
285
|
+
await this.piiHydrationPromise;
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
this.piiHydrationPromise = (async () => {
|
|
289
|
+
try {
|
|
290
|
+
const headers = {};
|
|
291
|
+
if (this.apiKey) {
|
|
292
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
293
|
+
}
|
|
294
|
+
const controller = new AbortController();
|
|
295
|
+
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
296
|
+
let response;
|
|
179
297
|
try {
|
|
180
|
-
|
|
181
|
-
|
|
298
|
+
response = await fetch(`${this.apiUrl}/api/sdk/pii-config`, {
|
|
299
|
+
method: "GET",
|
|
300
|
+
headers,
|
|
301
|
+
signal: controller.signal
|
|
302
|
+
});
|
|
303
|
+
} finally {
|
|
304
|
+
clearTimeout(timeoutId);
|
|
182
305
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
306
|
+
if (response.ok) {
|
|
307
|
+
const data = await response.json();
|
|
308
|
+
if (data.config) {
|
|
309
|
+
this.piiConfig = {
|
|
310
|
+
enabled: data.config.enabled,
|
|
311
|
+
mode: data.config.mode,
|
|
312
|
+
fields: data.config.fields,
|
|
313
|
+
builtinPatterns: data.config.builtinPatterns,
|
|
314
|
+
customPatterns: (data.config.customPatterns || []).map(
|
|
315
|
+
(cp) => ({
|
|
316
|
+
pattern: new RegExp(cp.pattern, "g"),
|
|
317
|
+
label: cp.label
|
|
318
|
+
})
|
|
319
|
+
),
|
|
320
|
+
enhancedDetection: data.config.enhancedDetection
|
|
321
|
+
};
|
|
322
|
+
}
|
|
191
323
|
}
|
|
192
|
-
|
|
324
|
+
} catch {
|
|
193
325
|
}
|
|
194
|
-
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
326
|
+
this.piiConfigNeedsHydration = false;
|
|
327
|
+
})();
|
|
328
|
+
await this.piiHydrationPromise;
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Make an HTTP request with retry logic and exponential backoff.
|
|
332
|
+
*
|
|
333
|
+
* Retries on transient failures (network errors, timeouts, 429/5xx).
|
|
334
|
+
* Up to MAX_RETRIES attempts with exponential backoff.
|
|
335
|
+
*/
|
|
336
|
+
async safeRequest(method, url, body) {
|
|
337
|
+
if (this.piiConfigNeedsHydration) {
|
|
338
|
+
await this.hydratePiiConfig();
|
|
339
|
+
}
|
|
340
|
+
let lastError;
|
|
341
|
+
let backoff = INITIAL_BACKOFF_MS;
|
|
342
|
+
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
|
343
|
+
try {
|
|
344
|
+
const headers = {
|
|
345
|
+
"Content-Type": "application/json"
|
|
346
|
+
};
|
|
347
|
+
if (this.apiKey) {
|
|
348
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
349
|
+
}
|
|
350
|
+
const finalBody = this.piiConfig && body && typeof body === "object" ? redactPayload(body, this.piiConfig) : body;
|
|
351
|
+
const controller = new AbortController();
|
|
352
|
+
const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
353
|
+
let response;
|
|
354
|
+
try {
|
|
355
|
+
response = await fetch(url, {
|
|
356
|
+
method,
|
|
357
|
+
headers,
|
|
358
|
+
body: finalBody ? JSON.stringify(finalBody) : void 0,
|
|
359
|
+
signal: controller.signal
|
|
360
|
+
});
|
|
361
|
+
} finally {
|
|
362
|
+
clearTimeout(timeoutId);
|
|
363
|
+
}
|
|
364
|
+
if (RETRYABLE_STATUS_CODES.has(response.status) && attempt < MAX_RETRIES) {
|
|
365
|
+
await this.sleep(backoff);
|
|
366
|
+
backoff = Math.min(backoff * BACKOFF_MULTIPLIER, MAX_BACKOFF_MS);
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
if (!response.ok) {
|
|
370
|
+
const errorBody = await response.text();
|
|
371
|
+
let errorData = {};
|
|
372
|
+
try {
|
|
373
|
+
errorData = JSON.parse(errorBody);
|
|
374
|
+
} catch {
|
|
375
|
+
}
|
|
376
|
+
const error = new ApiError(
|
|
377
|
+
errorData.error?.message || `HTTP ${response.status}: ${response.statusText}`,
|
|
378
|
+
response.status,
|
|
379
|
+
errorData.error?.code
|
|
380
|
+
);
|
|
381
|
+
if (this.failSilently) {
|
|
382
|
+
console.warn(`Sentrial: Request failed (${method} ${url}):`, error.message);
|
|
383
|
+
return null;
|
|
384
|
+
}
|
|
385
|
+
throw error;
|
|
386
|
+
}
|
|
387
|
+
return await response.json();
|
|
388
|
+
} catch (error) {
|
|
389
|
+
if (error instanceof ApiError) {
|
|
390
|
+
throw error;
|
|
391
|
+
}
|
|
392
|
+
lastError = error instanceof Error ? error : new Error(String(error));
|
|
393
|
+
if (attempt < MAX_RETRIES) {
|
|
394
|
+
await this.sleep(backoff);
|
|
395
|
+
backoff = Math.min(backoff * BACKOFF_MULTIPLIER, MAX_BACKOFF_MS);
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
206
398
|
}
|
|
207
|
-
throw networkError;
|
|
208
399
|
}
|
|
400
|
+
const networkError = new NetworkError(
|
|
401
|
+
lastError?.message ?? "Unknown network error",
|
|
402
|
+
lastError
|
|
403
|
+
);
|
|
404
|
+
if (this.failSilently) {
|
|
405
|
+
console.warn(`Sentrial: Request failed after ${MAX_RETRIES + 1} attempts (${method} ${url}):`, networkError.message);
|
|
406
|
+
return null;
|
|
407
|
+
}
|
|
408
|
+
throw networkError;
|
|
409
|
+
}
|
|
410
|
+
sleep(ms) {
|
|
411
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
209
412
|
}
|
|
210
413
|
/**
|
|
211
414
|
* Create a new session
|
|
@@ -221,6 +424,12 @@ var SentrialClient = class {
|
|
|
221
424
|
userId: params.userId,
|
|
222
425
|
metadata: params.metadata
|
|
223
426
|
};
|
|
427
|
+
if (params.parentSessionId) {
|
|
428
|
+
payload.parentSessionId = params.parentSessionId;
|
|
429
|
+
}
|
|
430
|
+
if (params.convoId) {
|
|
431
|
+
payload.convoId = params.convoId;
|
|
432
|
+
}
|
|
224
433
|
const response = await this.safeRequest(
|
|
225
434
|
"POST",
|
|
226
435
|
`${this.apiUrl}/api/sdk/sessions`,
|
|
@@ -314,6 +523,42 @@ var SentrialClient = class {
|
|
|
314
523
|
updateState(key, value) {
|
|
315
524
|
this.currentState[key] = value;
|
|
316
525
|
}
|
|
526
|
+
/**
|
|
527
|
+
* Set the user input for a session
|
|
528
|
+
*
|
|
529
|
+
* @param sessionId - Session ID
|
|
530
|
+
* @param input - User input text
|
|
531
|
+
* @returns Updated session or null on error
|
|
532
|
+
*/
|
|
533
|
+
async setInput(sessionId, input) {
|
|
534
|
+
return this.safeRequest(
|
|
535
|
+
"PATCH",
|
|
536
|
+
`${this.apiUrl}/api/sdk/sessions/${sessionId}`,
|
|
537
|
+
{ userInput: input }
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* Track a generic event
|
|
542
|
+
*
|
|
543
|
+
* @param params - Event parameters
|
|
544
|
+
* @returns Event data or null on error
|
|
545
|
+
*/
|
|
546
|
+
async trackEvent(params) {
|
|
547
|
+
const stateBefore = { ...this.currentState };
|
|
548
|
+
const payload = {
|
|
549
|
+
sessionId: params.sessionId,
|
|
550
|
+
eventType: params.eventType,
|
|
551
|
+
stateBefore,
|
|
552
|
+
stateAfter: { ...this.currentState }
|
|
553
|
+
};
|
|
554
|
+
if (params.eventData) {
|
|
555
|
+
Object.assign(payload, params.eventData);
|
|
556
|
+
}
|
|
557
|
+
if (params.metadata) {
|
|
558
|
+
payload.metadata = params.metadata;
|
|
559
|
+
}
|
|
560
|
+
return this.safeRequest("POST", `${this.apiUrl}/api/sdk/events`, payload);
|
|
561
|
+
}
|
|
317
562
|
/**
|
|
318
563
|
* Complete a session with performance metrics
|
|
319
564
|
*
|
|
@@ -353,7 +598,8 @@ var SentrialClient = class {
|
|
|
353
598
|
if (params.completionTokens !== void 0) payload.completionTokens = params.completionTokens;
|
|
354
599
|
if (params.totalTokens !== void 0) payload.totalTokens = params.totalTokens;
|
|
355
600
|
if (params.userInput !== void 0) payload.userInput = params.userInput;
|
|
356
|
-
|
|
601
|
+
const output = params.assistantOutput ?? params.output;
|
|
602
|
+
if (output !== void 0) payload.assistantOutput = output;
|
|
357
603
|
return this.safeRequest(
|
|
358
604
|
"PATCH",
|
|
359
605
|
`${this.apiUrl}/api/sdk/sessions/${params.sessionId}`,
|
|
@@ -390,6 +636,7 @@ var SentrialClient = class {
|
|
|
390
636
|
name: `${params.event}:${eventId.slice(0, 8)}`,
|
|
391
637
|
agentName: params.event,
|
|
392
638
|
userId: params.userId,
|
|
639
|
+
convoId: params.convoId,
|
|
393
640
|
metadata: fullMetadata
|
|
394
641
|
});
|
|
395
642
|
if (params.input) {
|
|
@@ -400,7 +647,8 @@ var SentrialClient = class {
|
|
|
400
647
|
sessionId,
|
|
401
648
|
eventId,
|
|
402
649
|
userId: params.userId,
|
|
403
|
-
event: params.event
|
|
650
|
+
event: params.event,
|
|
651
|
+
userInput: params.input
|
|
404
652
|
});
|
|
405
653
|
}
|
|
406
654
|
// Cost calculation static methods for convenience
|
|
@@ -421,6 +669,7 @@ var Interaction = class {
|
|
|
421
669
|
success = true;
|
|
422
670
|
failureReason;
|
|
423
671
|
output;
|
|
672
|
+
userInput;
|
|
424
673
|
degraded;
|
|
425
674
|
constructor(config) {
|
|
426
675
|
this.client = config.client;
|
|
@@ -428,6 +677,7 @@ var Interaction = class {
|
|
|
428
677
|
this.eventId = config.eventId;
|
|
429
678
|
this.userId = config.userId;
|
|
430
679
|
this.event = config.event;
|
|
680
|
+
this.userInput = config.userInput;
|
|
431
681
|
this.degraded = config.sessionId === null;
|
|
432
682
|
}
|
|
433
683
|
/**
|
|
@@ -473,6 +723,7 @@ var Interaction = class {
|
|
|
473
723
|
promptTokens: params.promptTokens,
|
|
474
724
|
completionTokens: params.completionTokens,
|
|
475
725
|
totalTokens: params.totalTokens,
|
|
726
|
+
userInput: this.userInput,
|
|
476
727
|
assistantOutput: finalOutput
|
|
477
728
|
});
|
|
478
729
|
}
|
|
@@ -538,19 +789,1598 @@ var sentrial = {
|
|
|
538
789
|
configure,
|
|
539
790
|
begin
|
|
540
791
|
};
|
|
792
|
+
|
|
793
|
+
// src/vercel.ts
|
|
794
|
+
var _defaultClient = null;
|
|
795
|
+
var _globalConfig = {};
|
|
796
|
+
function configureVercel(config) {
|
|
797
|
+
_defaultClient = new SentrialClient({
|
|
798
|
+
apiKey: config.apiKey,
|
|
799
|
+
apiUrl: config.apiUrl,
|
|
800
|
+
failSilently: config.failSilently ?? true
|
|
801
|
+
});
|
|
802
|
+
_globalConfig = {
|
|
803
|
+
defaultAgent: config.defaultAgent,
|
|
804
|
+
userId: config.userId,
|
|
805
|
+
convoId: config.convoId
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
function getClient2() {
|
|
809
|
+
if (!_defaultClient) {
|
|
810
|
+
_defaultClient = new SentrialClient();
|
|
811
|
+
}
|
|
812
|
+
return _defaultClient;
|
|
813
|
+
}
|
|
814
|
+
function extractModelInfo(model) {
|
|
815
|
+
const modelId = model.modelId || model.id || "unknown";
|
|
816
|
+
const provider = model.provider || guessProvider(modelId);
|
|
817
|
+
return { modelId, provider };
|
|
818
|
+
}
|
|
819
|
+
function guessProvider(modelId) {
|
|
820
|
+
const id = modelId.toLowerCase();
|
|
821
|
+
if (id.includes("gpt") || id.includes("o1") || id.includes("o3") || id.includes("o4") || id.startsWith("chatgpt")) return "openai";
|
|
822
|
+
if (id.includes("claude")) return "anthropic";
|
|
823
|
+
if (id.includes("gemini")) return "google";
|
|
824
|
+
if (id.includes("mistral") || id.includes("mixtral") || id.includes("codestral") || id.includes("pixtral")) return "mistral";
|
|
825
|
+
if (id.includes("llama")) return "meta";
|
|
826
|
+
if (id.includes("deepseek")) return "deepseek";
|
|
827
|
+
if (id.includes("command")) return "cohere";
|
|
828
|
+
if (id.includes("qwen")) return "alibaba";
|
|
829
|
+
return "unknown";
|
|
830
|
+
}
|
|
831
|
+
function calculateCostForCall(provider, modelId, promptTokens, completionTokens) {
|
|
832
|
+
const params = { model: modelId, inputTokens: promptTokens, outputTokens: completionTokens };
|
|
833
|
+
switch (provider.toLowerCase()) {
|
|
834
|
+
case "openai":
|
|
835
|
+
return calculateOpenAICost(params);
|
|
836
|
+
case "anthropic":
|
|
837
|
+
return calculateAnthropicCost(params);
|
|
838
|
+
case "google":
|
|
839
|
+
return calculateGoogleCost(params);
|
|
840
|
+
case "deepseek":
|
|
841
|
+
return promptTokens / 1e6 * 0.27 + completionTokens / 1e6 * 1.1;
|
|
842
|
+
case "cohere":
|
|
843
|
+
return promptTokens / 1e6 * 0.5 + completionTokens / 1e6 * 1.5;
|
|
844
|
+
case "mistral":
|
|
845
|
+
return promptTokens / 1e6 * 2 + completionTokens / 1e6 * 6;
|
|
846
|
+
default:
|
|
847
|
+
return promptTokens * 3e-6 + completionTokens * 6e-6;
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
function extractInput(params) {
|
|
851
|
+
if (params.prompt) return params.prompt;
|
|
852
|
+
if (params.messages && params.messages.length > 0) {
|
|
853
|
+
const lastUserMessage = [...params.messages].reverse().find((m) => m.role === "user");
|
|
854
|
+
if (lastUserMessage) {
|
|
855
|
+
return typeof lastUserMessage.content === "string" ? lastUserMessage.content : JSON.stringify(lastUserMessage.content);
|
|
856
|
+
}
|
|
857
|
+
return JSON.stringify(params.messages);
|
|
858
|
+
}
|
|
859
|
+
return "";
|
|
860
|
+
}
|
|
861
|
+
function wrapTools(tools, sessionId, client) {
|
|
862
|
+
if (!tools) return void 0;
|
|
863
|
+
const wrappedTools = {};
|
|
864
|
+
for (const [toolName, tool] of Object.entries(tools)) {
|
|
865
|
+
if (typeof tool.execute === "function") {
|
|
866
|
+
const originalExecute = tool.execute;
|
|
867
|
+
wrappedTools[toolName] = {
|
|
868
|
+
...tool,
|
|
869
|
+
execute: async (...args) => {
|
|
870
|
+
const startTime = Date.now();
|
|
871
|
+
try {
|
|
872
|
+
const result = await originalExecute(...args);
|
|
873
|
+
const durationMs = Date.now() - startTime;
|
|
874
|
+
client.trackToolCall({
|
|
875
|
+
sessionId,
|
|
876
|
+
toolName,
|
|
877
|
+
toolInput: args[0],
|
|
878
|
+
toolOutput: result,
|
|
879
|
+
reasoning: `Tool executed in ${durationMs}ms`
|
|
880
|
+
}).catch(() => {
|
|
881
|
+
});
|
|
882
|
+
return result;
|
|
883
|
+
} catch (error) {
|
|
884
|
+
const durationMs = Date.now() - startTime;
|
|
885
|
+
client.trackToolCall({
|
|
886
|
+
sessionId,
|
|
887
|
+
toolName,
|
|
888
|
+
toolInput: args[0],
|
|
889
|
+
toolOutput: {},
|
|
890
|
+
toolError: { message: error instanceof Error ? error.message : "Unknown error" },
|
|
891
|
+
reasoning: `Tool failed after ${durationMs}ms`
|
|
892
|
+
}).catch(() => {
|
|
893
|
+
});
|
|
894
|
+
throw error;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
};
|
|
898
|
+
} else {
|
|
899
|
+
wrappedTools[toolName] = tool;
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
return wrappedTools;
|
|
903
|
+
}
|
|
904
|
+
function wrapToolsAsync(tools, sessionPromise, client) {
|
|
905
|
+
const wrappedTools = {};
|
|
906
|
+
for (const [toolName, tool] of Object.entries(tools)) {
|
|
907
|
+
if (typeof tool.execute === "function") {
|
|
908
|
+
const originalExecute = tool.execute;
|
|
909
|
+
wrappedTools[toolName] = {
|
|
910
|
+
...tool,
|
|
911
|
+
execute: async (...args) => {
|
|
912
|
+
const startTime = Date.now();
|
|
913
|
+
const sid = await sessionPromise;
|
|
914
|
+
try {
|
|
915
|
+
const result = await originalExecute(...args);
|
|
916
|
+
const durationMs = Date.now() - startTime;
|
|
917
|
+
if (sid) {
|
|
918
|
+
client.trackToolCall({
|
|
919
|
+
sessionId: sid,
|
|
920
|
+
toolName,
|
|
921
|
+
toolInput: args[0],
|
|
922
|
+
toolOutput: result,
|
|
923
|
+
reasoning: `Tool executed in ${durationMs}ms`
|
|
924
|
+
}).catch(() => {
|
|
925
|
+
});
|
|
926
|
+
}
|
|
927
|
+
return result;
|
|
928
|
+
} catch (error) {
|
|
929
|
+
const durationMs = Date.now() - startTime;
|
|
930
|
+
if (sid) {
|
|
931
|
+
client.trackToolCall({
|
|
932
|
+
sessionId: sid,
|
|
933
|
+
toolName,
|
|
934
|
+
toolInput: args[0],
|
|
935
|
+
toolOutput: {},
|
|
936
|
+
toolError: { message: error instanceof Error ? error.message : "Unknown error" },
|
|
937
|
+
reasoning: `Tool failed after ${durationMs}ms`
|
|
938
|
+
}).catch(() => {
|
|
939
|
+
});
|
|
940
|
+
}
|
|
941
|
+
throw error;
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
};
|
|
945
|
+
} else {
|
|
946
|
+
wrappedTools[toolName] = tool;
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
return wrappedTools;
|
|
950
|
+
}
|
|
951
|
+
function wrapGenerateText(originalFn, client, config) {
|
|
952
|
+
return async (params) => {
|
|
953
|
+
const startTime = Date.now();
|
|
954
|
+
const { modelId, provider } = extractModelInfo(params.model);
|
|
955
|
+
const input = extractInput(params);
|
|
956
|
+
const sessionId = await client.createSession({
|
|
957
|
+
name: `generateText: ${input.slice(0, 50)}${input.length > 50 ? "..." : ""}`,
|
|
958
|
+
agentName: config.defaultAgent ?? "vercel-ai-sdk",
|
|
959
|
+
userId: config.userId ?? "anonymous",
|
|
960
|
+
convoId: config.convoId,
|
|
961
|
+
metadata: {
|
|
962
|
+
model: modelId,
|
|
963
|
+
provider,
|
|
964
|
+
function: "generateText",
|
|
965
|
+
...params.maxSteps ? { maxSteps: params.maxSteps } : {}
|
|
966
|
+
}
|
|
967
|
+
});
|
|
968
|
+
if (!sessionId) {
|
|
969
|
+
return originalFn(params);
|
|
970
|
+
}
|
|
971
|
+
await client.setInput(sessionId, input);
|
|
972
|
+
const wrappedParams = {
|
|
973
|
+
...params,
|
|
974
|
+
tools: wrapTools(params.tools, sessionId, client)
|
|
975
|
+
};
|
|
976
|
+
try {
|
|
977
|
+
const result = await originalFn(wrappedParams);
|
|
978
|
+
const durationMs = Date.now() - startTime;
|
|
979
|
+
const resolvedModelId = result.response?.modelId || modelId;
|
|
980
|
+
const promptTokens = result.usage?.promptTokens || 0;
|
|
981
|
+
const completionTokens = result.usage?.completionTokens || 0;
|
|
982
|
+
const totalTokens = result.usage?.totalTokens || promptTokens + completionTokens;
|
|
983
|
+
const cost = calculateCostForCall(provider, resolvedModelId, promptTokens, completionTokens);
|
|
984
|
+
const steps = result.steps;
|
|
985
|
+
if (steps && steps.length >= 1) {
|
|
986
|
+
for (let i = 0; i < steps.length; i++) {
|
|
987
|
+
const step = steps[i];
|
|
988
|
+
await client.trackEvent({
|
|
989
|
+
sessionId,
|
|
990
|
+
eventType: "llm_call",
|
|
991
|
+
eventData: {
|
|
992
|
+
model: resolvedModelId,
|
|
993
|
+
provider,
|
|
994
|
+
step: i + 1,
|
|
995
|
+
total_steps: steps.length,
|
|
996
|
+
prompt_tokens: step.usage?.promptTokens || 0,
|
|
997
|
+
completion_tokens: step.usage?.completionTokens || 0,
|
|
998
|
+
total_tokens: step.usage?.totalTokens || 0,
|
|
999
|
+
finish_reason: step.finishReason,
|
|
1000
|
+
tool_calls: step.toolCalls?.map((tc) => tc.toolName)
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
} else {
|
|
1005
|
+
await client.trackEvent({
|
|
1006
|
+
sessionId,
|
|
1007
|
+
eventType: "llm_call",
|
|
1008
|
+
eventData: {
|
|
1009
|
+
model: resolvedModelId,
|
|
1010
|
+
provider,
|
|
1011
|
+
prompt_tokens: promptTokens,
|
|
1012
|
+
completion_tokens: completionTokens,
|
|
1013
|
+
total_tokens: totalTokens,
|
|
1014
|
+
finish_reason: result.finishReason,
|
|
1015
|
+
tool_calls: result.toolCalls?.map((tc) => tc.toolName)
|
|
1016
|
+
}
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
await client.completeSession({
|
|
1020
|
+
sessionId,
|
|
1021
|
+
success: true,
|
|
1022
|
+
output: result.text,
|
|
1023
|
+
durationMs,
|
|
1024
|
+
estimatedCost: cost,
|
|
1025
|
+
promptTokens,
|
|
1026
|
+
completionTokens,
|
|
1027
|
+
totalTokens
|
|
1028
|
+
});
|
|
1029
|
+
return result;
|
|
1030
|
+
} catch (error) {
|
|
1031
|
+
const durationMs = Date.now() - startTime;
|
|
1032
|
+
await client.trackError({
|
|
1033
|
+
sessionId,
|
|
1034
|
+
errorType: error instanceof Error ? error.name : "Error",
|
|
1035
|
+
errorMessage: error instanceof Error ? error.message : "Unknown error"
|
|
1036
|
+
});
|
|
1037
|
+
await client.completeSession({
|
|
1038
|
+
sessionId,
|
|
1039
|
+
success: false,
|
|
1040
|
+
failureReason: error instanceof Error ? error.message : "Unknown error",
|
|
1041
|
+
durationMs
|
|
1042
|
+
});
|
|
1043
|
+
throw error;
|
|
1044
|
+
}
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
function wrapStreamText(originalFn, client, config) {
|
|
1048
|
+
return (params) => {
|
|
1049
|
+
const startTime = Date.now();
|
|
1050
|
+
const { modelId, provider } = extractModelInfo(params.model);
|
|
1051
|
+
const input = extractInput(params);
|
|
1052
|
+
let sessionId = null;
|
|
1053
|
+
const sessionPromise = (async () => {
|
|
1054
|
+
try {
|
|
1055
|
+
const id = await client.createSession({
|
|
1056
|
+
name: `streamText: ${input.slice(0, 50)}${input.length > 50 ? "..." : ""}`,
|
|
1057
|
+
agentName: config.defaultAgent ?? "vercel-ai-sdk",
|
|
1058
|
+
userId: config.userId ?? "anonymous",
|
|
1059
|
+
convoId: config.convoId,
|
|
1060
|
+
metadata: {
|
|
1061
|
+
model: modelId,
|
|
1062
|
+
provider,
|
|
1063
|
+
function: "streamText"
|
|
1064
|
+
}
|
|
1065
|
+
});
|
|
1066
|
+
sessionId = id;
|
|
1067
|
+
if (id) {
|
|
1068
|
+
client.setInput(id, input).catch(() => {
|
|
1069
|
+
});
|
|
1070
|
+
}
|
|
1071
|
+
return id;
|
|
1072
|
+
} catch {
|
|
1073
|
+
return null;
|
|
1074
|
+
}
|
|
1075
|
+
})();
|
|
1076
|
+
const wrappedParams = {
|
|
1077
|
+
...params,
|
|
1078
|
+
tools: params.tools ? wrapToolsAsync(params.tools, sessionPromise, client) : void 0
|
|
1079
|
+
};
|
|
1080
|
+
const result = originalFn(wrappedParams);
|
|
1081
|
+
let tracked = false;
|
|
1082
|
+
async function trackCompletion(fullText, error) {
|
|
1083
|
+
if (tracked) return;
|
|
1084
|
+
tracked = true;
|
|
1085
|
+
const durationMs = Date.now() - startTime;
|
|
1086
|
+
const sid = sessionId || await sessionPromise;
|
|
1087
|
+
if (!sid) return;
|
|
1088
|
+
if (error) {
|
|
1089
|
+
await client.trackError({
|
|
1090
|
+
sessionId: sid,
|
|
1091
|
+
errorType: error.name || "Error",
|
|
1092
|
+
errorMessage: error.message || "Unknown error"
|
|
1093
|
+
});
|
|
1094
|
+
await client.completeSession({
|
|
1095
|
+
sessionId: sid,
|
|
1096
|
+
success: false,
|
|
1097
|
+
failureReason: error.message || "Unknown error",
|
|
1098
|
+
durationMs
|
|
1099
|
+
});
|
|
1100
|
+
return;
|
|
1101
|
+
}
|
|
1102
|
+
let usage;
|
|
1103
|
+
try {
|
|
1104
|
+
usage = result.usage ? await result.usage : void 0;
|
|
1105
|
+
} catch {
|
|
1106
|
+
}
|
|
1107
|
+
const promptTokens = usage?.promptTokens || 0;
|
|
1108
|
+
const completionTokens = usage?.completionTokens || 0;
|
|
1109
|
+
const totalTokens = usage?.totalTokens || promptTokens + completionTokens;
|
|
1110
|
+
const cost = calculateCostForCall(provider, modelId, promptTokens, completionTokens);
|
|
1111
|
+
await client.trackEvent({
|
|
1112
|
+
sessionId: sid,
|
|
1113
|
+
eventType: "llm_call",
|
|
1114
|
+
eventData: {
|
|
1115
|
+
model: modelId,
|
|
1116
|
+
provider,
|
|
1117
|
+
prompt_tokens: promptTokens,
|
|
1118
|
+
completion_tokens: completionTokens,
|
|
1119
|
+
total_tokens: totalTokens
|
|
1120
|
+
}
|
|
1121
|
+
});
|
|
1122
|
+
await client.completeSession({
|
|
1123
|
+
sessionId: sid,
|
|
1124
|
+
success: true,
|
|
1125
|
+
output: fullText,
|
|
1126
|
+
durationMs,
|
|
1127
|
+
estimatedCost: cost,
|
|
1128
|
+
promptTokens,
|
|
1129
|
+
completionTokens,
|
|
1130
|
+
totalTokens
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
const textProp = result.text;
|
|
1134
|
+
if (typeof textProp === "string") {
|
|
1135
|
+
trackCompletion(textProp).catch(() => {
|
|
1136
|
+
});
|
|
1137
|
+
} else if (textProp != null && typeof textProp.then === "function") {
|
|
1138
|
+
const originalTextPromise = textProp;
|
|
1139
|
+
result.text = originalTextPromise.then((text) => {
|
|
1140
|
+
trackCompletion(text).catch(() => {
|
|
1141
|
+
});
|
|
1142
|
+
return text;
|
|
1143
|
+
}).catch((err) => {
|
|
1144
|
+
trackCompletion("", err instanceof Error ? err : new Error(String(err))).catch(() => {
|
|
1145
|
+
});
|
|
1146
|
+
throw err;
|
|
1147
|
+
});
|
|
1148
|
+
} else {
|
|
1149
|
+
const originalTextStream = result.textStream;
|
|
1150
|
+
let fullText = "";
|
|
1151
|
+
result.textStream = (async function* () {
|
|
1152
|
+
try {
|
|
1153
|
+
for await (const chunk of originalTextStream) {
|
|
1154
|
+
fullText += chunk;
|
|
1155
|
+
yield chunk;
|
|
1156
|
+
}
|
|
1157
|
+
await trackCompletion(fullText);
|
|
1158
|
+
} catch (error) {
|
|
1159
|
+
await trackCompletion(
|
|
1160
|
+
"",
|
|
1161
|
+
error instanceof Error ? error : new Error(String(error))
|
|
1162
|
+
);
|
|
1163
|
+
throw error;
|
|
1164
|
+
}
|
|
1165
|
+
})();
|
|
1166
|
+
}
|
|
1167
|
+
return result;
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
function wrapGenerateObject(originalFn, client, config) {
|
|
1171
|
+
return async (params) => {
|
|
1172
|
+
const startTime = Date.now();
|
|
1173
|
+
const { modelId, provider } = extractModelInfo(params.model);
|
|
1174
|
+
const input = extractInput(params);
|
|
1175
|
+
const sessionId = await client.createSession({
|
|
1176
|
+
name: `generateObject: ${input.slice(0, 50)}${input.length > 50 ? "..." : ""}`,
|
|
1177
|
+
agentName: config.defaultAgent ?? "vercel-ai-sdk",
|
|
1178
|
+
userId: config.userId ?? "anonymous",
|
|
1179
|
+
convoId: config.convoId,
|
|
1180
|
+
metadata: {
|
|
1181
|
+
model: modelId,
|
|
1182
|
+
provider,
|
|
1183
|
+
function: "generateObject"
|
|
1184
|
+
}
|
|
1185
|
+
});
|
|
1186
|
+
if (!sessionId) {
|
|
1187
|
+
return originalFn(params);
|
|
1188
|
+
}
|
|
1189
|
+
await client.setInput(sessionId, input);
|
|
1190
|
+
try {
|
|
1191
|
+
const result = await originalFn(params);
|
|
1192
|
+
const durationMs = Date.now() - startTime;
|
|
1193
|
+
const resolvedModelId = result.response?.modelId || modelId;
|
|
1194
|
+
const promptTokens = result.usage?.promptTokens || 0;
|
|
1195
|
+
const completionTokens = result.usage?.completionTokens || 0;
|
|
1196
|
+
const totalTokens = result.usage?.totalTokens || promptTokens + completionTokens;
|
|
1197
|
+
const cost = calculateCostForCall(provider, resolvedModelId, promptTokens, completionTokens);
|
|
1198
|
+
await client.completeSession({
|
|
1199
|
+
sessionId,
|
|
1200
|
+
success: true,
|
|
1201
|
+
output: JSON.stringify(result.object),
|
|
1202
|
+
durationMs,
|
|
1203
|
+
estimatedCost: cost,
|
|
1204
|
+
promptTokens,
|
|
1205
|
+
completionTokens,
|
|
1206
|
+
totalTokens
|
|
1207
|
+
});
|
|
1208
|
+
return result;
|
|
1209
|
+
} catch (error) {
|
|
1210
|
+
const durationMs = Date.now() - startTime;
|
|
1211
|
+
await client.trackError({
|
|
1212
|
+
sessionId,
|
|
1213
|
+
errorType: error instanceof Error ? error.name : "Error",
|
|
1214
|
+
errorMessage: error instanceof Error ? error.message : "Unknown error"
|
|
1215
|
+
});
|
|
1216
|
+
await client.completeSession({
|
|
1217
|
+
sessionId,
|
|
1218
|
+
success: false,
|
|
1219
|
+
failureReason: error instanceof Error ? error.message : "Unknown error",
|
|
1220
|
+
durationMs
|
|
1221
|
+
});
|
|
1222
|
+
throw error;
|
|
1223
|
+
}
|
|
1224
|
+
};
|
|
1225
|
+
}
|
|
1226
|
+
function wrapStreamObject(originalFn, client, config) {
|
|
1227
|
+
return (params) => {
|
|
1228
|
+
const startTime = Date.now();
|
|
1229
|
+
const { modelId, provider } = extractModelInfo(params.model);
|
|
1230
|
+
const input = extractInput(params);
|
|
1231
|
+
const sessionPromise = (async () => {
|
|
1232
|
+
try {
|
|
1233
|
+
const id = await client.createSession({
|
|
1234
|
+
name: `streamObject: ${input.slice(0, 50)}${input.length > 50 ? "..." : ""}`,
|
|
1235
|
+
agentName: config.defaultAgent ?? "vercel-ai-sdk",
|
|
1236
|
+
userId: config.userId ?? "anonymous",
|
|
1237
|
+
convoId: config.convoId,
|
|
1238
|
+
metadata: {
|
|
1239
|
+
model: modelId,
|
|
1240
|
+
provider,
|
|
1241
|
+
function: "streamObject"
|
|
1242
|
+
}
|
|
1243
|
+
});
|
|
1244
|
+
if (id) {
|
|
1245
|
+
client.setInput(id, input).catch(() => {
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
return id;
|
|
1249
|
+
} catch {
|
|
1250
|
+
return null;
|
|
1251
|
+
}
|
|
1252
|
+
})();
|
|
1253
|
+
const result = originalFn(params);
|
|
1254
|
+
if (result.object) {
|
|
1255
|
+
const originalObjectPromise = result.object;
|
|
1256
|
+
result.object = originalObjectPromise.then(async (obj) => {
|
|
1257
|
+
const durationMs = Date.now() - startTime;
|
|
1258
|
+
const sid = await sessionPromise;
|
|
1259
|
+
if (sid) {
|
|
1260
|
+
let usage;
|
|
1261
|
+
try {
|
|
1262
|
+
usage = result.usage ? await result.usage : void 0;
|
|
1263
|
+
} catch {
|
|
1264
|
+
}
|
|
1265
|
+
const promptTokens = usage?.promptTokens || 0;
|
|
1266
|
+
const completionTokens = usage?.completionTokens || 0;
|
|
1267
|
+
const totalTokens = usage?.totalTokens || promptTokens + completionTokens;
|
|
1268
|
+
const cost = calculateCostForCall(provider, modelId, promptTokens, completionTokens);
|
|
1269
|
+
await client.completeSession({
|
|
1270
|
+
sessionId: sid,
|
|
1271
|
+
success: true,
|
|
1272
|
+
output: JSON.stringify(obj),
|
|
1273
|
+
durationMs,
|
|
1274
|
+
estimatedCost: cost,
|
|
1275
|
+
promptTokens,
|
|
1276
|
+
completionTokens,
|
|
1277
|
+
totalTokens
|
|
1278
|
+
});
|
|
1279
|
+
}
|
|
1280
|
+
return obj;
|
|
1281
|
+
}).catch(async (error) => {
|
|
1282
|
+
const durationMs = Date.now() - startTime;
|
|
1283
|
+
const sid = await sessionPromise;
|
|
1284
|
+
if (sid) {
|
|
1285
|
+
await client.trackError({
|
|
1286
|
+
sessionId: sid,
|
|
1287
|
+
errorType: error instanceof Error ? error.name : "Error",
|
|
1288
|
+
errorMessage: error instanceof Error ? error.message : "Unknown error"
|
|
1289
|
+
});
|
|
1290
|
+
await client.completeSession({
|
|
1291
|
+
sessionId: sid,
|
|
1292
|
+
success: false,
|
|
1293
|
+
failureReason: error instanceof Error ? error.message : "Unknown error",
|
|
1294
|
+
durationMs
|
|
1295
|
+
});
|
|
1296
|
+
}
|
|
1297
|
+
throw error;
|
|
1298
|
+
});
|
|
1299
|
+
}
|
|
1300
|
+
return result;
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
function wrapAISDK(ai, options) {
|
|
1304
|
+
const client = options?.client ?? getClient2();
|
|
1305
|
+
const config = {
|
|
1306
|
+
defaultAgent: options?.defaultAgent ?? _globalConfig.defaultAgent,
|
|
1307
|
+
userId: options?.userId ?? _globalConfig.userId,
|
|
1308
|
+
convoId: options?.convoId ?? _globalConfig.convoId
|
|
1309
|
+
};
|
|
1310
|
+
return {
|
|
1311
|
+
generateText: ai.generateText ? wrapGenerateText(ai.generateText, client, config) : wrapGenerateText(
|
|
1312
|
+
() => Promise.reject(new Error("generateText not available")),
|
|
1313
|
+
client,
|
|
1314
|
+
config
|
|
1315
|
+
),
|
|
1316
|
+
streamText: ai.streamText ? wrapStreamText(ai.streamText, client, config) : wrapStreamText(() => ({ textStream: (async function* () {
|
|
1317
|
+
})() }), client, config),
|
|
1318
|
+
generateObject: ai.generateObject ? wrapGenerateObject(ai.generateObject, client, config) : wrapGenerateObject(
|
|
1319
|
+
() => Promise.reject(new Error("generateObject not available")),
|
|
1320
|
+
client,
|
|
1321
|
+
config
|
|
1322
|
+
),
|
|
1323
|
+
streamObject: ai.streamObject ? wrapStreamObject(ai.streamObject, client, config) : wrapStreamObject(() => ({}), client, config)
|
|
1324
|
+
};
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
// src/wrappers.ts
|
|
1328
|
+
var _currentSessionId = null;
|
|
1329
|
+
var _currentClient = null;
|
|
1330
|
+
var _defaultClient2 = null;
|
|
1331
|
+
function setSessionContext(sessionId, client) {
|
|
1332
|
+
_currentSessionId = sessionId;
|
|
1333
|
+
if (client) {
|
|
1334
|
+
_currentClient = client;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
function clearSessionContext() {
|
|
1338
|
+
_currentSessionId = null;
|
|
1339
|
+
_currentClient = null;
|
|
1340
|
+
}
|
|
1341
|
+
function getSessionContext() {
|
|
1342
|
+
return _currentSessionId;
|
|
1343
|
+
}
|
|
1344
|
+
function setDefaultClient(client) {
|
|
1345
|
+
_defaultClient2 = client;
|
|
1346
|
+
}
|
|
1347
|
+
function getTrackingClient() {
|
|
1348
|
+
return _currentClient ?? _defaultClient2;
|
|
1349
|
+
}
|
|
1350
|
+
function wrapOpenAI(client, options = {}) {
|
|
1351
|
+
const { trackWithoutSession = false } = options;
|
|
1352
|
+
const chat = client.chat;
|
|
1353
|
+
if (!chat?.completions?.create) {
|
|
1354
|
+
console.warn("Sentrial: OpenAI client does not have chat.completions.create");
|
|
1355
|
+
return client;
|
|
1356
|
+
}
|
|
1357
|
+
const originalCreate = chat.completions.create.bind(chat.completions);
|
|
1358
|
+
chat.completions.create = async function(...args) {
|
|
1359
|
+
const startTime = Date.now();
|
|
1360
|
+
const params = args[0] ?? {};
|
|
1361
|
+
const messages = params.messages ?? [];
|
|
1362
|
+
const model = params.model ?? "unknown";
|
|
1363
|
+
try {
|
|
1364
|
+
const response = await originalCreate(...args);
|
|
1365
|
+
const durationMs = Date.now() - startTime;
|
|
1366
|
+
const promptTokens = response.usage?.prompt_tokens ?? 0;
|
|
1367
|
+
const completionTokens = response.usage?.completion_tokens ?? 0;
|
|
1368
|
+
const totalTokens = response.usage?.total_tokens ?? 0;
|
|
1369
|
+
let outputContent = "";
|
|
1370
|
+
if (response.choices?.[0]?.message?.content) {
|
|
1371
|
+
outputContent = response.choices[0].message.content;
|
|
1372
|
+
}
|
|
1373
|
+
const cost = calculateOpenAICost({ model, inputTokens: promptTokens, outputTokens: completionTokens });
|
|
1374
|
+
trackLLMCall({
|
|
1375
|
+
provider: "openai",
|
|
1376
|
+
model,
|
|
1377
|
+
messages,
|
|
1378
|
+
output: outputContent,
|
|
1379
|
+
promptTokens,
|
|
1380
|
+
completionTokens,
|
|
1381
|
+
totalTokens,
|
|
1382
|
+
cost,
|
|
1383
|
+
durationMs,
|
|
1384
|
+
trackWithoutSession
|
|
1385
|
+
});
|
|
1386
|
+
return response;
|
|
1387
|
+
} catch (error) {
|
|
1388
|
+
const durationMs = Date.now() - startTime;
|
|
1389
|
+
trackLLMError({
|
|
1390
|
+
provider: "openai",
|
|
1391
|
+
model,
|
|
1392
|
+
messages,
|
|
1393
|
+
error,
|
|
1394
|
+
durationMs,
|
|
1395
|
+
trackWithoutSession
|
|
1396
|
+
});
|
|
1397
|
+
throw error;
|
|
1398
|
+
}
|
|
1399
|
+
};
|
|
1400
|
+
return client;
|
|
1401
|
+
}
|
|
1402
|
+
function wrapAnthropic(client, options = {}) {
|
|
1403
|
+
const { trackWithoutSession = false } = options;
|
|
1404
|
+
const messages = client.messages;
|
|
1405
|
+
if (!messages?.create) {
|
|
1406
|
+
console.warn("Sentrial: Anthropic client does not have messages.create");
|
|
1407
|
+
return client;
|
|
1408
|
+
}
|
|
1409
|
+
const originalCreate = messages.create.bind(messages);
|
|
1410
|
+
messages.create = async function(...args) {
|
|
1411
|
+
const startTime = Date.now();
|
|
1412
|
+
const params = args[0] ?? {};
|
|
1413
|
+
const inputMessages = params.messages ?? [];
|
|
1414
|
+
const model = params.model ?? "unknown";
|
|
1415
|
+
const system = params.system ?? "";
|
|
1416
|
+
try {
|
|
1417
|
+
const response = await originalCreate(...args);
|
|
1418
|
+
const durationMs = Date.now() - startTime;
|
|
1419
|
+
const promptTokens = response.usage?.input_tokens ?? 0;
|
|
1420
|
+
const completionTokens = response.usage?.output_tokens ?? 0;
|
|
1421
|
+
const totalTokens = promptTokens + completionTokens;
|
|
1422
|
+
let outputContent = "";
|
|
1423
|
+
if (response.content) {
|
|
1424
|
+
for (const block of response.content) {
|
|
1425
|
+
if (block.type === "text") {
|
|
1426
|
+
outputContent += block.text;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
const cost = calculateAnthropicCost({ model, inputTokens: promptTokens, outputTokens: completionTokens });
|
|
1431
|
+
const fullMessages = system ? [{ role: "system", content: system }, ...inputMessages] : inputMessages;
|
|
1432
|
+
trackLLMCall({
|
|
1433
|
+
provider: "anthropic",
|
|
1434
|
+
model,
|
|
1435
|
+
messages: fullMessages,
|
|
1436
|
+
output: outputContent,
|
|
1437
|
+
promptTokens,
|
|
1438
|
+
completionTokens,
|
|
1439
|
+
totalTokens,
|
|
1440
|
+
cost,
|
|
1441
|
+
durationMs,
|
|
1442
|
+
trackWithoutSession
|
|
1443
|
+
});
|
|
1444
|
+
return response;
|
|
1445
|
+
} catch (error) {
|
|
1446
|
+
const durationMs = Date.now() - startTime;
|
|
1447
|
+
trackLLMError({
|
|
1448
|
+
provider: "anthropic",
|
|
1449
|
+
model,
|
|
1450
|
+
messages: inputMessages,
|
|
1451
|
+
error,
|
|
1452
|
+
durationMs,
|
|
1453
|
+
trackWithoutSession
|
|
1454
|
+
});
|
|
1455
|
+
throw error;
|
|
1456
|
+
}
|
|
1457
|
+
};
|
|
1458
|
+
return client;
|
|
1459
|
+
}
|
|
1460
|
+
function wrapGoogle(model, options = {}) {
|
|
1461
|
+
const { trackWithoutSession = false } = options;
|
|
1462
|
+
const originalGenerate = model.generateContent;
|
|
1463
|
+
if (!originalGenerate) {
|
|
1464
|
+
console.warn("Sentrial: Google model does not have generateContent");
|
|
1465
|
+
return model;
|
|
1466
|
+
}
|
|
1467
|
+
model.generateContent = async function(...args) {
|
|
1468
|
+
const startTime = Date.now();
|
|
1469
|
+
const contents = args[0];
|
|
1470
|
+
const modelName = model.model ?? "gemini-unknown";
|
|
1471
|
+
const messages = googleContentsToMessages(contents);
|
|
1472
|
+
try {
|
|
1473
|
+
const response = await originalGenerate.apply(model, args);
|
|
1474
|
+
const durationMs = Date.now() - startTime;
|
|
1475
|
+
let promptTokens = 0;
|
|
1476
|
+
let completionTokens = 0;
|
|
1477
|
+
if (response.usageMetadata) {
|
|
1478
|
+
promptTokens = response.usageMetadata.promptTokenCount ?? 0;
|
|
1479
|
+
completionTokens = response.usageMetadata.candidatesTokenCount ?? 0;
|
|
1480
|
+
}
|
|
1481
|
+
const totalTokens = promptTokens + completionTokens;
|
|
1482
|
+
let outputContent = "";
|
|
1483
|
+
try {
|
|
1484
|
+
outputContent = response.response?.text() ?? "";
|
|
1485
|
+
} catch {
|
|
1486
|
+
}
|
|
1487
|
+
const cost = calculateGoogleCost({ model: modelName, inputTokens: promptTokens, outputTokens: completionTokens });
|
|
1488
|
+
trackLLMCall({
|
|
1489
|
+
provider: "google",
|
|
1490
|
+
model: modelName,
|
|
1491
|
+
messages,
|
|
1492
|
+
output: outputContent,
|
|
1493
|
+
promptTokens,
|
|
1494
|
+
completionTokens,
|
|
1495
|
+
totalTokens,
|
|
1496
|
+
cost,
|
|
1497
|
+
durationMs,
|
|
1498
|
+
trackWithoutSession
|
|
1499
|
+
});
|
|
1500
|
+
return response;
|
|
1501
|
+
} catch (error) {
|
|
1502
|
+
const durationMs = Date.now() - startTime;
|
|
1503
|
+
trackLLMError({
|
|
1504
|
+
provider: "google",
|
|
1505
|
+
model: modelName,
|
|
1506
|
+
messages,
|
|
1507
|
+
error,
|
|
1508
|
+
durationMs,
|
|
1509
|
+
trackWithoutSession
|
|
1510
|
+
});
|
|
1511
|
+
throw error;
|
|
1512
|
+
}
|
|
1513
|
+
};
|
|
1514
|
+
return model;
|
|
1515
|
+
}
|
|
1516
|
+
function googleContentsToMessages(contents) {
|
|
1517
|
+
if (typeof contents === "string") {
|
|
1518
|
+
return [{ role: "user", content: contents }];
|
|
1519
|
+
}
|
|
1520
|
+
if (Array.isArray(contents)) {
|
|
1521
|
+
return contents.map((item) => {
|
|
1522
|
+
if (typeof item === "string") {
|
|
1523
|
+
return { role: "user", content: item };
|
|
1524
|
+
}
|
|
1525
|
+
if (item && typeof item === "object") {
|
|
1526
|
+
return { role: item.role ?? "user", content: String(item.content ?? item) };
|
|
1527
|
+
}
|
|
1528
|
+
return { role: "user", content: String(item) };
|
|
1529
|
+
});
|
|
1530
|
+
}
|
|
1531
|
+
return [{ role: "user", content: String(contents) }];
|
|
1532
|
+
}
|
|
1533
|
+
function wrapLLM(client, provider) {
|
|
1534
|
+
if (provider === "openai" || client.chat?.completions?.create) {
|
|
1535
|
+
return wrapOpenAI(client);
|
|
1536
|
+
}
|
|
1537
|
+
if (provider === "anthropic" || client.messages?.create) {
|
|
1538
|
+
return wrapAnthropic(client);
|
|
1539
|
+
}
|
|
1540
|
+
if (provider === "google" || client.generateContent) {
|
|
1541
|
+
return wrapGoogle(client);
|
|
1542
|
+
}
|
|
1543
|
+
console.warn("Sentrial: Unknown LLM client type. No auto-tracking applied.");
|
|
1544
|
+
return client;
|
|
1545
|
+
}
|
|
1546
|
+
function trackLLMCall(params) {
|
|
1547
|
+
const client = getTrackingClient();
|
|
1548
|
+
if (!client) return;
|
|
1549
|
+
const sessionId = _currentSessionId;
|
|
1550
|
+
if (!sessionId && !params.trackWithoutSession) {
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
if (sessionId) {
|
|
1554
|
+
client.trackToolCall({
|
|
1555
|
+
sessionId,
|
|
1556
|
+
toolName: `llm:${params.provider}:${params.model}`,
|
|
1557
|
+
toolInput: {
|
|
1558
|
+
messages: params.messages,
|
|
1559
|
+
model: params.model,
|
|
1560
|
+
provider: params.provider
|
|
1561
|
+
},
|
|
1562
|
+
toolOutput: {
|
|
1563
|
+
content: params.output,
|
|
1564
|
+
tokens: {
|
|
1565
|
+
prompt: params.promptTokens,
|
|
1566
|
+
completion: params.completionTokens,
|
|
1567
|
+
total: params.totalTokens
|
|
1568
|
+
},
|
|
1569
|
+
cost_usd: params.cost
|
|
1570
|
+
},
|
|
1571
|
+
reasoning: `LLM call to ${params.provider} ${params.model}`,
|
|
1572
|
+
estimatedCost: params.cost,
|
|
1573
|
+
tokenCount: params.totalTokens,
|
|
1574
|
+
metadata: {
|
|
1575
|
+
provider: params.provider,
|
|
1576
|
+
model: params.model,
|
|
1577
|
+
duration_ms: params.durationMs,
|
|
1578
|
+
prompt_tokens: params.promptTokens,
|
|
1579
|
+
completion_tokens: params.completionTokens
|
|
1580
|
+
}
|
|
1581
|
+
}).catch((err) => {
|
|
1582
|
+
console.warn("Sentrial: Failed to track LLM call:", err.message);
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
function trackLLMError(params) {
|
|
1587
|
+
const client = getTrackingClient();
|
|
1588
|
+
if (!client) return;
|
|
1589
|
+
const sessionId = _currentSessionId;
|
|
1590
|
+
if (!sessionId && !params.trackWithoutSession) {
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
if (sessionId) {
|
|
1594
|
+
client.trackError({
|
|
1595
|
+
sessionId,
|
|
1596
|
+
errorMessage: params.error.message,
|
|
1597
|
+
errorType: params.error.name,
|
|
1598
|
+
toolName: `llm:${params.provider}:${params.model}`,
|
|
1599
|
+
metadata: {
|
|
1600
|
+
provider: params.provider,
|
|
1601
|
+
model: params.model,
|
|
1602
|
+
duration_ms: params.durationMs
|
|
1603
|
+
}
|
|
1604
|
+
}).catch((err) => {
|
|
1605
|
+
console.warn("Sentrial: Failed to track LLM error:", err.message);
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
// src/decorators.ts
|
|
1611
|
+
var _defaultClient3 = null;
|
|
1612
|
+
var _currentInteraction = null;
|
|
1613
|
+
function getClient3() {
|
|
1614
|
+
if (!_defaultClient3) {
|
|
1615
|
+
try {
|
|
1616
|
+
_defaultClient3 = new SentrialClient();
|
|
1617
|
+
setDefaultClient(_defaultClient3);
|
|
1618
|
+
} catch {
|
|
1619
|
+
return null;
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
return _defaultClient3;
|
|
1623
|
+
}
|
|
1624
|
+
function setClient(client) {
|
|
1625
|
+
_defaultClient3 = client;
|
|
1626
|
+
setDefaultClient(client);
|
|
1627
|
+
}
|
|
1628
|
+
function getCurrentSessionId() {
|
|
1629
|
+
return getSessionContext();
|
|
1630
|
+
}
|
|
1631
|
+
function getCurrentInteraction() {
|
|
1632
|
+
return _currentInteraction;
|
|
1633
|
+
}
|
|
1634
|
+
function withTool(name, fn) {
|
|
1635
|
+
const isAsync = fn.constructor.name === "AsyncFunction";
|
|
1636
|
+
if (isAsync) {
|
|
1637
|
+
return async function(...args) {
|
|
1638
|
+
return trackToolAsync(name, fn, args);
|
|
1639
|
+
};
|
|
1640
|
+
} else {
|
|
1641
|
+
return function(...args) {
|
|
1642
|
+
return trackToolSync(name, fn, args);
|
|
1643
|
+
};
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
async function trackToolAsync(toolName, fn, args) {
|
|
1647
|
+
const startTime = Date.now();
|
|
1648
|
+
const toolInput = buildToolInput(args);
|
|
1649
|
+
const client = getClient3();
|
|
1650
|
+
const sessionId = getSessionContext();
|
|
1651
|
+
try {
|
|
1652
|
+
const result = await fn(...args);
|
|
1653
|
+
const durationMs = Date.now() - startTime;
|
|
1654
|
+
if (client && sessionId) {
|
|
1655
|
+
const toolOutput = serializeOutput(result);
|
|
1656
|
+
client.trackToolCall({
|
|
1657
|
+
sessionId,
|
|
1658
|
+
toolName,
|
|
1659
|
+
toolInput,
|
|
1660
|
+
toolOutput,
|
|
1661
|
+
metadata: { duration_ms: durationMs }
|
|
1662
|
+
}).catch((err) => {
|
|
1663
|
+
console.warn(`Sentrial: Failed to track tool ${toolName}:`, err.message);
|
|
1664
|
+
});
|
|
1665
|
+
}
|
|
1666
|
+
return result;
|
|
1667
|
+
} catch (error) {
|
|
1668
|
+
const durationMs = Date.now() - startTime;
|
|
1669
|
+
if (client && sessionId) {
|
|
1670
|
+
client.trackError({
|
|
1671
|
+
sessionId,
|
|
1672
|
+
errorMessage: error.message,
|
|
1673
|
+
errorType: error.name,
|
|
1674
|
+
toolName,
|
|
1675
|
+
stackTrace: error.stack,
|
|
1676
|
+
metadata: { duration_ms: durationMs }
|
|
1677
|
+
}).catch(() => {
|
|
1678
|
+
});
|
|
1679
|
+
}
|
|
1680
|
+
throw error;
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
function trackToolSync(toolName, fn, args) {
|
|
1684
|
+
const startTime = Date.now();
|
|
1685
|
+
const toolInput = buildToolInput(args);
|
|
1686
|
+
const client = getClient3();
|
|
1687
|
+
const sessionId = getSessionContext();
|
|
1688
|
+
try {
|
|
1689
|
+
const result = fn(...args);
|
|
1690
|
+
const durationMs = Date.now() - startTime;
|
|
1691
|
+
if (client && sessionId) {
|
|
1692
|
+
const toolOutput = serializeOutput(result);
|
|
1693
|
+
client.trackToolCall({
|
|
1694
|
+
sessionId,
|
|
1695
|
+
toolName,
|
|
1696
|
+
toolInput,
|
|
1697
|
+
toolOutput,
|
|
1698
|
+
metadata: { duration_ms: durationMs }
|
|
1699
|
+
}).catch((err) => {
|
|
1700
|
+
console.warn(`Sentrial: Failed to track tool ${toolName}:`, err.message);
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
return result;
|
|
1704
|
+
} catch (error) {
|
|
1705
|
+
const durationMs = Date.now() - startTime;
|
|
1706
|
+
if (client && sessionId) {
|
|
1707
|
+
client.trackError({
|
|
1708
|
+
sessionId,
|
|
1709
|
+
errorMessage: error.message,
|
|
1710
|
+
errorType: error.name,
|
|
1711
|
+
toolName,
|
|
1712
|
+
stackTrace: error.stack,
|
|
1713
|
+
metadata: { duration_ms: durationMs }
|
|
1714
|
+
}).catch(() => {
|
|
1715
|
+
});
|
|
1716
|
+
}
|
|
1717
|
+
throw error;
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
function withSession(agentName, fn, options = {}) {
|
|
1721
|
+
return async function(...args) {
|
|
1722
|
+
const client = getClient3();
|
|
1723
|
+
if (!client) {
|
|
1724
|
+
return fn(...args);
|
|
1725
|
+
}
|
|
1726
|
+
const { userId, userInput } = extractParams(args, options);
|
|
1727
|
+
const interaction = await client.begin({
|
|
1728
|
+
userId,
|
|
1729
|
+
event: agentName,
|
|
1730
|
+
input: userInput
|
|
1731
|
+
});
|
|
1732
|
+
const sessionId = interaction.getSessionId();
|
|
1733
|
+
if (sessionId) {
|
|
1734
|
+
setSessionContext(sessionId, client);
|
|
1735
|
+
}
|
|
1736
|
+
_currentInteraction = interaction;
|
|
1737
|
+
try {
|
|
1738
|
+
const result = await fn(...args);
|
|
1739
|
+
let output;
|
|
1740
|
+
if (typeof result === "string") {
|
|
1741
|
+
output = result;
|
|
1742
|
+
} else if (result && typeof result === "object") {
|
|
1743
|
+
if ("response" in result) {
|
|
1744
|
+
output = String(result.response);
|
|
1745
|
+
} else if ("output" in result) {
|
|
1746
|
+
output = String(result.output);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
if (output === void 0 && result !== null && result !== void 0) {
|
|
1750
|
+
output = String(result).slice(0, 1e3);
|
|
1751
|
+
}
|
|
1752
|
+
await interaction.finish({ output, success: true });
|
|
1753
|
+
return result;
|
|
1754
|
+
} catch (error) {
|
|
1755
|
+
await interaction.finish({
|
|
1756
|
+
success: false,
|
|
1757
|
+
failureReason: `${error.name}: ${error.message}`
|
|
1758
|
+
});
|
|
1759
|
+
throw error;
|
|
1760
|
+
} finally {
|
|
1761
|
+
clearSessionContext();
|
|
1762
|
+
_currentInteraction = null;
|
|
1763
|
+
}
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
function extractParams(args, options) {
|
|
1767
|
+
let userId = "anonymous";
|
|
1768
|
+
let userInput;
|
|
1769
|
+
if (typeof options.userIdParam === "number") {
|
|
1770
|
+
userId = String(args[options.userIdParam] ?? "anonymous");
|
|
1771
|
+
} else if (typeof options.userIdParam === "string") {
|
|
1772
|
+
userId = String(args[0] ?? "anonymous");
|
|
1773
|
+
} else {
|
|
1774
|
+
userId = String(args[0] ?? "anonymous");
|
|
1775
|
+
}
|
|
1776
|
+
if (typeof options.inputParam === "number") {
|
|
1777
|
+
userInput = String(args[options.inputParam] ?? "");
|
|
1778
|
+
} else {
|
|
1779
|
+
const input = args[1];
|
|
1780
|
+
if (typeof input === "string") {
|
|
1781
|
+
userInput = input;
|
|
1782
|
+
} else if (Array.isArray(input)) {
|
|
1783
|
+
userInput = JSON.stringify(input).slice(0, 500);
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
return { userId, userInput };
|
|
1787
|
+
}
|
|
1788
|
+
function Tool(name) {
|
|
1789
|
+
return function(_target, propertyKey, descriptor) {
|
|
1790
|
+
const originalMethod = descriptor.value;
|
|
1791
|
+
const toolName = name ?? String(propertyKey);
|
|
1792
|
+
descriptor.value = async function(...args) {
|
|
1793
|
+
const startTime = Date.now();
|
|
1794
|
+
const toolInput = buildToolInput(args);
|
|
1795
|
+
const client = getClient3();
|
|
1796
|
+
const sessionId = getSessionContext();
|
|
1797
|
+
try {
|
|
1798
|
+
const result = await originalMethod.apply(this, args);
|
|
1799
|
+
const durationMs = Date.now() - startTime;
|
|
1800
|
+
if (client && sessionId) {
|
|
1801
|
+
const toolOutput = serializeOutput(result);
|
|
1802
|
+
client.trackToolCall({
|
|
1803
|
+
sessionId,
|
|
1804
|
+
toolName,
|
|
1805
|
+
toolInput,
|
|
1806
|
+
toolOutput,
|
|
1807
|
+
metadata: { duration_ms: durationMs }
|
|
1808
|
+
}).catch((err) => {
|
|
1809
|
+
console.warn(`Sentrial: Failed to track tool ${toolName}:`, err.message);
|
|
1810
|
+
});
|
|
1811
|
+
}
|
|
1812
|
+
return result;
|
|
1813
|
+
} catch (error) {
|
|
1814
|
+
const durationMs = Date.now() - startTime;
|
|
1815
|
+
if (client && sessionId) {
|
|
1816
|
+
client.trackError({
|
|
1817
|
+
sessionId,
|
|
1818
|
+
errorMessage: error.message,
|
|
1819
|
+
errorType: error.name,
|
|
1820
|
+
toolName,
|
|
1821
|
+
stackTrace: error.stack,
|
|
1822
|
+
metadata: { duration_ms: durationMs }
|
|
1823
|
+
}).catch(() => {
|
|
1824
|
+
});
|
|
1825
|
+
}
|
|
1826
|
+
throw error;
|
|
1827
|
+
}
|
|
1828
|
+
};
|
|
1829
|
+
return descriptor;
|
|
1830
|
+
};
|
|
1831
|
+
}
|
|
1832
|
+
function TrackSession(agentName, options) {
|
|
1833
|
+
return function(target, _propertyKey, descriptor) {
|
|
1834
|
+
const originalMethod = descriptor.value;
|
|
1835
|
+
const agent = agentName ?? target.constructor.name;
|
|
1836
|
+
descriptor.value = async function(...args) {
|
|
1837
|
+
const client = getClient3();
|
|
1838
|
+
if (!client) {
|
|
1839
|
+
return originalMethod.apply(this, args);
|
|
1840
|
+
}
|
|
1841
|
+
const { userId, userInput } = extractParams(args, options ?? {});
|
|
1842
|
+
const interaction = await client.begin({
|
|
1843
|
+
userId,
|
|
1844
|
+
event: agent,
|
|
1845
|
+
input: userInput
|
|
1846
|
+
});
|
|
1847
|
+
const sessionId = interaction.getSessionId();
|
|
1848
|
+
if (sessionId) {
|
|
1849
|
+
setSessionContext(sessionId, client);
|
|
1850
|
+
}
|
|
1851
|
+
_currentInteraction = interaction;
|
|
1852
|
+
try {
|
|
1853
|
+
const result = await originalMethod.apply(this, args);
|
|
1854
|
+
let output;
|
|
1855
|
+
if (typeof result === "string") {
|
|
1856
|
+
output = result;
|
|
1857
|
+
} else if (result && typeof result === "object") {
|
|
1858
|
+
if ("response" in result) {
|
|
1859
|
+
output = String(result.response);
|
|
1860
|
+
} else if ("output" in result) {
|
|
1861
|
+
output = String(result.output);
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
if (output === void 0 && result !== null && result !== void 0) {
|
|
1865
|
+
output = String(result).slice(0, 1e3);
|
|
1866
|
+
}
|
|
1867
|
+
await interaction.finish({ output, success: true });
|
|
1868
|
+
return result;
|
|
1869
|
+
} catch (error) {
|
|
1870
|
+
await interaction.finish({
|
|
1871
|
+
success: false,
|
|
1872
|
+
failureReason: `${error.name}: ${error.message}`
|
|
1873
|
+
});
|
|
1874
|
+
throw error;
|
|
1875
|
+
} finally {
|
|
1876
|
+
clearSessionContext();
|
|
1877
|
+
_currentInteraction = null;
|
|
1878
|
+
}
|
|
1879
|
+
};
|
|
1880
|
+
return descriptor;
|
|
1881
|
+
};
|
|
1882
|
+
}
|
|
1883
|
+
var SessionContext = class {
|
|
1884
|
+
userId;
|
|
1885
|
+
agent;
|
|
1886
|
+
input;
|
|
1887
|
+
client;
|
|
1888
|
+
interaction = null;
|
|
1889
|
+
output;
|
|
1890
|
+
constructor(options) {
|
|
1891
|
+
this.userId = options.userId;
|
|
1892
|
+
this.agent = options.agent;
|
|
1893
|
+
this.input = options.input;
|
|
1894
|
+
this.client = options.client ?? getClient3();
|
|
1895
|
+
}
|
|
1896
|
+
/**
|
|
1897
|
+
* Start the session
|
|
1898
|
+
*/
|
|
1899
|
+
async start() {
|
|
1900
|
+
if (!this.client) return this;
|
|
1901
|
+
this.interaction = await this.client.begin({
|
|
1902
|
+
userId: this.userId,
|
|
1903
|
+
event: this.agent,
|
|
1904
|
+
input: this.input
|
|
1905
|
+
});
|
|
1906
|
+
const sessionId = this.interaction.getSessionId();
|
|
1907
|
+
if (sessionId) {
|
|
1908
|
+
setSessionContext(sessionId, this.client);
|
|
1909
|
+
}
|
|
1910
|
+
_currentInteraction = this.interaction;
|
|
1911
|
+
return this;
|
|
1912
|
+
}
|
|
1913
|
+
/**
|
|
1914
|
+
* Set the output for this session
|
|
1915
|
+
*/
|
|
1916
|
+
setOutput(output) {
|
|
1917
|
+
this.output = output;
|
|
1918
|
+
}
|
|
1919
|
+
/**
|
|
1920
|
+
* Finish the session
|
|
1921
|
+
*/
|
|
1922
|
+
async finish(options) {
|
|
1923
|
+
if (this.interaction) {
|
|
1924
|
+
await this.interaction.finish({
|
|
1925
|
+
output: this.output,
|
|
1926
|
+
success: options?.success ?? true,
|
|
1927
|
+
failureReason: options?.error
|
|
1928
|
+
});
|
|
1929
|
+
}
|
|
1930
|
+
clearSessionContext();
|
|
1931
|
+
_currentInteraction = null;
|
|
1932
|
+
}
|
|
1933
|
+
/**
|
|
1934
|
+
* Get the session ID
|
|
1935
|
+
*/
|
|
1936
|
+
get sessionId() {
|
|
1937
|
+
return this.interaction?.getSessionId() ?? null;
|
|
1938
|
+
}
|
|
1939
|
+
};
|
|
1940
|
+
function buildToolInput(args) {
|
|
1941
|
+
if (args.length === 0) {
|
|
1942
|
+
return {};
|
|
1943
|
+
}
|
|
1944
|
+
if (args.length === 1 && typeof args[0] === "object" && args[0] !== null) {
|
|
1945
|
+
return serializeValue(args[0]);
|
|
1946
|
+
}
|
|
1947
|
+
return {
|
|
1948
|
+
args: args.map(serializeValue)
|
|
1949
|
+
};
|
|
1950
|
+
}
|
|
1951
|
+
function serializeValue(value) {
|
|
1952
|
+
if (value === null || value === void 0) {
|
|
1953
|
+
return value;
|
|
1954
|
+
}
|
|
1955
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1956
|
+
return value;
|
|
1957
|
+
}
|
|
1958
|
+
if (Array.isArray(value)) {
|
|
1959
|
+
return value.map(serializeValue);
|
|
1960
|
+
}
|
|
1961
|
+
if (typeof value === "object") {
|
|
1962
|
+
const result = {};
|
|
1963
|
+
for (const [k, v] of Object.entries(value)) {
|
|
1964
|
+
result[k] = serializeValue(v);
|
|
1965
|
+
}
|
|
1966
|
+
return result;
|
|
1967
|
+
}
|
|
1968
|
+
try {
|
|
1969
|
+
return String(value).slice(0, 1e3);
|
|
1970
|
+
} catch {
|
|
1971
|
+
return `<${typeof value}>`;
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
function serializeOutput(value) {
|
|
1975
|
+
if (value === null || value === void 0) {
|
|
1976
|
+
return { result: null };
|
|
1977
|
+
}
|
|
1978
|
+
if (typeof value === "object" && !Array.isArray(value)) {
|
|
1979
|
+
return serializeValue(value);
|
|
1980
|
+
}
|
|
1981
|
+
return { result: serializeValue(value) };
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
// src/context.ts
|
|
1985
|
+
var _experimentContext = null;
|
|
1986
|
+
function getSystemPrompt(defaultPrompt) {
|
|
1987
|
+
if (_experimentContext?.systemPrompt) {
|
|
1988
|
+
return _experimentContext.systemPrompt;
|
|
1989
|
+
}
|
|
1990
|
+
return defaultPrompt ?? "";
|
|
1991
|
+
}
|
|
1992
|
+
function getExperimentContext() {
|
|
1993
|
+
return _experimentContext;
|
|
1994
|
+
}
|
|
1995
|
+
function isExperimentMode() {
|
|
1996
|
+
return _experimentContext !== null;
|
|
1997
|
+
}
|
|
1998
|
+
function getVariantName() {
|
|
1999
|
+
return _experimentContext?.variantName ?? null;
|
|
2000
|
+
}
|
|
2001
|
+
function getExperimentId() {
|
|
2002
|
+
return _experimentContext?.experimentId ?? null;
|
|
2003
|
+
}
|
|
2004
|
+
function setExperimentContext(context) {
|
|
2005
|
+
_experimentContext = context;
|
|
2006
|
+
}
|
|
2007
|
+
function clearExperimentContext() {
|
|
2008
|
+
_experimentContext = null;
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
// src/experiment.ts
|
|
2012
|
+
var ExperimentRunTracker = class {
|
|
2013
|
+
experiment;
|
|
2014
|
+
variantName;
|
|
2015
|
+
baseSessionId;
|
|
2016
|
+
runId;
|
|
2017
|
+
resultSessionId;
|
|
2018
|
+
_success = true;
|
|
2019
|
+
_errorMessage;
|
|
2020
|
+
/** @internal */
|
|
2021
|
+
constructor(experiment, variantName, baseSessionId) {
|
|
2022
|
+
this.experiment = experiment;
|
|
2023
|
+
this.variantName = variantName;
|
|
2024
|
+
this.baseSessionId = baseSessionId;
|
|
2025
|
+
}
|
|
2026
|
+
/**
|
|
2027
|
+
* Start the run - creates a run record via API.
|
|
2028
|
+
*/
|
|
2029
|
+
async start() {
|
|
2030
|
+
try {
|
|
2031
|
+
const response = await this.experiment.request(
|
|
2032
|
+
"POST",
|
|
2033
|
+
`/api/experiments/${this.experiment.experimentId}/runs`,
|
|
2034
|
+
{
|
|
2035
|
+
variantName: this.variantName,
|
|
2036
|
+
baseSessionId: this.baseSessionId
|
|
2037
|
+
}
|
|
2038
|
+
);
|
|
2039
|
+
this.runId = response?.run?.id;
|
|
2040
|
+
} catch {
|
|
2041
|
+
}
|
|
2042
|
+
return this;
|
|
2043
|
+
}
|
|
2044
|
+
/**
|
|
2045
|
+
* Set the session ID of the result session.
|
|
2046
|
+
*/
|
|
2047
|
+
setResultSessionId(sessionId) {
|
|
2048
|
+
this.resultSessionId = sessionId;
|
|
2049
|
+
}
|
|
2050
|
+
/**
|
|
2051
|
+
* Mark the run as complete.
|
|
2052
|
+
*/
|
|
2053
|
+
async complete() {
|
|
2054
|
+
if (!this.runId) return;
|
|
2055
|
+
try {
|
|
2056
|
+
await this.experiment.request(
|
|
2057
|
+
"PATCH",
|
|
2058
|
+
`/api/experiments/${this.experiment.experimentId}/runs/${this.runId}`,
|
|
2059
|
+
{
|
|
2060
|
+
status: "completed",
|
|
2061
|
+
resultSessionId: this.resultSessionId
|
|
2062
|
+
}
|
|
2063
|
+
);
|
|
2064
|
+
} catch {
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
2067
|
+
/**
|
|
2068
|
+
* Mark the run as failed.
|
|
2069
|
+
*/
|
|
2070
|
+
async fail(errorMessage) {
|
|
2071
|
+
this._success = false;
|
|
2072
|
+
this._errorMessage = errorMessage;
|
|
2073
|
+
if (!this.runId) return;
|
|
2074
|
+
try {
|
|
2075
|
+
await this.experiment.request(
|
|
2076
|
+
"PATCH",
|
|
2077
|
+
`/api/experiments/${this.experiment.experimentId}/runs/${this.runId}`,
|
|
2078
|
+
{
|
|
2079
|
+
status: "failed",
|
|
2080
|
+
resultSessionId: this.resultSessionId,
|
|
2081
|
+
errorMessage
|
|
2082
|
+
}
|
|
2083
|
+
);
|
|
2084
|
+
} catch {
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
/**
|
|
2088
|
+
* Get the result of this run.
|
|
2089
|
+
*/
|
|
2090
|
+
getResult() {
|
|
2091
|
+
return {
|
|
2092
|
+
variantName: this.variantName,
|
|
2093
|
+
testCaseSessionId: this.baseSessionId,
|
|
2094
|
+
resultSessionId: this.resultSessionId,
|
|
2095
|
+
success: this._success,
|
|
2096
|
+
errorMessage: this._errorMessage
|
|
2097
|
+
};
|
|
2098
|
+
}
|
|
2099
|
+
};
|
|
2100
|
+
var Experiment = class {
|
|
2101
|
+
/** The experiment ID */
|
|
2102
|
+
experimentId;
|
|
2103
|
+
/** @internal */
|
|
2104
|
+
client;
|
|
2105
|
+
/** @internal */
|
|
2106
|
+
apiUrl;
|
|
2107
|
+
/** @internal */
|
|
2108
|
+
apiKey;
|
|
2109
|
+
config;
|
|
2110
|
+
_variants;
|
|
2111
|
+
_testCases;
|
|
2112
|
+
/**
|
|
2113
|
+
* Create an experiment instance.
|
|
2114
|
+
*
|
|
2115
|
+
* @param experimentId - The experiment ID from Sentrial dashboard
|
|
2116
|
+
* @param options - Configuration options
|
|
2117
|
+
*/
|
|
2118
|
+
constructor(experimentId, options = {}) {
|
|
2119
|
+
this.experimentId = experimentId;
|
|
2120
|
+
this.apiUrl = (options.apiUrl ?? (typeof process !== "undefined" ? process.env?.SENTRIAL_API_URL : void 0) ?? "https://api.sentrial.com").replace(/\/$/, "");
|
|
2121
|
+
this.apiKey = options.apiKey ?? (typeof process !== "undefined" ? process.env?.SENTRIAL_API_KEY : void 0);
|
|
2122
|
+
this.client = new SentrialClient({
|
|
2123
|
+
apiKey: this.apiKey,
|
|
2124
|
+
apiUrl: this.apiUrl,
|
|
2125
|
+
failSilently: false
|
|
2126
|
+
// We want errors for experiments
|
|
2127
|
+
});
|
|
2128
|
+
}
|
|
2129
|
+
/**
|
|
2130
|
+
* Make an HTTP request to the API
|
|
2131
|
+
* @internal
|
|
2132
|
+
*/
|
|
2133
|
+
async request(method, path, body) {
|
|
2134
|
+
const headers = {
|
|
2135
|
+
"Content-Type": "application/json"
|
|
2136
|
+
};
|
|
2137
|
+
if (this.apiKey) {
|
|
2138
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
2139
|
+
}
|
|
2140
|
+
const response = await fetch(`${this.apiUrl}${path}`, {
|
|
2141
|
+
method,
|
|
2142
|
+
headers,
|
|
2143
|
+
body: body ? JSON.stringify(body) : void 0
|
|
2144
|
+
});
|
|
2145
|
+
if (!response.ok) {
|
|
2146
|
+
return null;
|
|
2147
|
+
}
|
|
2148
|
+
return response.json();
|
|
2149
|
+
}
|
|
2150
|
+
/**
|
|
2151
|
+
* Load the experiment configuration from the API.
|
|
2152
|
+
*/
|
|
2153
|
+
async load() {
|
|
2154
|
+
const response = await this.request(
|
|
2155
|
+
"GET",
|
|
2156
|
+
`/api/experiments/${this.experimentId}/runs`
|
|
2157
|
+
);
|
|
2158
|
+
if (!response?.experiment) {
|
|
2159
|
+
throw new Error(`Failed to load experiment config for ${this.experimentId}`);
|
|
2160
|
+
}
|
|
2161
|
+
this.config = response.experiment;
|
|
2162
|
+
const variants = this.config.variants;
|
|
2163
|
+
this._variants = variants?.map((v) => ({
|
|
2164
|
+
name: v.name ?? "",
|
|
2165
|
+
systemPrompt: v.systemPrompt ?? "",
|
|
2166
|
+
description: v.description
|
|
2167
|
+
})) ?? [];
|
|
2168
|
+
const testCases = this.config.testCases;
|
|
2169
|
+
this._testCases = testCases?.filter((tc) => tc.userInput)?.map((tc) => ({
|
|
2170
|
+
sessionId: tc.sessionId ?? "",
|
|
2171
|
+
userInput: tc.userInput ?? ""
|
|
2172
|
+
})) ?? [];
|
|
2173
|
+
return this;
|
|
2174
|
+
}
|
|
2175
|
+
/**
|
|
2176
|
+
* Get experiment name.
|
|
2177
|
+
*/
|
|
2178
|
+
get name() {
|
|
2179
|
+
return this.config?.name ?? "";
|
|
2180
|
+
}
|
|
2181
|
+
/**
|
|
2182
|
+
* Get experiment status.
|
|
2183
|
+
*/
|
|
2184
|
+
get status() {
|
|
2185
|
+
return this.config?.status ?? "";
|
|
2186
|
+
}
|
|
2187
|
+
/**
|
|
2188
|
+
* Get list of variants to test.
|
|
2189
|
+
*/
|
|
2190
|
+
get variants() {
|
|
2191
|
+
if (!this._variants) {
|
|
2192
|
+
throw new Error("Experiment not loaded. Call load() first.");
|
|
2193
|
+
}
|
|
2194
|
+
return this._variants;
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Get list of test cases.
|
|
2198
|
+
*/
|
|
2199
|
+
get testCases() {
|
|
2200
|
+
if (!this._testCases) {
|
|
2201
|
+
throw new Error("Experiment not loaded. Call load() first.");
|
|
2202
|
+
}
|
|
2203
|
+
return this._testCases;
|
|
2204
|
+
}
|
|
2205
|
+
/**
|
|
2206
|
+
* Get a specific variant by name.
|
|
2207
|
+
*/
|
|
2208
|
+
getVariant(name) {
|
|
2209
|
+
return this.variants.find((v) => v.name === name);
|
|
2210
|
+
}
|
|
2211
|
+
/**
|
|
2212
|
+
* Create a run tracker for manual experiment runs.
|
|
2213
|
+
*/
|
|
2214
|
+
async trackRun(variantName, baseSessionId) {
|
|
2215
|
+
const tracker = new ExperimentRunTracker(this, variantName, baseSessionId);
|
|
2216
|
+
await tracker.start();
|
|
2217
|
+
return tracker;
|
|
2218
|
+
}
|
|
2219
|
+
/**
|
|
2220
|
+
* Run the experiment with all variants and test cases.
|
|
2221
|
+
*
|
|
2222
|
+
* @param agentFn - Function that runs your agent with a test case and variant
|
|
2223
|
+
* @param options - Run options
|
|
2224
|
+
* @returns List of run results
|
|
2225
|
+
*/
|
|
2226
|
+
async run(agentFn, options = {}) {
|
|
2227
|
+
const { parallel = false, maxWorkers = 4 } = options;
|
|
2228
|
+
if (!this._variants || !this._testCases) {
|
|
2229
|
+
await this.load();
|
|
2230
|
+
}
|
|
2231
|
+
const results = [];
|
|
2232
|
+
const totalRuns = this.variants.length * this.testCases.length;
|
|
2233
|
+
let completed = 0;
|
|
2234
|
+
console.log(`Running experiment: ${this.name || this.experimentId}`);
|
|
2235
|
+
console.log(` Variants: ${this.variants.length}`);
|
|
2236
|
+
console.log(` Test cases: ${this.testCases.length}`);
|
|
2237
|
+
console.log(` Total runs: ${totalRuns}`);
|
|
2238
|
+
console.log();
|
|
2239
|
+
const runSingle = async (variant, testCase) => {
|
|
2240
|
+
const tracker = await this.trackRun(variant.name, testCase.sessionId);
|
|
2241
|
+
setExperimentContext({
|
|
2242
|
+
systemPrompt: variant.systemPrompt,
|
|
2243
|
+
variantName: variant.name,
|
|
2244
|
+
experimentId: this.experimentId,
|
|
2245
|
+
testCaseId: testCase.sessionId
|
|
2246
|
+
});
|
|
2247
|
+
try {
|
|
2248
|
+
await agentFn(testCase, variant, tracker);
|
|
2249
|
+
await tracker.complete();
|
|
2250
|
+
return tracker.getResult();
|
|
2251
|
+
} catch (error) {
|
|
2252
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
2253
|
+
await tracker.fail(errorMessage);
|
|
2254
|
+
return tracker.getResult();
|
|
2255
|
+
} finally {
|
|
2256
|
+
clearExperimentContext();
|
|
2257
|
+
}
|
|
2258
|
+
};
|
|
2259
|
+
if (parallel) {
|
|
2260
|
+
const queue = [];
|
|
2261
|
+
for (const variant of this.variants) {
|
|
2262
|
+
for (const testCase of this.testCases) {
|
|
2263
|
+
queue.push(async () => {
|
|
2264
|
+
const result = await runSingle(variant, testCase);
|
|
2265
|
+
results.push(result);
|
|
2266
|
+
completed++;
|
|
2267
|
+
const status = result.success ? "\u2713" : "\u2717";
|
|
2268
|
+
console.log(
|
|
2269
|
+
` [${completed}/${totalRuns}] ${status} ${variant.name} x ${testCase.sessionId.slice(0, 8)}...`
|
|
2270
|
+
);
|
|
2271
|
+
});
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
const executing = [];
|
|
2275
|
+
for (const task of queue) {
|
|
2276
|
+
const promise = task().then(() => {
|
|
2277
|
+
executing.splice(executing.indexOf(promise), 1);
|
|
2278
|
+
});
|
|
2279
|
+
executing.push(promise);
|
|
2280
|
+
if (executing.length >= maxWorkers) {
|
|
2281
|
+
await Promise.race(executing);
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
await Promise.all(executing);
|
|
2285
|
+
} else {
|
|
2286
|
+
for (const variant of this.variants) {
|
|
2287
|
+
for (const testCase of this.testCases) {
|
|
2288
|
+
const result = await runSingle(variant, testCase);
|
|
2289
|
+
results.push(result);
|
|
2290
|
+
completed++;
|
|
2291
|
+
const status = result.success ? "\u2713" : "\u2717";
|
|
2292
|
+
console.log(
|
|
2293
|
+
` [${completed}/${totalRuns}] ${status} ${variant.name} x ${testCase.sessionId.slice(0, 8)}...`
|
|
2294
|
+
);
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
console.log();
|
|
2299
|
+
console.log("Experiment complete!");
|
|
2300
|
+
console.log(` Successful: ${results.filter((r) => r.success).length}/${totalRuns}`);
|
|
2301
|
+
console.log(` Failed: ${results.filter((r) => !r.success).length}/${totalRuns}`);
|
|
2302
|
+
return results;
|
|
2303
|
+
}
|
|
2304
|
+
/**
|
|
2305
|
+
* Reset all runs for this experiment (for re-running).
|
|
2306
|
+
*/
|
|
2307
|
+
async reset() {
|
|
2308
|
+
try {
|
|
2309
|
+
const response = await this.request(
|
|
2310
|
+
"DELETE",
|
|
2311
|
+
`/api/experiments/${this.experimentId}/runs`
|
|
2312
|
+
);
|
|
2313
|
+
if (response !== null) {
|
|
2314
|
+
this.config = void 0;
|
|
2315
|
+
this._variants = void 0;
|
|
2316
|
+
this._testCases = void 0;
|
|
2317
|
+
return true;
|
|
2318
|
+
}
|
|
2319
|
+
return false;
|
|
2320
|
+
} catch {
|
|
2321
|
+
return false;
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
/**
|
|
2325
|
+
* Fetch aggregated results from the API.
|
|
2326
|
+
*/
|
|
2327
|
+
async getResults() {
|
|
2328
|
+
try {
|
|
2329
|
+
const response = await this.request(
|
|
2330
|
+
"GET",
|
|
2331
|
+
`/api/experiments/${this.experimentId}/results`
|
|
2332
|
+
);
|
|
2333
|
+
return response;
|
|
2334
|
+
} catch {
|
|
2335
|
+
return null;
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
};
|
|
541
2339
|
export {
|
|
542
2340
|
ApiError,
|
|
543
2341
|
EventType,
|
|
2342
|
+
Experiment,
|
|
2343
|
+
ExperimentRunTracker,
|
|
544
2344
|
Interaction,
|
|
545
2345
|
NetworkError,
|
|
546
2346
|
SentrialClient,
|
|
547
2347
|
SentrialError,
|
|
2348
|
+
SessionContext,
|
|
2349
|
+
Tool,
|
|
2350
|
+
TrackSession,
|
|
548
2351
|
ValidationError,
|
|
549
2352
|
begin,
|
|
550
2353
|
calculateAnthropicCost,
|
|
551
2354
|
calculateGoogleCost,
|
|
552
2355
|
calculateOpenAICost,
|
|
2356
|
+
clearExperimentContext,
|
|
2357
|
+
clearSessionContext,
|
|
553
2358
|
configure,
|
|
554
|
-
|
|
2359
|
+
configureVercel,
|
|
2360
|
+
getCurrentInteraction,
|
|
2361
|
+
getCurrentSessionId,
|
|
2362
|
+
getExperimentContext,
|
|
2363
|
+
getExperimentId,
|
|
2364
|
+
getSessionContext,
|
|
2365
|
+
getSystemPrompt,
|
|
2366
|
+
getVariantName,
|
|
2367
|
+
hashValue,
|
|
2368
|
+
isExperimentMode,
|
|
2369
|
+
redactPayload,
|
|
2370
|
+
redactString,
|
|
2371
|
+
redactValue,
|
|
2372
|
+
replaceMatch,
|
|
2373
|
+
sentrial,
|
|
2374
|
+
setClient,
|
|
2375
|
+
setDefaultClient,
|
|
2376
|
+
setExperimentContext,
|
|
2377
|
+
setSessionContext,
|
|
2378
|
+
withSession,
|
|
2379
|
+
withTool,
|
|
2380
|
+
wrapAISDK,
|
|
2381
|
+
wrapAnthropic,
|
|
2382
|
+
wrapGoogle,
|
|
2383
|
+
wrapLLM,
|
|
2384
|
+
wrapOpenAI
|
|
555
2385
|
};
|
|
556
2386
|
//# sourceMappingURL=index.js.map
|