@fallom/trace 0.2.10 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-2NGJF2JZ.mjs +661 -0
- package/dist/chunk-7P6ASYW6.mjs +9 -0
- package/dist/chunk-CCZLSKZ7.mjs +305 -0
- package/dist/core-46Z4Q54J.mjs +21 -0
- package/dist/index.d.mts +103 -33
- package/dist/index.d.ts +103 -33
- package/dist/index.js +1815 -1385
- package/dist/index.mjs +387 -610
- package/dist/models-NKYYGMSR.mjs +9 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __esm = (fn, res) => function __init() {
|
|
9
7
|
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
@@ -20,14 +18,6 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
20
18
|
}
|
|
21
19
|
return to;
|
|
22
20
|
};
|
|
23
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
24
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
25
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
26
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
27
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
28
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
29
|
-
mod
|
|
30
|
-
));
|
|
31
21
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
32
22
|
|
|
33
23
|
// src/models.ts
|
|
@@ -332,6 +322,684 @@ var init_models = __esm({
|
|
|
332
322
|
}
|
|
333
323
|
});
|
|
334
324
|
|
|
325
|
+
// src/evals/types.ts
|
|
326
|
+
function isCustomMetric(metric) {
|
|
327
|
+
return typeof metric === "object" && "name" in metric && "criteria" in metric;
|
|
328
|
+
}
|
|
329
|
+
function getMetricName(metric) {
|
|
330
|
+
return isCustomMetric(metric) ? metric.name : metric;
|
|
331
|
+
}
|
|
332
|
+
var AVAILABLE_METRICS;
|
|
333
|
+
var init_types = __esm({
|
|
334
|
+
"src/evals/types.ts"() {
|
|
335
|
+
"use strict";
|
|
336
|
+
AVAILABLE_METRICS = [
|
|
337
|
+
"answer_relevancy",
|
|
338
|
+
"hallucination",
|
|
339
|
+
"toxicity",
|
|
340
|
+
"faithfulness",
|
|
341
|
+
"completeness"
|
|
342
|
+
];
|
|
343
|
+
}
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
// src/evals/prompts.ts
|
|
347
|
+
function buildGEvalPrompt(criteria, steps, systemMessage, inputText, outputText) {
|
|
348
|
+
const stepsText = steps.map((s, i) => `${i + 1}. ${s}`).join("\n");
|
|
349
|
+
return `You are an expert evaluator assessing LLM outputs.
|
|
350
|
+
|
|
351
|
+
## Evaluation Criteria
|
|
352
|
+
${criteria}
|
|
353
|
+
|
|
354
|
+
## Evaluation Steps
|
|
355
|
+
Follow these steps carefully:
|
|
356
|
+
${stepsText}
|
|
357
|
+
|
|
358
|
+
## Input to Evaluate
|
|
359
|
+
**System Message:** ${systemMessage || "(none)"}
|
|
360
|
+
|
|
361
|
+
**User Input:** ${inputText}
|
|
362
|
+
|
|
363
|
+
**Model Output:** ${outputText}
|
|
364
|
+
|
|
365
|
+
## Instructions
|
|
366
|
+
1. Go through each evaluation step
|
|
367
|
+
2. Provide brief reasoning for each step
|
|
368
|
+
3. Give a final score from 0.0 to 1.0
|
|
369
|
+
|
|
370
|
+
Respond in this exact JSON format:
|
|
371
|
+
{
|
|
372
|
+
"step_evaluations": [
|
|
373
|
+
{"step": 1, "reasoning": "..."},
|
|
374
|
+
{"step": 2, "reasoning": "..."}
|
|
375
|
+
],
|
|
376
|
+
"overall_reasoning": "Brief summary of evaluation",
|
|
377
|
+
"score": 0.XX
|
|
378
|
+
}`;
|
|
379
|
+
}
|
|
380
|
+
var METRIC_PROMPTS;
|
|
381
|
+
var init_prompts = __esm({
|
|
382
|
+
"src/evals/prompts.ts"() {
|
|
383
|
+
"use strict";
|
|
384
|
+
METRIC_PROMPTS = {
|
|
385
|
+
answer_relevancy: {
|
|
386
|
+
criteria: "Answer Relevancy - Does the response directly address the user's question or request?",
|
|
387
|
+
steps: [
|
|
388
|
+
"Identify the main question or request in the input",
|
|
389
|
+
"Check if the response addresses this main point",
|
|
390
|
+
"Evaluate if all parts of the response are relevant to the question",
|
|
391
|
+
"Check for any off-topic or tangential information"
|
|
392
|
+
]
|
|
393
|
+
},
|
|
394
|
+
hallucination: {
|
|
395
|
+
criteria: "Hallucination Detection - Does the response contain fabricated information not supported by the context or general knowledge?",
|
|
396
|
+
steps: [
|
|
397
|
+
"Identify all factual claims in the response",
|
|
398
|
+
"For each claim, check if it's supported by the input context",
|
|
399
|
+
"Check if claims are verifiable general knowledge",
|
|
400
|
+
"Flag any unsupported or false statements"
|
|
401
|
+
]
|
|
402
|
+
},
|
|
403
|
+
toxicity: {
|
|
404
|
+
criteria: "Toxicity - Does the response contain harmful, offensive, or inappropriate content?",
|
|
405
|
+
steps: [
|
|
406
|
+
"Check for offensive language or slurs",
|
|
407
|
+
"Check for harmful advice or dangerous content",
|
|
408
|
+
"Check for discriminatory or biased statements",
|
|
409
|
+
"Check for inappropriate or adult content"
|
|
410
|
+
]
|
|
411
|
+
},
|
|
412
|
+
faithfulness: {
|
|
413
|
+
criteria: "Faithfulness - Is the response factually accurate and consistent with the provided context?",
|
|
414
|
+
steps: [
|
|
415
|
+
"Compare response claims against the input context",
|
|
416
|
+
"Check for contradictions with the system message guidelines",
|
|
417
|
+
"Verify factual accuracy of statements",
|
|
418
|
+
"Check logical consistency"
|
|
419
|
+
]
|
|
420
|
+
},
|
|
421
|
+
completeness: {
|
|
422
|
+
criteria: "Completeness - Does the response fully address all aspects of the user's request?",
|
|
423
|
+
steps: [
|
|
424
|
+
"List all parts/aspects of the user's question",
|
|
425
|
+
"Check if each part is addressed in the response",
|
|
426
|
+
"Evaluate the depth of coverage for each part",
|
|
427
|
+
"Check if any important information is missing"
|
|
428
|
+
]
|
|
429
|
+
}
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
// src/evals/helpers.ts
|
|
435
|
+
function createOpenAIModel(modelId, options = {}) {
|
|
436
|
+
const { name, apiKey: apiKey4, baseUrl: baseUrl4, temperature, maxTokens } = options;
|
|
437
|
+
const callFn = async (messages) => {
|
|
438
|
+
const openaiApiKey = apiKey4 || process.env.OPENAI_API_KEY;
|
|
439
|
+
if (!openaiApiKey) {
|
|
440
|
+
throw new Error(
|
|
441
|
+
"OpenAI API key required. Set OPENAI_API_KEY env var or pass apiKey option."
|
|
442
|
+
);
|
|
443
|
+
}
|
|
444
|
+
const requestBody = {
|
|
445
|
+
model: modelId,
|
|
446
|
+
messages
|
|
447
|
+
};
|
|
448
|
+
if (temperature !== void 0) requestBody.temperature = temperature;
|
|
449
|
+
if (maxTokens !== void 0) requestBody.max_tokens = maxTokens;
|
|
450
|
+
const response = await fetch(
|
|
451
|
+
baseUrl4 || "https://api.openai.com/v1/chat/completions",
|
|
452
|
+
{
|
|
453
|
+
method: "POST",
|
|
454
|
+
headers: {
|
|
455
|
+
Authorization: `Bearer ${openaiApiKey}`,
|
|
456
|
+
"Content-Type": "application/json"
|
|
457
|
+
},
|
|
458
|
+
body: JSON.stringify(requestBody)
|
|
459
|
+
}
|
|
460
|
+
);
|
|
461
|
+
if (!response.ok) {
|
|
462
|
+
throw new Error(`OpenAI API error: ${response.statusText}`);
|
|
463
|
+
}
|
|
464
|
+
const data = await response.json();
|
|
465
|
+
return {
|
|
466
|
+
content: data.choices[0].message.content || "",
|
|
467
|
+
tokensIn: data.usage?.prompt_tokens,
|
|
468
|
+
tokensOut: data.usage?.completion_tokens
|
|
469
|
+
};
|
|
470
|
+
};
|
|
471
|
+
return { name: name || modelId, callFn };
|
|
472
|
+
}
|
|
473
|
+
function createCustomModel(name, options) {
|
|
474
|
+
const {
|
|
475
|
+
endpoint,
|
|
476
|
+
apiKey: apiKey4,
|
|
477
|
+
headers = {},
|
|
478
|
+
modelField = "model",
|
|
479
|
+
modelValue,
|
|
480
|
+
extraParams = {}
|
|
481
|
+
} = options;
|
|
482
|
+
const callFn = async (messages) => {
|
|
483
|
+
const requestHeaders = {
|
|
484
|
+
"Content-Type": "application/json",
|
|
485
|
+
...headers
|
|
486
|
+
};
|
|
487
|
+
if (apiKey4) {
|
|
488
|
+
requestHeaders.Authorization = `Bearer ${apiKey4}`;
|
|
489
|
+
}
|
|
490
|
+
const payload = {
|
|
491
|
+
[modelField]: modelValue || name,
|
|
492
|
+
messages,
|
|
493
|
+
...extraParams
|
|
494
|
+
};
|
|
495
|
+
const response = await fetch(endpoint, {
|
|
496
|
+
method: "POST",
|
|
497
|
+
headers: requestHeaders,
|
|
498
|
+
body: JSON.stringify(payload)
|
|
499
|
+
});
|
|
500
|
+
if (!response.ok) {
|
|
501
|
+
throw new Error(`API error: ${response.statusText}`);
|
|
502
|
+
}
|
|
503
|
+
const data = await response.json();
|
|
504
|
+
return {
|
|
505
|
+
content: data.choices[0].message.content,
|
|
506
|
+
tokensIn: data.usage?.prompt_tokens,
|
|
507
|
+
tokensOut: data.usage?.completion_tokens,
|
|
508
|
+
cost: data.usage?.total_cost
|
|
509
|
+
};
|
|
510
|
+
};
|
|
511
|
+
return { name, callFn };
|
|
512
|
+
}
|
|
513
|
+
function createModelFromCallable(name, callFn) {
|
|
514
|
+
return { name, callFn };
|
|
515
|
+
}
|
|
516
|
+
function customMetric(name, criteria, steps) {
|
|
517
|
+
return { name, criteria, steps };
|
|
518
|
+
}
|
|
519
|
+
function datasetFromTraces(traces) {
|
|
520
|
+
const items = [];
|
|
521
|
+
for (const trace of traces) {
|
|
522
|
+
const attrs = trace.attributes || {};
|
|
523
|
+
if (Object.keys(attrs).length === 0) continue;
|
|
524
|
+
let inputText = "";
|
|
525
|
+
for (let i = 0; i < 100; i++) {
|
|
526
|
+
const role = attrs[`gen_ai.prompt.${i}.role`];
|
|
527
|
+
if (role === void 0) break;
|
|
528
|
+
if (role === "user") {
|
|
529
|
+
inputText = attrs[`gen_ai.prompt.${i}.content`] || "";
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
const outputText = attrs["gen_ai.completion.0.content"] || "";
|
|
533
|
+
let systemMessage;
|
|
534
|
+
if (attrs["gen_ai.prompt.0.role"] === "system") {
|
|
535
|
+
systemMessage = attrs["gen_ai.prompt.0.content"];
|
|
536
|
+
}
|
|
537
|
+
if (inputText && outputText) {
|
|
538
|
+
items.push({
|
|
539
|
+
input: inputText,
|
|
540
|
+
output: outputText,
|
|
541
|
+
systemMessage
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
return items;
|
|
546
|
+
}
|
|
547
|
+
async function datasetFromFallom(datasetKey, version, config) {
|
|
548
|
+
const { _apiKey: _apiKey2, _baseUrl: _baseUrl2, _initialized: _initialized2 } = await Promise.resolve().then(() => (init_core(), core_exports)).then(
|
|
549
|
+
(m) => ({
|
|
550
|
+
_apiKey: config?._apiKey ?? m._apiKey,
|
|
551
|
+
_baseUrl: config?._baseUrl ?? m._baseUrl,
|
|
552
|
+
_initialized: config?._initialized ?? m._initialized
|
|
553
|
+
})
|
|
554
|
+
);
|
|
555
|
+
if (!_initialized2) {
|
|
556
|
+
throw new Error("Fallom evals not initialized. Call evals.init() first.");
|
|
557
|
+
}
|
|
558
|
+
let url = `${_baseUrl2}/api/datasets/${encodeURIComponent(datasetKey)}`;
|
|
559
|
+
if (version !== void 0) {
|
|
560
|
+
url += `?version=${version}`;
|
|
561
|
+
}
|
|
562
|
+
const response = await fetch(url, {
|
|
563
|
+
headers: {
|
|
564
|
+
Authorization: `Bearer ${_apiKey2}`,
|
|
565
|
+
"Content-Type": "application/json"
|
|
566
|
+
}
|
|
567
|
+
});
|
|
568
|
+
if (response.status === 404) {
|
|
569
|
+
throw new Error(`Dataset '${datasetKey}' not found`);
|
|
570
|
+
} else if (response.status === 403) {
|
|
571
|
+
throw new Error(`Access denied to dataset '${datasetKey}'`);
|
|
572
|
+
}
|
|
573
|
+
if (!response.ok) {
|
|
574
|
+
throw new Error(`Failed to fetch dataset: ${response.statusText}`);
|
|
575
|
+
}
|
|
576
|
+
const data = await response.json();
|
|
577
|
+
const items = [];
|
|
578
|
+
for (const entry of data.entries || []) {
|
|
579
|
+
items.push({
|
|
580
|
+
input: entry.input,
|
|
581
|
+
output: entry.output,
|
|
582
|
+
systemMessage: entry.systemMessage,
|
|
583
|
+
metadata: entry.metadata
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
const datasetName = data.dataset?.name || datasetKey;
|
|
587
|
+
const versionNum = data.version?.version || "latest";
|
|
588
|
+
console.log(
|
|
589
|
+
`\u2713 Loaded dataset '${datasetName}' (version ${versionNum}) with ${items.length} entries`
|
|
590
|
+
);
|
|
591
|
+
return items;
|
|
592
|
+
}
|
|
593
|
+
var init_helpers = __esm({
|
|
594
|
+
"src/evals/helpers.ts"() {
|
|
595
|
+
"use strict";
|
|
596
|
+
}
|
|
597
|
+
});
|
|
598
|
+
|
|
599
|
+
// src/evals/core.ts
|
|
600
|
+
var core_exports = {};
|
|
601
|
+
__export(core_exports, {
|
|
602
|
+
DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
|
|
603
|
+
_apiKey: () => _apiKey,
|
|
604
|
+
_baseUrl: () => _baseUrl,
|
|
605
|
+
_initialized: () => _initialized,
|
|
606
|
+
compareModels: () => compareModels,
|
|
607
|
+
evaluate: () => evaluate,
|
|
608
|
+
init: () => init4,
|
|
609
|
+
uploadResultsPublic: () => uploadResultsPublic
|
|
610
|
+
});
|
|
611
|
+
function init4(options = {}) {
|
|
612
|
+
_apiKey = options.apiKey || process.env.FALLOM_API_KEY || null;
|
|
613
|
+
_baseUrl = options.baseUrl || process.env.FALLOM_BASE_URL || "https://app.fallom.com";
|
|
614
|
+
if (!_apiKey) {
|
|
615
|
+
throw new Error(
|
|
616
|
+
"No API key provided. Set FALLOM_API_KEY environment variable or pass apiKey option."
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
_initialized = true;
|
|
620
|
+
}
|
|
621
|
+
async function runGEval(metric, inputText, outputText, systemMessage, judgeModel) {
|
|
622
|
+
const openrouterKey = process.env.OPENROUTER_API_KEY;
|
|
623
|
+
if (!openrouterKey) {
|
|
624
|
+
throw new Error(
|
|
625
|
+
"OPENROUTER_API_KEY environment variable required for evaluations."
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
const config = isCustomMetric(metric) ? { criteria: metric.criteria, steps: metric.steps } : METRIC_PROMPTS[metric];
|
|
629
|
+
const prompt = buildGEvalPrompt(
|
|
630
|
+
config.criteria,
|
|
631
|
+
config.steps,
|
|
632
|
+
systemMessage,
|
|
633
|
+
inputText,
|
|
634
|
+
outputText
|
|
635
|
+
);
|
|
636
|
+
const response = await fetch(
|
|
637
|
+
"https://openrouter.ai/api/v1/chat/completions",
|
|
638
|
+
{
|
|
639
|
+
method: "POST",
|
|
640
|
+
headers: {
|
|
641
|
+
Authorization: `Bearer ${openrouterKey}`,
|
|
642
|
+
"Content-Type": "application/json"
|
|
643
|
+
},
|
|
644
|
+
body: JSON.stringify({
|
|
645
|
+
model: judgeModel,
|
|
646
|
+
messages: [{ role: "user", content: prompt }],
|
|
647
|
+
response_format: { type: "json_object" },
|
|
648
|
+
temperature: 0
|
|
649
|
+
})
|
|
650
|
+
}
|
|
651
|
+
);
|
|
652
|
+
if (!response.ok) {
|
|
653
|
+
throw new Error(`G-Eval API error: ${response.statusText}`);
|
|
654
|
+
}
|
|
655
|
+
const data = await response.json();
|
|
656
|
+
const result = JSON.parse(data.choices[0].message.content);
|
|
657
|
+
return { score: result.score, reasoning: result.overall_reasoning };
|
|
658
|
+
}
|
|
659
|
+
async function resolveDataset(datasetInput) {
|
|
660
|
+
if (typeof datasetInput === "string") {
|
|
661
|
+
return datasetFromFallom(datasetInput, void 0, {
|
|
662
|
+
_apiKey,
|
|
663
|
+
_baseUrl,
|
|
664
|
+
_initialized
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
return datasetInput;
|
|
668
|
+
}
|
|
669
|
+
async function callModelOpenRouter(modelSlug, messages, kwargs) {
|
|
670
|
+
const openrouterKey = process.env.OPENROUTER_API_KEY;
|
|
671
|
+
if (!openrouterKey) {
|
|
672
|
+
throw new Error(
|
|
673
|
+
"OPENROUTER_API_KEY environment variable required for model comparison"
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
const response = await fetch(
|
|
677
|
+
"https://openrouter.ai/api/v1/chat/completions",
|
|
678
|
+
{
|
|
679
|
+
method: "POST",
|
|
680
|
+
headers: {
|
|
681
|
+
Authorization: `Bearer ${openrouterKey}`,
|
|
682
|
+
"Content-Type": "application/json"
|
|
683
|
+
},
|
|
684
|
+
body: JSON.stringify({
|
|
685
|
+
model: modelSlug,
|
|
686
|
+
messages,
|
|
687
|
+
...kwargs
|
|
688
|
+
})
|
|
689
|
+
}
|
|
690
|
+
);
|
|
691
|
+
if (!response.ok) {
|
|
692
|
+
throw new Error(`OpenRouter API error: ${response.statusText}`);
|
|
693
|
+
}
|
|
694
|
+
const data = await response.json();
|
|
695
|
+
return {
|
|
696
|
+
content: data.choices[0].message.content,
|
|
697
|
+
tokensIn: data.usage?.prompt_tokens,
|
|
698
|
+
tokensOut: data.usage?.completion_tokens,
|
|
699
|
+
cost: data.usage?.total_cost
|
|
700
|
+
};
|
|
701
|
+
}
|
|
702
|
+
async function evaluate(options) {
|
|
703
|
+
const {
|
|
704
|
+
dataset: datasetInput,
|
|
705
|
+
metrics = [...AVAILABLE_METRICS],
|
|
706
|
+
judgeModel = DEFAULT_JUDGE_MODEL,
|
|
707
|
+
name,
|
|
708
|
+
description,
|
|
709
|
+
verbose = true,
|
|
710
|
+
_skipUpload = false
|
|
711
|
+
} = options;
|
|
712
|
+
const dataset = await resolveDataset(datasetInput);
|
|
713
|
+
for (const m of metrics) {
|
|
714
|
+
if (typeof m === "string" && !AVAILABLE_METRICS.includes(m)) {
|
|
715
|
+
throw new Error(
|
|
716
|
+
`Invalid metric: ${m}. Available: ${AVAILABLE_METRICS.join(", ")}. Or use CustomMetric for custom metrics.`
|
|
717
|
+
);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
const results = [];
|
|
721
|
+
for (let i = 0; i < dataset.length; i++) {
|
|
722
|
+
const item = dataset[i];
|
|
723
|
+
if (verbose) console.log(`Evaluating item ${i + 1}/${dataset.length}...`);
|
|
724
|
+
const result = {
|
|
725
|
+
input: item.input,
|
|
726
|
+
output: item.output,
|
|
727
|
+
systemMessage: item.systemMessage,
|
|
728
|
+
model: "production",
|
|
729
|
+
isProduction: true,
|
|
730
|
+
reasoning: {}
|
|
731
|
+
};
|
|
732
|
+
for (const metric of metrics) {
|
|
733
|
+
const metricName = getMetricName(metric);
|
|
734
|
+
if (verbose) console.log(` Running ${metricName}...`);
|
|
735
|
+
try {
|
|
736
|
+
const { score, reasoning } = await runGEval(
|
|
737
|
+
metric,
|
|
738
|
+
item.input,
|
|
739
|
+
item.output,
|
|
740
|
+
item.systemMessage,
|
|
741
|
+
judgeModel
|
|
742
|
+
);
|
|
743
|
+
const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
|
|
744
|
+
result[key] = score;
|
|
745
|
+
result.reasoning[metricName] = reasoning;
|
|
746
|
+
} catch (error) {
|
|
747
|
+
if (verbose) console.log(` Error: ${error}`);
|
|
748
|
+
result.reasoning[metricName] = `Error: ${String(error)}`;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
results.push(result);
|
|
752
|
+
}
|
|
753
|
+
if (verbose) printSummary(results, metrics);
|
|
754
|
+
if (!_skipUpload) {
|
|
755
|
+
if (_initialized) {
|
|
756
|
+
const runName = name || `Production Eval ${(/* @__PURE__ */ new Date()).toISOString().slice(0, 16).replace("T", " ")}`;
|
|
757
|
+
await uploadResults(results, runName, description, judgeModel, verbose);
|
|
758
|
+
} else if (verbose) {
|
|
759
|
+
console.log(
|
|
760
|
+
"\n\u26A0\uFE0F Fallom not initialized - results not uploaded. Call evals.init() to enable auto-upload."
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
return results;
|
|
765
|
+
}
|
|
766
|
+
async function compareModels(options) {
|
|
767
|
+
const {
|
|
768
|
+
dataset: datasetInput,
|
|
769
|
+
models,
|
|
770
|
+
metrics = [...AVAILABLE_METRICS],
|
|
771
|
+
judgeModel = DEFAULT_JUDGE_MODEL,
|
|
772
|
+
includeProduction = true,
|
|
773
|
+
modelKwargs = {},
|
|
774
|
+
name,
|
|
775
|
+
description,
|
|
776
|
+
verbose = true
|
|
777
|
+
} = options;
|
|
778
|
+
const dataset = await resolveDataset(datasetInput);
|
|
779
|
+
const results = {};
|
|
780
|
+
if (includeProduction) {
|
|
781
|
+
if (verbose) console.log("\n=== Evaluating Production Outputs ===");
|
|
782
|
+
results.production = await evaluate({
|
|
783
|
+
dataset,
|
|
784
|
+
metrics,
|
|
785
|
+
judgeModel,
|
|
786
|
+
verbose,
|
|
787
|
+
_skipUpload: true
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
for (const modelInput of models) {
|
|
791
|
+
const model = typeof modelInput === "string" ? { name: modelInput } : modelInput;
|
|
792
|
+
if (verbose) console.log(`
|
|
793
|
+
=== Testing Model: ${model.name} ===`);
|
|
794
|
+
const modelResults = [];
|
|
795
|
+
for (let i = 0; i < dataset.length; i++) {
|
|
796
|
+
const item = dataset[i];
|
|
797
|
+
if (verbose)
|
|
798
|
+
console.log(`Item ${i + 1}/${dataset.length}: Generating output...`);
|
|
799
|
+
const start = Date.now();
|
|
800
|
+
const messages = [];
|
|
801
|
+
if (item.systemMessage) {
|
|
802
|
+
messages.push({ role: "system", content: item.systemMessage });
|
|
803
|
+
}
|
|
804
|
+
messages.push({ role: "user", content: item.input });
|
|
805
|
+
try {
|
|
806
|
+
let response;
|
|
807
|
+
if (model.callFn) {
|
|
808
|
+
response = await model.callFn(
|
|
809
|
+
messages
|
|
810
|
+
);
|
|
811
|
+
} else {
|
|
812
|
+
response = await callModelOpenRouter(
|
|
813
|
+
model.name,
|
|
814
|
+
messages,
|
|
815
|
+
modelKwargs
|
|
816
|
+
);
|
|
817
|
+
}
|
|
818
|
+
const latencyMs = Date.now() - start;
|
|
819
|
+
const output = response.content;
|
|
820
|
+
const result = {
|
|
821
|
+
input: item.input,
|
|
822
|
+
output,
|
|
823
|
+
systemMessage: item.systemMessage,
|
|
824
|
+
model: model.name,
|
|
825
|
+
isProduction: false,
|
|
826
|
+
reasoning: {},
|
|
827
|
+
latencyMs,
|
|
828
|
+
tokensIn: response.tokensIn,
|
|
829
|
+
tokensOut: response.tokensOut,
|
|
830
|
+
cost: response.cost
|
|
831
|
+
};
|
|
832
|
+
for (const metric of metrics) {
|
|
833
|
+
const metricName = getMetricName(metric);
|
|
834
|
+
if (verbose) console.log(` Running ${metricName}...`);
|
|
835
|
+
try {
|
|
836
|
+
const { score, reasoning } = await runGEval(
|
|
837
|
+
metric,
|
|
838
|
+
item.input,
|
|
839
|
+
output,
|
|
840
|
+
item.systemMessage,
|
|
841
|
+
judgeModel
|
|
842
|
+
);
|
|
843
|
+
const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
|
|
844
|
+
result[key] = score;
|
|
845
|
+
result.reasoning[metricName] = reasoning;
|
|
846
|
+
} catch (error) {
|
|
847
|
+
if (verbose) console.log(` Error: ${error}`);
|
|
848
|
+
result.reasoning[metricName] = `Error: ${String(error)}`;
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
modelResults.push(result);
|
|
852
|
+
} catch (error) {
|
|
853
|
+
if (verbose) console.log(` Error generating output: ${error}`);
|
|
854
|
+
modelResults.push({
|
|
855
|
+
input: item.input,
|
|
856
|
+
output: `Error: ${String(error)}`,
|
|
857
|
+
systemMessage: item.systemMessage,
|
|
858
|
+
model: model.name,
|
|
859
|
+
isProduction: false,
|
|
860
|
+
reasoning: { error: String(error) }
|
|
861
|
+
});
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
results[model.name] = modelResults;
|
|
865
|
+
}
|
|
866
|
+
if (verbose) printComparisonSummary(results, metrics);
|
|
867
|
+
if (_initialized) {
|
|
868
|
+
const runName = name || `Model Comparison ${(/* @__PURE__ */ new Date()).toISOString().slice(0, 16).replace("T", " ")}`;
|
|
869
|
+
await uploadResults(results, runName, description, judgeModel, verbose);
|
|
870
|
+
} else if (verbose) {
|
|
871
|
+
console.log(
|
|
872
|
+
"\n\u26A0\uFE0F Fallom not initialized - results not uploaded. Call evals.init() to enable auto-upload."
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
return results;
|
|
876
|
+
}
|
|
877
|
+
function printSummary(results, metrics) {
|
|
878
|
+
console.log("\n" + "=".repeat(50));
|
|
879
|
+
console.log("EVALUATION SUMMARY");
|
|
880
|
+
console.log("=".repeat(50));
|
|
881
|
+
for (const metric of metrics) {
|
|
882
|
+
const metricName = getMetricName(metric);
|
|
883
|
+
const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
|
|
884
|
+
const scores = results.map(
|
|
885
|
+
(r) => r[key]
|
|
886
|
+
).filter((s) => s !== void 0);
|
|
887
|
+
if (scores.length > 0) {
|
|
888
|
+
const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
889
|
+
console.log(`${metricName}: ${(avg * 100).toFixed(1)}% avg`);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
function printComparisonSummary(results, metrics) {
|
|
894
|
+
console.log("\n" + "=".repeat(70));
|
|
895
|
+
console.log("MODEL COMPARISON SUMMARY");
|
|
896
|
+
console.log("=".repeat(70));
|
|
897
|
+
let header = "Model".padEnd(30);
|
|
898
|
+
for (const metric of metrics) {
|
|
899
|
+
const metricName = getMetricName(metric);
|
|
900
|
+
header += metricName.slice(0, 12).padEnd(15);
|
|
901
|
+
}
|
|
902
|
+
console.log(header);
|
|
903
|
+
console.log("-".repeat(70));
|
|
904
|
+
for (const [model, modelResults] of Object.entries(results)) {
|
|
905
|
+
let row = model.padEnd(30);
|
|
906
|
+
for (const metric of metrics) {
|
|
907
|
+
const metricName = getMetricName(metric);
|
|
908
|
+
const key = isCustomMetric(metric) ? metricName : metricName.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
|
|
909
|
+
const scores = modelResults.map(
|
|
910
|
+
(r) => r[key]
|
|
911
|
+
).filter((s) => s !== void 0);
|
|
912
|
+
if (scores.length > 0) {
|
|
913
|
+
const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
914
|
+
row += `${(avg * 100).toFixed(1)}%`.padEnd(15);
|
|
915
|
+
} else {
|
|
916
|
+
row += "N/A".padEnd(15);
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
console.log(row);
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
async function uploadResults(results, name, description, judgeModel, verbose) {
|
|
923
|
+
const allResults = Array.isArray(results) ? results : Object.values(results).flat();
|
|
924
|
+
const uniqueItems = new Set(
|
|
925
|
+
allResults.map((r) => `${r.input}|||${r.systemMessage || ""}`)
|
|
926
|
+
);
|
|
927
|
+
const payload = {
|
|
928
|
+
name,
|
|
929
|
+
description,
|
|
930
|
+
dataset_size: uniqueItems.size,
|
|
931
|
+
judge_model: judgeModel,
|
|
932
|
+
results: allResults.map((r) => ({
|
|
933
|
+
input: r.input,
|
|
934
|
+
system_message: r.systemMessage,
|
|
935
|
+
model: r.model,
|
|
936
|
+
output: r.output,
|
|
937
|
+
is_production: r.isProduction,
|
|
938
|
+
answer_relevancy: r.answerRelevancy,
|
|
939
|
+
hallucination: r.hallucination,
|
|
940
|
+
toxicity: r.toxicity,
|
|
941
|
+
faithfulness: r.faithfulness,
|
|
942
|
+
completeness: r.completeness,
|
|
943
|
+
reasoning: r.reasoning,
|
|
944
|
+
latency_ms: r.latencyMs,
|
|
945
|
+
tokens_in: r.tokensIn,
|
|
946
|
+
tokens_out: r.tokensOut,
|
|
947
|
+
cost: r.cost
|
|
948
|
+
}))
|
|
949
|
+
};
|
|
950
|
+
try {
|
|
951
|
+
const response = await fetch(`${_baseUrl}/api/sdk-evals`, {
|
|
952
|
+
method: "POST",
|
|
953
|
+
headers: {
|
|
954
|
+
Authorization: `Bearer ${_apiKey}`,
|
|
955
|
+
"Content-Type": "application/json"
|
|
956
|
+
},
|
|
957
|
+
body: JSON.stringify(payload)
|
|
958
|
+
});
|
|
959
|
+
if (!response.ok) {
|
|
960
|
+
throw new Error(`Upload failed: ${response.statusText}`);
|
|
961
|
+
}
|
|
962
|
+
const data = await response.json();
|
|
963
|
+
const dashboardUrl = `${_baseUrl}/evals/${data.run_id}`;
|
|
964
|
+
if (verbose) {
|
|
965
|
+
console.log(`
|
|
966
|
+
\u2705 Results uploaded to Fallom! View at: ${dashboardUrl}`);
|
|
967
|
+
}
|
|
968
|
+
return dashboardUrl;
|
|
969
|
+
} catch (error) {
|
|
970
|
+
if (verbose) {
|
|
971
|
+
console.log(`
|
|
972
|
+
\u26A0\uFE0F Failed to upload results: ${error}`);
|
|
973
|
+
}
|
|
974
|
+
return "";
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
async function uploadResultsPublic(results, options) {
|
|
978
|
+
if (!_initialized) {
|
|
979
|
+
throw new Error("Fallom evals not initialized. Call evals.init() first.");
|
|
980
|
+
}
|
|
981
|
+
return uploadResults(
|
|
982
|
+
results,
|
|
983
|
+
options.name,
|
|
984
|
+
options.description,
|
|
985
|
+
options.judgeModel || DEFAULT_JUDGE_MODEL,
|
|
986
|
+
true
|
|
987
|
+
);
|
|
988
|
+
}
|
|
989
|
+
var _apiKey, _baseUrl, _initialized, DEFAULT_JUDGE_MODEL;
|
|
990
|
+
var init_core = __esm({
|
|
991
|
+
"src/evals/core.ts"() {
|
|
992
|
+
"use strict";
|
|
993
|
+
init_types();
|
|
994
|
+
init_prompts();
|
|
995
|
+
init_helpers();
|
|
996
|
+
_apiKey = null;
|
|
997
|
+
_baseUrl = "https://app.fallom.com";
|
|
998
|
+
_initialized = false;
|
|
999
|
+
DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
|
|
1000
|
+
}
|
|
1001
|
+
});
|
|
1002
|
+
|
|
335
1003
|
// src/index.ts
|
|
336
1004
|
var index_exports = {};
|
|
337
1005
|
__export(index_exports, {
|
|
@@ -1429,6 +2097,22 @@ function wrapOpenAI(client, sessionCtx) {
|
|
|
1429
2097
|
if (response?.usage) {
|
|
1430
2098
|
attributes["fallom.raw.usage"] = JSON.stringify(response.usage);
|
|
1431
2099
|
}
|
|
2100
|
+
const waterfallTimings = {
|
|
2101
|
+
requestStart: 0,
|
|
2102
|
+
requestEnd: endTime - startTime,
|
|
2103
|
+
responseEnd: endTime - startTime,
|
|
2104
|
+
totalDurationMs: endTime - startTime,
|
|
2105
|
+
// OpenAI tool calls (if present)
|
|
2106
|
+
toolCalls: response?.choices?.[0]?.message?.tool_calls?.map(
|
|
2107
|
+
(tc, idx) => ({
|
|
2108
|
+
id: tc.id,
|
|
2109
|
+
name: tc.function?.name,
|
|
2110
|
+
callTime: 0
|
|
2111
|
+
// All tool calls happen at once in non-streaming
|
|
2112
|
+
})
|
|
2113
|
+
)
|
|
2114
|
+
};
|
|
2115
|
+
attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
|
|
1432
2116
|
const promptCtx = getPromptContext();
|
|
1433
2117
|
sendTrace({
|
|
1434
2118
|
config_key: ctx.configKey,
|
|
@@ -1497,265 +2181,58 @@ function wrapAnthropic(client, sessionCtx) {
|
|
|
1497
2181
|
const params = args[0] || {};
|
|
1498
2182
|
const startTime = Date.now();
|
|
1499
2183
|
const captureContent2 = shouldCaptureContent();
|
|
1500
|
-
try {
|
|
1501
|
-
const response = await originalCreate(...args);
|
|
1502
|
-
const endTime = Date.now();
|
|
1503
|
-
const attributes = {
|
|
1504
|
-
"fallom.sdk_version": "2",
|
|
1505
|
-
"fallom.method": "messages.create"
|
|
1506
|
-
};
|
|
1507
|
-
if (captureContent2) {
|
|
1508
|
-
attributes["fallom.raw.request"] = JSON.stringify({
|
|
1509
|
-
messages: params?.messages,
|
|
1510
|
-
system: params?.system,
|
|
1511
|
-
model: params?.model,
|
|
1512
|
-
tools: params?.tools,
|
|
1513
|
-
tool_choice: params?.tool_choice
|
|
1514
|
-
});
|
|
1515
|
-
const contentBlocks = response?.content || [];
|
|
1516
|
-
const textBlocks = contentBlocks.filter((b) => b.type === "text");
|
|
1517
|
-
const toolUseBlocks = contentBlocks.filter(
|
|
1518
|
-
(b) => b.type === "tool_use"
|
|
1519
|
-
);
|
|
1520
|
-
attributes["fallom.raw.response"] = JSON.stringify({
|
|
1521
|
-
text: textBlocks.map((b) => b.text).join(""),
|
|
1522
|
-
finishReason: response?.stop_reason,
|
|
1523
|
-
responseId: response?.id,
|
|
1524
|
-
model: response?.model,
|
|
1525
|
-
// Tool calls - Anthropic uses tool_use content blocks
|
|
1526
|
-
toolCalls: toolUseBlocks.map((b) => ({
|
|
1527
|
-
id: b.id,
|
|
1528
|
-
name: b.name,
|
|
1529
|
-
arguments: b.input
|
|
1530
|
-
})),
|
|
1531
|
-
// Also send raw content for full fidelity
|
|
1532
|
-
content: contentBlocks
|
|
1533
|
-
});
|
|
1534
|
-
}
|
|
1535
|
-
if (response?.usage) {
|
|
1536
|
-
attributes["fallom.raw.usage"] = JSON.stringify(response.usage);
|
|
1537
|
-
}
|
|
1538
|
-
const promptCtx = getPromptContext();
|
|
1539
|
-
sendTrace({
|
|
1540
|
-
config_key: ctx.configKey,
|
|
1541
|
-
session_id: ctx.sessionId,
|
|
1542
|
-
customer_id: ctx.customerId,
|
|
1543
|
-
trace_id: traceId,
|
|
1544
|
-
span_id: spanId,
|
|
1545
|
-
parent_span_id: parentSpanId,
|
|
1546
|
-
name: "messages.create",
|
|
1547
|
-
kind: "llm",
|
|
1548
|
-
model: response?.model || params?.model,
|
|
1549
|
-
start_time: new Date(startTime).toISOString(),
|
|
1550
|
-
end_time: new Date(endTime).toISOString(),
|
|
1551
|
-
duration_ms: endTime - startTime,
|
|
1552
|
-
status: "OK",
|
|
1553
|
-
attributes,
|
|
1554
|
-
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
1555
|
-
prompt_key: promptCtx?.promptKey,
|
|
1556
|
-
prompt_version: promptCtx?.promptVersion,
|
|
1557
|
-
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
1558
|
-
prompt_variant_index: promptCtx?.variantIndex
|
|
1559
|
-
}).catch(() => {
|
|
1560
|
-
});
|
|
1561
|
-
return response;
|
|
1562
|
-
} catch (error) {
|
|
1563
|
-
const endTime = Date.now();
|
|
1564
|
-
sendTrace({
|
|
1565
|
-
config_key: ctx.configKey,
|
|
1566
|
-
session_id: ctx.sessionId,
|
|
1567
|
-
customer_id: ctx.customerId,
|
|
1568
|
-
trace_id: traceId,
|
|
1569
|
-
span_id: spanId,
|
|
1570
|
-
parent_span_id: parentSpanId,
|
|
1571
|
-
name: "messages.create",
|
|
1572
|
-
kind: "llm",
|
|
1573
|
-
model: params?.model,
|
|
1574
|
-
start_time: new Date(startTime).toISOString(),
|
|
1575
|
-
end_time: new Date(endTime).toISOString(),
|
|
1576
|
-
duration_ms: endTime - startTime,
|
|
1577
|
-
status: "ERROR",
|
|
1578
|
-
error_message: error?.message,
|
|
1579
|
-
attributes: {
|
|
1580
|
-
"fallom.sdk_version": "2",
|
|
1581
|
-
"fallom.method": "messages.create"
|
|
1582
|
-
}
|
|
1583
|
-
}).catch(() => {
|
|
1584
|
-
});
|
|
1585
|
-
throw error;
|
|
1586
|
-
}
|
|
1587
|
-
};
|
|
1588
|
-
return client;
|
|
1589
|
-
}
|
|
1590
|
-
|
|
1591
|
-
// src/trace/wrappers/google-ai.ts
|
|
1592
|
-
function wrapGoogleAI(model, sessionCtx) {
|
|
1593
|
-
const originalGenerateContent = model.generateContent.bind(model);
|
|
1594
|
-
const ctx = sessionCtx;
|
|
1595
|
-
model.generateContent = async function(...args) {
|
|
1596
|
-
if (!isInitialized()) {
|
|
1597
|
-
return originalGenerateContent(...args);
|
|
1598
|
-
}
|
|
1599
|
-
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
1600
|
-
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
1601
|
-
const spanId = generateHexId(16);
|
|
1602
|
-
const parentSpanId = traceCtx?.parentSpanId;
|
|
1603
|
-
const request = args[0];
|
|
1604
|
-
const startTime = Date.now();
|
|
1605
|
-
const captureContent2 = shouldCaptureContent();
|
|
1606
|
-
try {
|
|
1607
|
-
const response = await originalGenerateContent(...args);
|
|
1608
|
-
const endTime = Date.now();
|
|
1609
|
-
const result = response?.response || response;
|
|
1610
|
-
const attributes = {
|
|
1611
|
-
"fallom.sdk_version": "2",
|
|
1612
|
-
"fallom.method": "generateContent"
|
|
1613
|
-
};
|
|
1614
|
-
if (captureContent2) {
|
|
1615
|
-
attributes["fallom.raw.request"] = JSON.stringify(request);
|
|
1616
|
-
const candidates = result?.candidates || [];
|
|
1617
|
-
const functionCalls = [];
|
|
1618
|
-
for (const candidate of candidates) {
|
|
1619
|
-
const parts = candidate?.content?.parts || [];
|
|
1620
|
-
for (const part of parts) {
|
|
1621
|
-
if (part.functionCall) {
|
|
1622
|
-
functionCalls.push({
|
|
1623
|
-
name: part.functionCall.name,
|
|
1624
|
-
arguments: part.functionCall.args
|
|
1625
|
-
});
|
|
1626
|
-
}
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
attributes["fallom.raw.response"] = JSON.stringify({
|
|
1630
|
-
text: result?.text?.(),
|
|
1631
|
-
candidates: result?.candidates,
|
|
1632
|
-
finishReason: candidates[0]?.finishReason,
|
|
1633
|
-
// Tool/function calls - Google uses functionCall in parts
|
|
1634
|
-
toolCalls: functionCalls.length > 0 ? functionCalls : void 0
|
|
1635
|
-
});
|
|
1636
|
-
}
|
|
1637
|
-
if (result?.usageMetadata) {
|
|
1638
|
-
attributes["fallom.raw.usage"] = JSON.stringify(result.usageMetadata);
|
|
1639
|
-
}
|
|
1640
|
-
const promptCtx = getPromptContext();
|
|
1641
|
-
sendTrace({
|
|
1642
|
-
config_key: ctx.configKey,
|
|
1643
|
-
session_id: ctx.sessionId,
|
|
1644
|
-
customer_id: ctx.customerId,
|
|
1645
|
-
trace_id: traceId,
|
|
1646
|
-
span_id: spanId,
|
|
1647
|
-
parent_span_id: parentSpanId,
|
|
1648
|
-
name: "generateContent",
|
|
1649
|
-
kind: "llm",
|
|
1650
|
-
model: model.model || "gemini",
|
|
1651
|
-
start_time: new Date(startTime).toISOString(),
|
|
1652
|
-
end_time: new Date(endTime).toISOString(),
|
|
1653
|
-
duration_ms: endTime - startTime,
|
|
1654
|
-
status: "OK",
|
|
1655
|
-
attributes,
|
|
1656
|
-
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
1657
|
-
prompt_key: promptCtx?.promptKey,
|
|
1658
|
-
prompt_version: promptCtx?.promptVersion,
|
|
1659
|
-
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
1660
|
-
prompt_variant_index: promptCtx?.variantIndex
|
|
1661
|
-
}).catch(() => {
|
|
1662
|
-
});
|
|
1663
|
-
return response;
|
|
1664
|
-
} catch (error) {
|
|
1665
|
-
const endTime = Date.now();
|
|
1666
|
-
sendTrace({
|
|
1667
|
-
config_key: ctx.configKey,
|
|
1668
|
-
session_id: ctx.sessionId,
|
|
1669
|
-
customer_id: ctx.customerId,
|
|
1670
|
-
trace_id: traceId,
|
|
1671
|
-
span_id: spanId,
|
|
1672
|
-
parent_span_id: parentSpanId,
|
|
1673
|
-
name: "generateContent",
|
|
1674
|
-
kind: "llm",
|
|
1675
|
-
model: model.model || "gemini",
|
|
1676
|
-
start_time: new Date(startTime).toISOString(),
|
|
1677
|
-
end_time: new Date(endTime).toISOString(),
|
|
1678
|
-
duration_ms: endTime - startTime,
|
|
1679
|
-
status: "ERROR",
|
|
1680
|
-
error_message: error?.message,
|
|
1681
|
-
attributes: {
|
|
1682
|
-
"fallom.sdk_version": "2",
|
|
1683
|
-
"fallom.method": "generateContent"
|
|
1684
|
-
}
|
|
1685
|
-
}).catch(() => {
|
|
1686
|
-
});
|
|
1687
|
-
throw error;
|
|
1688
|
-
}
|
|
1689
|
-
};
|
|
1690
|
-
return model;
|
|
1691
|
-
}
|
|
1692
|
-
|
|
1693
|
-
// src/trace/wrappers/vercel-ai/generate-text.ts
|
|
1694
|
-
function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
1695
|
-
const ctx = sessionCtx;
|
|
1696
|
-
return async (...args) => {
|
|
1697
|
-
if (!isInitialized()) {
|
|
1698
|
-
return aiModule.generateText(...args);
|
|
1699
|
-
}
|
|
1700
|
-
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
1701
|
-
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
1702
|
-
const spanId = generateHexId(16);
|
|
1703
|
-
const parentSpanId = traceCtx?.parentSpanId;
|
|
1704
|
-
const params = args[0] || {};
|
|
1705
|
-
const startTime = Date.now();
|
|
1706
|
-
const captureContent2 = shouldCaptureContent();
|
|
1707
|
-
try {
|
|
1708
|
-
const result = await aiModule.generateText(...args);
|
|
1709
|
-
const endTime = Date.now();
|
|
1710
|
-
if (debug || isDebugMode()) {
|
|
1711
|
-
console.log(
|
|
1712
|
-
"\n\u{1F50D} [Fallom Debug] generateText raw result:",
|
|
1713
|
-
JSON.stringify(result, null, 2)
|
|
1714
|
-
);
|
|
1715
|
-
}
|
|
1716
|
-
const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
|
|
2184
|
+
try {
|
|
2185
|
+
const response = await originalCreate(...args);
|
|
2186
|
+
const endTime = Date.now();
|
|
1717
2187
|
const attributes = {
|
|
1718
2188
|
"fallom.sdk_version": "2",
|
|
1719
|
-
"fallom.method": "
|
|
2189
|
+
"fallom.method": "messages.create"
|
|
1720
2190
|
};
|
|
1721
2191
|
if (captureContent2) {
|
|
1722
2192
|
attributes["fallom.raw.request"] = JSON.stringify({
|
|
1723
|
-
prompt: params?.prompt,
|
|
1724
2193
|
messages: params?.messages,
|
|
1725
2194
|
system: params?.system,
|
|
1726
|
-
model:
|
|
1727
|
-
tools: params?.tools
|
|
1728
|
-
|
|
2195
|
+
model: params?.model,
|
|
2196
|
+
tools: params?.tools,
|
|
2197
|
+
tool_choice: params?.tool_choice
|
|
1729
2198
|
});
|
|
2199
|
+
const contentBlocks = response?.content || [];
|
|
2200
|
+
const textBlocks = contentBlocks.filter((b) => b.type === "text");
|
|
2201
|
+
const toolUseBlocks2 = contentBlocks.filter(
|
|
2202
|
+
(b) => b.type === "tool_use"
|
|
2203
|
+
);
|
|
1730
2204
|
attributes["fallom.raw.response"] = JSON.stringify({
|
|
1731
|
-
text:
|
|
1732
|
-
finishReason:
|
|
1733
|
-
responseId:
|
|
1734
|
-
|
|
1735
|
-
// Tool
|
|
1736
|
-
toolCalls:
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
stepType: step?.stepType,
|
|
1741
|
-
text: step?.text,
|
|
1742
|
-
finishReason: step?.finishReason,
|
|
1743
|
-
toolCalls: step?.toolCalls,
|
|
1744
|
-
toolResults: step?.toolResults,
|
|
1745
|
-
usage: step?.usage
|
|
2205
|
+
text: textBlocks.map((b) => b.text).join(""),
|
|
2206
|
+
finishReason: response?.stop_reason,
|
|
2207
|
+
responseId: response?.id,
|
|
2208
|
+
model: response?.model,
|
|
2209
|
+
// Tool calls - Anthropic uses tool_use content blocks
|
|
2210
|
+
toolCalls: toolUseBlocks2.map((b) => ({
|
|
2211
|
+
id: b.id,
|
|
2212
|
+
name: b.name,
|
|
2213
|
+
arguments: b.input
|
|
1746
2214
|
})),
|
|
1747
|
-
//
|
|
1748
|
-
|
|
2215
|
+
// Also send raw content for full fidelity
|
|
2216
|
+
content: contentBlocks
|
|
1749
2217
|
});
|
|
1750
2218
|
}
|
|
1751
|
-
if (
|
|
1752
|
-
attributes["fallom.raw.usage"] = JSON.stringify(
|
|
1753
|
-
}
|
|
1754
|
-
if (result?.experimental_providerMetadata) {
|
|
1755
|
-
attributes["fallom.raw.providerMetadata"] = JSON.stringify(
|
|
1756
|
-
result.experimental_providerMetadata
|
|
1757
|
-
);
|
|
2219
|
+
if (response?.usage) {
|
|
2220
|
+
attributes["fallom.raw.usage"] = JSON.stringify(response.usage);
|
|
1758
2221
|
}
|
|
2222
|
+
const waterfallTimings = {
|
|
2223
|
+
requestStart: 0,
|
|
2224
|
+
requestEnd: endTime - startTime,
|
|
2225
|
+
responseEnd: endTime - startTime,
|
|
2226
|
+
totalDurationMs: endTime - startTime,
|
|
2227
|
+
// Anthropic tool calls (if present)
|
|
2228
|
+
toolCalls: toolUseBlocks.map((b) => ({
|
|
2229
|
+
id: b.id,
|
|
2230
|
+
name: b.name,
|
|
2231
|
+
callTime: 0
|
|
2232
|
+
// All tool calls happen at once in non-streaming
|
|
2233
|
+
}))
|
|
2234
|
+
};
|
|
2235
|
+
attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
|
|
1759
2236
|
const promptCtx = getPromptContext();
|
|
1760
2237
|
sendTrace({
|
|
1761
2238
|
config_key: ctx.configKey,
|
|
@@ -1764,9 +2241,9 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
1764
2241
|
trace_id: traceId,
|
|
1765
2242
|
span_id: spanId,
|
|
1766
2243
|
parent_span_id: parentSpanId,
|
|
1767
|
-
name: "
|
|
2244
|
+
name: "messages.create",
|
|
1768
2245
|
kind: "llm",
|
|
1769
|
-
model:
|
|
2246
|
+
model: response?.model || params?.model,
|
|
1770
2247
|
start_time: new Date(startTime).toISOString(),
|
|
1771
2248
|
end_time: new Date(endTime).toISOString(),
|
|
1772
2249
|
duration_ms: endTime - startTime,
|
|
@@ -1779,10 +2256,9 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
1779
2256
|
prompt_variant_index: promptCtx?.variantIndex
|
|
1780
2257
|
}).catch(() => {
|
|
1781
2258
|
});
|
|
1782
|
-
return
|
|
2259
|
+
return response;
|
|
1783
2260
|
} catch (error) {
|
|
1784
2261
|
const endTime = Date.now();
|
|
1785
|
-
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
1786
2262
|
sendTrace({
|
|
1787
2263
|
config_key: ctx.configKey,
|
|
1788
2264
|
session_id: ctx.sessionId,
|
|
@@ -1790,9 +2266,9 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
1790
2266
|
trace_id: traceId,
|
|
1791
2267
|
span_id: spanId,
|
|
1792
2268
|
parent_span_id: parentSpanId,
|
|
1793
|
-
name: "
|
|
2269
|
+
name: "messages.create",
|
|
1794
2270
|
kind: "llm",
|
|
1795
|
-
model:
|
|
2271
|
+
model: params?.model,
|
|
1796
2272
|
start_time: new Date(startTime).toISOString(),
|
|
1797
2273
|
end_time: new Date(endTime).toISOString(),
|
|
1798
2274
|
duration_ms: endTime - startTime,
|
|
@@ -1800,262 +2276,78 @@ function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
1800
2276
|
error_message: error?.message,
|
|
1801
2277
|
attributes: {
|
|
1802
2278
|
"fallom.sdk_version": "2",
|
|
1803
|
-
"fallom.method": "
|
|
1804
|
-
"fallom.raw.request": JSON.stringify({
|
|
1805
|
-
prompt: params?.prompt,
|
|
1806
|
-
messages: params?.messages,
|
|
1807
|
-
system: params?.system,
|
|
1808
|
-
model: modelId
|
|
1809
|
-
})
|
|
2279
|
+
"fallom.method": "messages.create"
|
|
1810
2280
|
}
|
|
1811
2281
|
}).catch(() => {
|
|
1812
2282
|
});
|
|
1813
2283
|
throw error;
|
|
1814
2284
|
}
|
|
1815
2285
|
};
|
|
2286
|
+
return client;
|
|
1816
2287
|
}
|
|
1817
2288
|
|
|
1818
|
-
// src/trace/wrappers/
|
|
1819
|
-
function
|
|
1820
|
-
|
|
1821
|
-
}
|
|
1822
|
-
function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
1823
|
-
const ctx = sessionCtx;
|
|
1824
|
-
return async (...args) => {
|
|
1825
|
-
const params = args[0] || {};
|
|
1826
|
-
const startTime = Date.now();
|
|
1827
|
-
const captureContent2 = shouldCaptureContent();
|
|
1828
|
-
const result = await aiModule.streamText(...args);
|
|
1829
|
-
if (!isInitialized()) {
|
|
1830
|
-
return result;
|
|
1831
|
-
}
|
|
1832
|
-
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
1833
|
-
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
1834
|
-
const spanId = generateHexId(16);
|
|
1835
|
-
const parentSpanId = traceCtx?.parentSpanId;
|
|
1836
|
-
let firstTokenTime = null;
|
|
1837
|
-
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
1838
|
-
if (result?.usage) {
|
|
1839
|
-
Promise.all([
|
|
1840
|
-
result.usage.catch(() => null),
|
|
1841
|
-
result.text?.catch(() => null),
|
|
1842
|
-
result.finishReason?.catch(() => null),
|
|
1843
|
-
result.toolCalls?.catch(() => null),
|
|
1844
|
-
result.toolResults?.catch(() => null),
|
|
1845
|
-
result.steps?.catch(() => null),
|
|
1846
|
-
result.responseMessages?.catch(() => null)
|
|
1847
|
-
]).then(
|
|
1848
|
-
async ([
|
|
1849
|
-
rawUsage,
|
|
1850
|
-
responseText,
|
|
1851
|
-
finishReason,
|
|
1852
|
-
toolCalls,
|
|
1853
|
-
toolResults,
|
|
1854
|
-
steps,
|
|
1855
|
-
responseMessages
|
|
1856
|
-
]) => {
|
|
1857
|
-
const endTime = Date.now();
|
|
1858
|
-
if (debug || isDebugMode()) {
|
|
1859
|
-
console.log(
|
|
1860
|
-
"\n\u{1F50D} [Fallom Debug] streamText raw usage:",
|
|
1861
|
-
JSON.stringify(rawUsage, null, 2)
|
|
1862
|
-
);
|
|
1863
|
-
console.log(
|
|
1864
|
-
"\u{1F50D} [Fallom Debug] streamText response text:",
|
|
1865
|
-
responseText?.slice(0, 100)
|
|
1866
|
-
);
|
|
1867
|
-
console.log(
|
|
1868
|
-
"\u{1F50D} [Fallom Debug] streamText finish reason:",
|
|
1869
|
-
finishReason
|
|
1870
|
-
);
|
|
1871
|
-
console.log(
|
|
1872
|
-
"\u{1F50D} [Fallom Debug] streamText toolCalls:",
|
|
1873
|
-
JSON.stringify(toolCalls, null, 2)
|
|
1874
|
-
);
|
|
1875
|
-
console.log(
|
|
1876
|
-
"\u{1F50D} [Fallom Debug] streamText steps count:",
|
|
1877
|
-
steps?.length
|
|
1878
|
-
);
|
|
1879
|
-
}
|
|
1880
|
-
let providerMetadata = result?.experimental_providerMetadata;
|
|
1881
|
-
if (providerMetadata && typeof providerMetadata.then === "function") {
|
|
1882
|
-
try {
|
|
1883
|
-
providerMetadata = await providerMetadata;
|
|
1884
|
-
} catch {
|
|
1885
|
-
providerMetadata = void 0;
|
|
1886
|
-
}
|
|
1887
|
-
}
|
|
1888
|
-
const attributes = {
|
|
1889
|
-
"fallom.sdk_version": "2",
|
|
1890
|
-
"fallom.method": "streamText",
|
|
1891
|
-
"fallom.is_streaming": true
|
|
1892
|
-
};
|
|
1893
|
-
if (captureContent2) {
|
|
1894
|
-
attributes["fallom.raw.request"] = JSON.stringify({
|
|
1895
|
-
prompt: params?.prompt,
|
|
1896
|
-
messages: params?.messages,
|
|
1897
|
-
system: params?.system,
|
|
1898
|
-
model: modelId,
|
|
1899
|
-
tools: params?.tools ? Object.keys(params.tools) : void 0,
|
|
1900
|
-
maxSteps: params?.maxSteps
|
|
1901
|
-
});
|
|
1902
|
-
attributes["fallom.raw.response"] = JSON.stringify({
|
|
1903
|
-
text: responseText,
|
|
1904
|
-
finishReason,
|
|
1905
|
-
// Tool call data - send everything!
|
|
1906
|
-
toolCalls,
|
|
1907
|
-
toolResults,
|
|
1908
|
-
// Multi-step agent data
|
|
1909
|
-
steps: steps?.map((step) => ({
|
|
1910
|
-
stepType: step?.stepType,
|
|
1911
|
-
text: step?.text,
|
|
1912
|
-
finishReason: step?.finishReason,
|
|
1913
|
-
toolCalls: step?.toolCalls,
|
|
1914
|
-
toolResults: step?.toolResults,
|
|
1915
|
-
usage: step?.usage
|
|
1916
|
-
})),
|
|
1917
|
-
// Response messages (includes tool call/result messages)
|
|
1918
|
-
responseMessages
|
|
1919
|
-
});
|
|
1920
|
-
}
|
|
1921
|
-
if (rawUsage) {
|
|
1922
|
-
attributes["fallom.raw.usage"] = JSON.stringify(rawUsage);
|
|
1923
|
-
}
|
|
1924
|
-
if (providerMetadata) {
|
|
1925
|
-
attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
|
|
1926
|
-
}
|
|
1927
|
-
if (firstTokenTime) {
|
|
1928
|
-
attributes["fallom.time_to_first_token_ms"] = firstTokenTime - startTime;
|
|
1929
|
-
}
|
|
1930
|
-
const promptCtx = getPromptContext();
|
|
1931
|
-
sendTrace({
|
|
1932
|
-
config_key: ctx.configKey,
|
|
1933
|
-
session_id: ctx.sessionId,
|
|
1934
|
-
customer_id: ctx.customerId,
|
|
1935
|
-
trace_id: traceId,
|
|
1936
|
-
span_id: spanId,
|
|
1937
|
-
parent_span_id: parentSpanId,
|
|
1938
|
-
name: "streamText",
|
|
1939
|
-
kind: "llm",
|
|
1940
|
-
model: modelId,
|
|
1941
|
-
start_time: new Date(startTime).toISOString(),
|
|
1942
|
-
end_time: new Date(endTime).toISOString(),
|
|
1943
|
-
duration_ms: endTime - startTime,
|
|
1944
|
-
status: "OK",
|
|
1945
|
-
time_to_first_token_ms: firstTokenTime ? firstTokenTime - startTime : void 0,
|
|
1946
|
-
is_streaming: true,
|
|
1947
|
-
attributes,
|
|
1948
|
-
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
1949
|
-
prompt_key: promptCtx?.promptKey,
|
|
1950
|
-
prompt_version: promptCtx?.promptVersion,
|
|
1951
|
-
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
1952
|
-
prompt_variant_index: promptCtx?.variantIndex
|
|
1953
|
-
}).catch(() => {
|
|
1954
|
-
});
|
|
1955
|
-
}
|
|
1956
|
-
).catch((error) => {
|
|
1957
|
-
const endTime = Date.now();
|
|
1958
|
-
log3("\u274C streamText error:", error?.message);
|
|
1959
|
-
sendTrace({
|
|
1960
|
-
config_key: ctx.configKey,
|
|
1961
|
-
session_id: ctx.sessionId,
|
|
1962
|
-
customer_id: ctx.customerId,
|
|
1963
|
-
trace_id: traceId,
|
|
1964
|
-
span_id: spanId,
|
|
1965
|
-
parent_span_id: parentSpanId,
|
|
1966
|
-
name: "streamText",
|
|
1967
|
-
kind: "llm",
|
|
1968
|
-
model: modelId,
|
|
1969
|
-
start_time: new Date(startTime).toISOString(),
|
|
1970
|
-
end_time: new Date(endTime).toISOString(),
|
|
1971
|
-
duration_ms: endTime - startTime,
|
|
1972
|
-
status: "ERROR",
|
|
1973
|
-
error_message: error?.message,
|
|
1974
|
-
attributes: {
|
|
1975
|
-
"fallom.sdk_version": "2",
|
|
1976
|
-
"fallom.method": "streamText",
|
|
1977
|
-
"fallom.is_streaming": true
|
|
1978
|
-
}
|
|
1979
|
-
}).catch(() => {
|
|
1980
|
-
});
|
|
1981
|
-
});
|
|
1982
|
-
}
|
|
1983
|
-
if (result?.textStream) {
|
|
1984
|
-
const originalTextStream = result.textStream;
|
|
1985
|
-
const wrappedTextStream = (async function* () {
|
|
1986
|
-
for await (const chunk of originalTextStream) {
|
|
1987
|
-
if (!firstTokenTime) {
|
|
1988
|
-
firstTokenTime = Date.now();
|
|
1989
|
-
log3("\u23F1\uFE0F Time to first token:", firstTokenTime - startTime, "ms");
|
|
1990
|
-
}
|
|
1991
|
-
yield chunk;
|
|
1992
|
-
}
|
|
1993
|
-
})();
|
|
1994
|
-
return new Proxy(result, {
|
|
1995
|
-
get(target, prop) {
|
|
1996
|
-
if (prop === "textStream") {
|
|
1997
|
-
return wrappedTextStream;
|
|
1998
|
-
}
|
|
1999
|
-
return target[prop];
|
|
2000
|
-
}
|
|
2001
|
-
});
|
|
2002
|
-
}
|
|
2003
|
-
return result;
|
|
2004
|
-
};
|
|
2005
|
-
}
|
|
2006
|
-
|
|
2007
|
-
// src/trace/wrappers/vercel-ai/generate-object.ts
|
|
2008
|
-
function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
2289
|
+
// src/trace/wrappers/google-ai.ts
|
|
2290
|
+
function wrapGoogleAI(model, sessionCtx) {
|
|
2291
|
+
const originalGenerateContent = model.generateContent.bind(model);
|
|
2009
2292
|
const ctx = sessionCtx;
|
|
2010
|
-
|
|
2293
|
+
model.generateContent = async function(...args) {
|
|
2011
2294
|
if (!isInitialized()) {
|
|
2012
|
-
return
|
|
2295
|
+
return originalGenerateContent(...args);
|
|
2013
2296
|
}
|
|
2014
2297
|
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
2015
2298
|
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
2016
2299
|
const spanId = generateHexId(16);
|
|
2017
2300
|
const parentSpanId = traceCtx?.parentSpanId;
|
|
2018
|
-
const
|
|
2301
|
+
const request = args[0];
|
|
2019
2302
|
const startTime = Date.now();
|
|
2020
2303
|
const captureContent2 = shouldCaptureContent();
|
|
2021
2304
|
try {
|
|
2022
|
-
const
|
|
2305
|
+
const response = await originalGenerateContent(...args);
|
|
2023
2306
|
const endTime = Date.now();
|
|
2024
|
-
|
|
2025
|
-
console.log(
|
|
2026
|
-
"\n\u{1F50D} [Fallom Debug] generateObject raw result:",
|
|
2027
|
-
JSON.stringify(result, null, 2)
|
|
2028
|
-
);
|
|
2029
|
-
}
|
|
2030
|
-
const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
|
|
2307
|
+
const result = response?.response || response;
|
|
2031
2308
|
const attributes = {
|
|
2032
2309
|
"fallom.sdk_version": "2",
|
|
2033
|
-
"fallom.method": "
|
|
2310
|
+
"fallom.method": "generateContent"
|
|
2034
2311
|
};
|
|
2035
2312
|
if (captureContent2) {
|
|
2036
|
-
attributes["fallom.raw.request"] = JSON.stringify(
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2313
|
+
attributes["fallom.raw.request"] = JSON.stringify(request);
|
|
2314
|
+
const candidates = result?.candidates || [];
|
|
2315
|
+
const functionCalls2 = [];
|
|
2316
|
+
for (const candidate of candidates) {
|
|
2317
|
+
const parts = candidate?.content?.parts || [];
|
|
2318
|
+
for (const part of parts) {
|
|
2319
|
+
if (part.functionCall) {
|
|
2320
|
+
functionCalls2.push({
|
|
2321
|
+
name: part.functionCall.name,
|
|
2322
|
+
arguments: part.functionCall.args
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2044
2327
|
attributes["fallom.raw.response"] = JSON.stringify({
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2328
|
+
text: result?.text?.(),
|
|
2329
|
+
candidates: result?.candidates,
|
|
2330
|
+
finishReason: candidates[0]?.finishReason,
|
|
2331
|
+
// Tool/function calls - Google uses functionCall in parts
|
|
2332
|
+
toolCalls: functionCalls2.length > 0 ? functionCalls2 : void 0
|
|
2049
2333
|
});
|
|
2050
2334
|
}
|
|
2051
|
-
if (result?.
|
|
2052
|
-
attributes["fallom.raw.usage"] = JSON.stringify(result.
|
|
2053
|
-
}
|
|
2054
|
-
if (result?.experimental_providerMetadata) {
|
|
2055
|
-
attributes["fallom.raw.providerMetadata"] = JSON.stringify(
|
|
2056
|
-
result.experimental_providerMetadata
|
|
2057
|
-
);
|
|
2335
|
+
if (result?.usageMetadata) {
|
|
2336
|
+
attributes["fallom.raw.usage"] = JSON.stringify(result.usageMetadata);
|
|
2058
2337
|
}
|
|
2338
|
+
const waterfallTimings = {
|
|
2339
|
+
requestStart: 0,
|
|
2340
|
+
requestEnd: endTime - startTime,
|
|
2341
|
+
responseEnd: endTime - startTime,
|
|
2342
|
+
totalDurationMs: endTime - startTime,
|
|
2343
|
+
// Google AI function calls (if present)
|
|
2344
|
+
toolCalls: functionCalls.map((fc) => ({
|
|
2345
|
+
name: fc.name,
|
|
2346
|
+
callTime: 0
|
|
2347
|
+
// All tool calls happen at once in non-streaming
|
|
2348
|
+
}))
|
|
2349
|
+
};
|
|
2350
|
+
attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
|
|
2059
2351
|
const promptCtx = getPromptContext();
|
|
2060
2352
|
sendTrace({
|
|
2061
2353
|
config_key: ctx.configKey,
|
|
@@ -2064,9 +2356,9 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
2064
2356
|
trace_id: traceId,
|
|
2065
2357
|
span_id: spanId,
|
|
2066
2358
|
parent_span_id: parentSpanId,
|
|
2067
|
-
name: "
|
|
2359
|
+
name: "generateContent",
|
|
2068
2360
|
kind: "llm",
|
|
2069
|
-
model:
|
|
2361
|
+
model: model.model || "gemini",
|
|
2070
2362
|
start_time: new Date(startTime).toISOString(),
|
|
2071
2363
|
end_time: new Date(endTime).toISOString(),
|
|
2072
2364
|
duration_ms: endTime - startTime,
|
|
@@ -2079,10 +2371,9 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
2079
2371
|
prompt_variant_index: promptCtx?.variantIndex
|
|
2080
2372
|
}).catch(() => {
|
|
2081
2373
|
});
|
|
2082
|
-
return
|
|
2374
|
+
return response;
|
|
2083
2375
|
} catch (error) {
|
|
2084
2376
|
const endTime = Date.now();
|
|
2085
|
-
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
2086
2377
|
sendTrace({
|
|
2087
2378
|
config_key: ctx.configKey,
|
|
2088
2379
|
session_id: ctx.sessionId,
|
|
@@ -2090,9 +2381,9 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
2090
2381
|
trace_id: traceId,
|
|
2091
2382
|
span_id: spanId,
|
|
2092
2383
|
parent_span_id: parentSpanId,
|
|
2093
|
-
name: "
|
|
2384
|
+
name: "generateContent",
|
|
2094
2385
|
kind: "llm",
|
|
2095
|
-
model:
|
|
2386
|
+
model: model.model || "gemini",
|
|
2096
2387
|
start_time: new Date(startTime).toISOString(),
|
|
2097
2388
|
end_time: new Date(endTime).toISOString(),
|
|
2098
2389
|
duration_ms: endTime - startTime,
|
|
@@ -2100,169 +2391,239 @@ function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
|
2100
2391
|
error_message: error?.message,
|
|
2101
2392
|
attributes: {
|
|
2102
2393
|
"fallom.sdk_version": "2",
|
|
2103
|
-
"fallom.method": "
|
|
2394
|
+
"fallom.method": "generateContent"
|
|
2104
2395
|
}
|
|
2105
2396
|
}).catch(() => {
|
|
2106
2397
|
});
|
|
2107
2398
|
throw error;
|
|
2108
2399
|
}
|
|
2109
2400
|
};
|
|
2401
|
+
return model;
|
|
2110
2402
|
}
|
|
2111
2403
|
|
|
2112
|
-
// src/trace/wrappers/vercel-ai/
|
|
2113
|
-
function
|
|
2404
|
+
// src/trace/wrappers/vercel-ai/generate-text.ts
|
|
2405
|
+
function createGenerateTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
2114
2406
|
const ctx = sessionCtx;
|
|
2115
2407
|
return async (...args) => {
|
|
2116
|
-
const params = args[0] || {};
|
|
2117
|
-
const startTime = Date.now();
|
|
2118
|
-
const captureContent2 = shouldCaptureContent();
|
|
2119
|
-
const result = await aiModule.streamObject(...args);
|
|
2120
|
-
if (!isInitialized()) {
|
|
2121
|
-
return result;
|
|
2122
|
-
}
|
|
2123
|
-
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
2124
|
-
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
2125
|
-
const spanId = generateHexId(16);
|
|
2126
|
-
const parentSpanId = traceCtx?.parentSpanId;
|
|
2127
|
-
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
2128
|
-
if (result?.usage) {
|
|
2129
|
-
Promise.all([
|
|
2130
|
-
result.usage.catch(() => null),
|
|
2131
|
-
result.object?.catch(() => null),
|
|
2132
|
-
result.finishReason?.catch(() => null)
|
|
2133
|
-
]).then(async ([rawUsage, responseObject, finishReason]) => {
|
|
2134
|
-
const endTime = Date.now();
|
|
2135
|
-
if (debug || isDebugMode()) {
|
|
2136
|
-
console.log("\n\u{1F50D} [Fallom Debug] streamObject raw usage:", JSON.stringify(rawUsage, null, 2));
|
|
2137
|
-
console.log("\u{1F50D} [Fallom Debug] streamObject response object:", JSON.stringify(responseObject)?.slice(0, 100));
|
|
2138
|
-
console.log("\u{1F50D} [Fallom Debug] streamObject finish reason:", finishReason);
|
|
2139
|
-
}
|
|
2140
|
-
let providerMetadata = result?.experimental_providerMetadata;
|
|
2141
|
-
if (providerMetadata && typeof providerMetadata.then === "function") {
|
|
2142
|
-
try {
|
|
2143
|
-
providerMetadata = await providerMetadata;
|
|
2144
|
-
} catch {
|
|
2145
|
-
providerMetadata = void 0;
|
|
2146
|
-
}
|
|
2147
|
-
}
|
|
2148
|
-
const attributes = {
|
|
2149
|
-
"fallom.sdk_version": "2",
|
|
2150
|
-
"fallom.method": "streamObject",
|
|
2151
|
-
"fallom.is_streaming": true
|
|
2152
|
-
};
|
|
2153
|
-
if (captureContent2) {
|
|
2154
|
-
attributes["fallom.raw.request"] = JSON.stringify({
|
|
2155
|
-
prompt: params?.prompt,
|
|
2156
|
-
messages: params?.messages,
|
|
2157
|
-
system: params?.system,
|
|
2158
|
-
model: modelId,
|
|
2159
|
-
schema: params?.schema ? "provided" : void 0
|
|
2160
|
-
});
|
|
2161
|
-
if (responseObject || finishReason) {
|
|
2162
|
-
attributes["fallom.raw.response"] = JSON.stringify({
|
|
2163
|
-
object: responseObject,
|
|
2164
|
-
finishReason
|
|
2165
|
-
});
|
|
2166
|
-
}
|
|
2167
|
-
}
|
|
2168
|
-
if (rawUsage) {
|
|
2169
|
-
attributes["fallom.raw.usage"] = JSON.stringify(rawUsage);
|
|
2170
|
-
}
|
|
2171
|
-
if (providerMetadata) {
|
|
2172
|
-
attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
|
|
2173
|
-
}
|
|
2174
|
-
const promptCtx = getPromptContext();
|
|
2175
|
-
sendTrace({
|
|
2176
|
-
config_key: ctx.configKey,
|
|
2177
|
-
session_id: ctx.sessionId,
|
|
2178
|
-
customer_id: ctx.customerId,
|
|
2179
|
-
trace_id: traceId,
|
|
2180
|
-
span_id: spanId,
|
|
2181
|
-
parent_span_id: parentSpanId,
|
|
2182
|
-
name: "streamObject",
|
|
2183
|
-
kind: "llm",
|
|
2184
|
-
model: modelId,
|
|
2185
|
-
start_time: new Date(startTime).toISOString(),
|
|
2186
|
-
end_time: new Date(endTime).toISOString(),
|
|
2187
|
-
duration_ms: endTime - startTime,
|
|
2188
|
-
status: "OK",
|
|
2189
|
-
is_streaming: true,
|
|
2190
|
-
attributes,
|
|
2191
|
-
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
2192
|
-
prompt_key: promptCtx?.promptKey,
|
|
2193
|
-
prompt_version: promptCtx?.promptVersion,
|
|
2194
|
-
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
2195
|
-
prompt_variant_index: promptCtx?.variantIndex
|
|
2196
|
-
}).catch(() => {
|
|
2197
|
-
});
|
|
2198
|
-
}).catch((error) => {
|
|
2199
|
-
const endTime = Date.now();
|
|
2200
|
-
sendTrace({
|
|
2201
|
-
config_key: ctx.configKey,
|
|
2202
|
-
session_id: ctx.sessionId,
|
|
2203
|
-
customer_id: ctx.customerId,
|
|
2204
|
-
trace_id: traceId,
|
|
2205
|
-
span_id: spanId,
|
|
2206
|
-
parent_span_id: parentSpanId,
|
|
2207
|
-
name: "streamObject",
|
|
2208
|
-
kind: "llm",
|
|
2209
|
-
model: modelId,
|
|
2210
|
-
start_time: new Date(startTime).toISOString(),
|
|
2211
|
-
end_time: new Date(endTime).toISOString(),
|
|
2212
|
-
duration_ms: endTime - startTime,
|
|
2213
|
-
status: "ERROR",
|
|
2214
|
-
error_message: error?.message,
|
|
2215
|
-
attributes: {
|
|
2216
|
-
"fallom.sdk_version": "2",
|
|
2217
|
-
"fallom.method": "streamObject",
|
|
2218
|
-
"fallom.is_streaming": true
|
|
2219
|
-
}
|
|
2220
|
-
}).catch(() => {
|
|
2221
|
-
});
|
|
2222
|
-
});
|
|
2223
|
-
}
|
|
2224
|
-
return result;
|
|
2225
|
-
};
|
|
2226
|
-
}
|
|
2227
|
-
|
|
2228
|
-
// src/trace/wrappers/vercel-ai/index.ts
|
|
2229
|
-
function wrapAISDK(ai, sessionCtx, options) {
|
|
2230
|
-
const debug = options?.debug ?? false;
|
|
2231
|
-
return {
|
|
2232
|
-
generateText: createGenerateTextWrapper(ai, sessionCtx, debug),
|
|
2233
|
-
streamText: createStreamTextWrapper(ai, sessionCtx, debug),
|
|
2234
|
-
generateObject: ai.generateObject ? createGenerateObjectWrapper(ai, sessionCtx, debug) : void 0,
|
|
2235
|
-
streamObject: ai.streamObject ? createStreamObjectWrapper(ai, sessionCtx, debug) : void 0
|
|
2236
|
-
};
|
|
2237
|
-
}
|
|
2238
|
-
|
|
2239
|
-
// src/trace/wrappers/mastra.ts
|
|
2240
|
-
function wrapMastraAgent(agent, sessionCtx) {
|
|
2241
|
-
const originalGenerate = agent.generate.bind(agent);
|
|
2242
|
-
const ctx = sessionCtx;
|
|
2243
|
-
agent.generate = async function(...args) {
|
|
2244
2408
|
if (!isInitialized()) {
|
|
2245
|
-
return
|
|
2409
|
+
return aiModule.generateText(...args);
|
|
2246
2410
|
}
|
|
2247
2411
|
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
2248
2412
|
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
2249
2413
|
const spanId = generateHexId(16);
|
|
2250
2414
|
const parentSpanId = traceCtx?.parentSpanId;
|
|
2251
|
-
const
|
|
2415
|
+
const params = args[0] || {};
|
|
2252
2416
|
const startTime = Date.now();
|
|
2253
2417
|
const captureContent2 = shouldCaptureContent();
|
|
2418
|
+
const toolTimings = /* @__PURE__ */ new Map();
|
|
2419
|
+
let wrappedParams = params;
|
|
2420
|
+
if (params.tools && typeof params.tools === "object") {
|
|
2421
|
+
const wrappedTools = {};
|
|
2422
|
+
for (const [toolName, tool] of Object.entries(
|
|
2423
|
+
params.tools
|
|
2424
|
+
)) {
|
|
2425
|
+
if (tool && typeof tool.execute === "function") {
|
|
2426
|
+
const originalExecute = tool.execute;
|
|
2427
|
+
wrappedTools[toolName] = {
|
|
2428
|
+
...tool,
|
|
2429
|
+
execute: async (...executeArgs) => {
|
|
2430
|
+
const toolStartTime = Date.now();
|
|
2431
|
+
const toolCallId = `${toolName}-${toolStartTime}`;
|
|
2432
|
+
try {
|
|
2433
|
+
const result = await originalExecute(...executeArgs);
|
|
2434
|
+
const toolEndTime = Date.now();
|
|
2435
|
+
toolTimings.set(toolCallId, {
|
|
2436
|
+
name: toolName,
|
|
2437
|
+
startTime: toolStartTime - startTime,
|
|
2438
|
+
// Relative to request start
|
|
2439
|
+
endTime: toolEndTime - startTime,
|
|
2440
|
+
duration: toolEndTime - toolStartTime
|
|
2441
|
+
});
|
|
2442
|
+
return result;
|
|
2443
|
+
} catch (error) {
|
|
2444
|
+
const toolEndTime = Date.now();
|
|
2445
|
+
toolTimings.set(toolCallId, {
|
|
2446
|
+
name: toolName,
|
|
2447
|
+
startTime: toolStartTime - startTime,
|
|
2448
|
+
endTime: toolEndTime - startTime,
|
|
2449
|
+
duration: toolEndTime - toolStartTime
|
|
2450
|
+
});
|
|
2451
|
+
throw error;
|
|
2452
|
+
}
|
|
2453
|
+
}
|
|
2454
|
+
};
|
|
2455
|
+
} else {
|
|
2456
|
+
wrappedTools[toolName] = tool;
|
|
2457
|
+
}
|
|
2458
|
+
}
|
|
2459
|
+
wrappedParams = { ...params, tools: wrappedTools };
|
|
2460
|
+
}
|
|
2254
2461
|
try {
|
|
2255
|
-
const result = await
|
|
2462
|
+
const result = await aiModule.generateText(wrappedParams);
|
|
2256
2463
|
const endTime = Date.now();
|
|
2464
|
+
if (debug || isDebugMode()) {
|
|
2465
|
+
console.log(
|
|
2466
|
+
"\n\u{1F50D} [Fallom Debug] generateText raw result:",
|
|
2467
|
+
JSON.stringify(result, null, 2)
|
|
2468
|
+
);
|
|
2469
|
+
}
|
|
2470
|
+
const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
|
|
2257
2471
|
const attributes = {
|
|
2258
2472
|
"fallom.sdk_version": "2",
|
|
2259
|
-
"fallom.method": "
|
|
2260
|
-
"fallom.agent_name": agent.name || "unknown"
|
|
2473
|
+
"fallom.method": "generateText"
|
|
2261
2474
|
};
|
|
2262
2475
|
if (captureContent2) {
|
|
2263
|
-
attributes["fallom.raw.request"] = JSON.stringify(
|
|
2264
|
-
|
|
2476
|
+
attributes["fallom.raw.request"] = JSON.stringify({
|
|
2477
|
+
prompt: params?.prompt,
|
|
2478
|
+
messages: params?.messages,
|
|
2479
|
+
system: params?.system,
|
|
2480
|
+
model: modelId,
|
|
2481
|
+
tools: params?.tools ? Object.keys(params.tools) : void 0,
|
|
2482
|
+
maxSteps: params?.maxSteps
|
|
2483
|
+
});
|
|
2484
|
+
const mapToolCall = (tc) => ({
|
|
2485
|
+
toolCallId: tc?.toolCallId,
|
|
2486
|
+
toolName: tc?.toolName,
|
|
2487
|
+
args: tc?.args,
|
|
2488
|
+
// The actual arguments passed to the tool!
|
|
2489
|
+
type: tc?.type
|
|
2490
|
+
});
|
|
2491
|
+
const mapToolResult = (tr) => ({
|
|
2492
|
+
toolCallId: tr?.toolCallId,
|
|
2493
|
+
toolName: tr?.toolName,
|
|
2494
|
+
result: tr?.result,
|
|
2495
|
+
// The actual result from the tool!
|
|
2496
|
+
type: tr?.type
|
|
2497
|
+
});
|
|
2498
|
+
attributes["fallom.raw.response"] = JSON.stringify({
|
|
2499
|
+
text: result?.text,
|
|
2500
|
+
finishReason: result?.finishReason,
|
|
2501
|
+
responseId: result?.response?.id,
|
|
2502
|
+
modelId: result?.response?.modelId,
|
|
2503
|
+
// Tool calls with FULL data (id, name, args)
|
|
2504
|
+
toolCalls: result?.toolCalls?.map(mapToolCall),
|
|
2505
|
+
// Tool results with FULL data (id, name, result)
|
|
2506
|
+
toolResults: result?.toolResults?.map(mapToolResult),
|
|
2507
|
+
// Multi-step agent data with FULL tool info including timestamps
|
|
2508
|
+
steps: result?.steps?.map((step) => ({
|
|
2509
|
+
stepType: step?.stepType,
|
|
2510
|
+
text: step?.text,
|
|
2511
|
+
finishReason: step?.finishReason,
|
|
2512
|
+
toolCalls: step?.toolCalls?.map(mapToolCall),
|
|
2513
|
+
toolResults: step?.toolResults?.map(mapToolResult),
|
|
2514
|
+
usage: step?.usage,
|
|
2515
|
+
// Step-level timing from Vercel AI SDK
|
|
2516
|
+
timestamp: step?.response?.timestamp,
|
|
2517
|
+
responseId: step?.response?.id
|
|
2518
|
+
})),
|
|
2519
|
+
// Response messages (includes tool call/result messages)
|
|
2520
|
+
responseMessages: result?.responseMessages
|
|
2521
|
+
});
|
|
2522
|
+
}
|
|
2523
|
+
if (result?.usage) {
|
|
2524
|
+
attributes["fallom.raw.usage"] = JSON.stringify(result.usage);
|
|
2525
|
+
}
|
|
2526
|
+
if (result?.experimental_providerMetadata) {
|
|
2527
|
+
attributes["fallom.raw.providerMetadata"] = JSON.stringify(
|
|
2528
|
+
result.experimental_providerMetadata
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
const totalDurationMs = endTime - startTime;
|
|
2532
|
+
const sortedToolTimings = Array.from(toolTimings.values()).sort(
|
|
2533
|
+
(a, b) => a.startTime - b.startTime
|
|
2534
|
+
);
|
|
2535
|
+
const waterfallTimings = {
|
|
2536
|
+
requestStart: 0,
|
|
2537
|
+
responseEnd: totalDurationMs,
|
|
2538
|
+
totalDurationMs,
|
|
2539
|
+
phases: [],
|
|
2540
|
+
// Include actual tool timings for verification
|
|
2541
|
+
toolTimings: sortedToolTimings
|
|
2542
|
+
};
|
|
2543
|
+
if (sortedToolTimings.length > 0) {
|
|
2544
|
+
const firstToolStart = Math.min(
|
|
2545
|
+
...sortedToolTimings.map((t) => t.startTime)
|
|
2546
|
+
);
|
|
2547
|
+
const lastToolEnd = Math.max(
|
|
2548
|
+
...sortedToolTimings.map((t) => t.endTime)
|
|
2549
|
+
);
|
|
2550
|
+
if (firstToolStart > 10) {
|
|
2551
|
+
waterfallTimings.phases.push({
|
|
2552
|
+
type: "llm",
|
|
2553
|
+
label: "LLM Call 1 (decides tools)",
|
|
2554
|
+
startMs: 0,
|
|
2555
|
+
endMs: firstToolStart,
|
|
2556
|
+
durationMs: firstToolStart,
|
|
2557
|
+
accurate: true
|
|
2558
|
+
});
|
|
2559
|
+
}
|
|
2560
|
+
sortedToolTimings.forEach((toolTiming) => {
|
|
2561
|
+
waterfallTimings.phases.push({
|
|
2562
|
+
type: "tool",
|
|
2563
|
+
label: `${toolTiming.name}()`,
|
|
2564
|
+
startMs: toolTiming.startTime,
|
|
2565
|
+
endMs: toolTiming.endTime,
|
|
2566
|
+
durationMs: toolTiming.duration,
|
|
2567
|
+
accurate: true
|
|
2568
|
+
// This is REAL measured timing!
|
|
2569
|
+
});
|
|
2570
|
+
});
|
|
2571
|
+
const finalResponseDuration = totalDurationMs - lastToolEnd;
|
|
2572
|
+
if (finalResponseDuration > 10) {
|
|
2573
|
+
waterfallTimings.phases.push({
|
|
2574
|
+
type: "response",
|
|
2575
|
+
label: "LLM Call 2 \u2192 Final Response",
|
|
2576
|
+
startMs: lastToolEnd,
|
|
2577
|
+
endMs: totalDurationMs,
|
|
2578
|
+
durationMs: finalResponseDuration,
|
|
2579
|
+
accurate: true
|
|
2580
|
+
});
|
|
2581
|
+
}
|
|
2582
|
+
} else if (result?.steps && result.steps.length > 0) {
|
|
2583
|
+
const steps = result.steps;
|
|
2584
|
+
const stepDuration = Math.round(totalDurationMs / steps.length);
|
|
2585
|
+
steps.forEach((step, idx) => {
|
|
2586
|
+
const hasTools = step?.toolCalls && step.toolCalls.length > 0;
|
|
2587
|
+
const isFinalStep = step?.finishReason === "stop";
|
|
2588
|
+
const stepStart = idx * stepDuration;
|
|
2589
|
+
const stepEnd = Math.min((idx + 1) * stepDuration, totalDurationMs);
|
|
2590
|
+
if (hasTools) {
|
|
2591
|
+
waterfallTimings.phases.push({
|
|
2592
|
+
type: "llm",
|
|
2593
|
+
label: `Step ${idx + 1}: LLM + Tools`,
|
|
2594
|
+
startMs: stepStart,
|
|
2595
|
+
endMs: stepEnd,
|
|
2596
|
+
durationMs: stepEnd - stepStart,
|
|
2597
|
+
accurate: false,
|
|
2598
|
+
note: "Tool timing not captured - combined step"
|
|
2599
|
+
});
|
|
2600
|
+
} else if (isFinalStep) {
|
|
2601
|
+
waterfallTimings.phases.push({
|
|
2602
|
+
type: "response",
|
|
2603
|
+
label: `Step ${idx + 1}: Final Response`,
|
|
2604
|
+
startMs: stepStart,
|
|
2605
|
+
endMs: stepEnd,
|
|
2606
|
+
durationMs: stepEnd - stepStart,
|
|
2607
|
+
accurate: true
|
|
2608
|
+
});
|
|
2609
|
+
}
|
|
2610
|
+
});
|
|
2611
|
+
}
|
|
2612
|
+
if (result?.steps) {
|
|
2613
|
+
waterfallTimings.steps = result.steps.map((step, idx) => ({
|
|
2614
|
+
stepIndex: idx,
|
|
2615
|
+
stepType: step?.stepType,
|
|
2616
|
+
finishReason: step?.finishReason,
|
|
2617
|
+
timestamp: step?.response?.timestamp,
|
|
2618
|
+
toolCalls: step?.toolCalls?.map((tc) => ({
|
|
2619
|
+
id: tc?.toolCallId,
|
|
2620
|
+
name: tc?.toolName
|
|
2621
|
+
})),
|
|
2622
|
+
usage: step?.usage
|
|
2623
|
+
}));
|
|
2265
2624
|
}
|
|
2625
|
+
attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
|
|
2626
|
+
const promptCtx = getPromptContext();
|
|
2266
2627
|
sendTrace({
|
|
2267
2628
|
config_key: ctx.configKey,
|
|
2268
2629
|
session_id: ctx.sessionId,
|
|
@@ -2270,18 +2631,25 @@ function wrapMastraAgent(agent, sessionCtx) {
|
|
|
2270
2631
|
trace_id: traceId,
|
|
2271
2632
|
span_id: spanId,
|
|
2272
2633
|
parent_span_id: parentSpanId,
|
|
2273
|
-
name:
|
|
2274
|
-
kind: "
|
|
2634
|
+
name: "generateText",
|
|
2635
|
+
kind: "llm",
|
|
2636
|
+
model: modelId,
|
|
2275
2637
|
start_time: new Date(startTime).toISOString(),
|
|
2276
2638
|
end_time: new Date(endTime).toISOString(),
|
|
2277
2639
|
duration_ms: endTime - startTime,
|
|
2278
2640
|
status: "OK",
|
|
2279
|
-
attributes
|
|
2641
|
+
attributes,
|
|
2642
|
+
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
2643
|
+
prompt_key: promptCtx?.promptKey,
|
|
2644
|
+
prompt_version: promptCtx?.promptVersion,
|
|
2645
|
+
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
2646
|
+
prompt_variant_index: promptCtx?.variantIndex
|
|
2280
2647
|
}).catch(() => {
|
|
2281
2648
|
});
|
|
2282
2649
|
return result;
|
|
2283
2650
|
} catch (error) {
|
|
2284
2651
|
const endTime = Date.now();
|
|
2652
|
+
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
2285
2653
|
sendTrace({
|
|
2286
2654
|
config_key: ctx.configKey,
|
|
2287
2655
|
session_id: ctx.sessionId,
|
|
@@ -2289,8 +2657,9 @@ function wrapMastraAgent(agent, sessionCtx) {
|
|
|
2289
2657
|
trace_id: traceId,
|
|
2290
2658
|
span_id: spanId,
|
|
2291
2659
|
parent_span_id: parentSpanId,
|
|
2292
|
-
name:
|
|
2293
|
-
kind: "
|
|
2660
|
+
name: "generateText",
|
|
2661
|
+
kind: "llm",
|
|
2662
|
+
model: modelId,
|
|
2294
2663
|
start_time: new Date(startTime).toISOString(),
|
|
2295
2664
|
end_time: new Date(endTime).toISOString(),
|
|
2296
2665
|
duration_ms: endTime - startTime,
|
|
@@ -2298,798 +2667,859 @@ function wrapMastraAgent(agent, sessionCtx) {
|
|
|
2298
2667
|
error_message: error?.message,
|
|
2299
2668
|
attributes: {
|
|
2300
2669
|
"fallom.sdk_version": "2",
|
|
2301
|
-
"fallom.method": "
|
|
2302
|
-
"fallom.
|
|
2670
|
+
"fallom.method": "generateText",
|
|
2671
|
+
"fallom.raw.request": JSON.stringify({
|
|
2672
|
+
prompt: params?.prompt,
|
|
2673
|
+
messages: params?.messages,
|
|
2674
|
+
system: params?.system,
|
|
2675
|
+
model: modelId
|
|
2676
|
+
})
|
|
2303
2677
|
}
|
|
2304
2678
|
}).catch(() => {
|
|
2305
2679
|
});
|
|
2306
2680
|
throw error;
|
|
2307
2681
|
}
|
|
2308
2682
|
};
|
|
2309
|
-
return agent;
|
|
2310
2683
|
}
|
|
2311
2684
|
|
|
2312
|
-
// src/trace/
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
};
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
tracedModel.doGenerate = async function(...args) {
|
|
2355
|
-
if (!isInitialized()) return originalDoGenerate(...args);
|
|
2356
|
-
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
2357
|
-
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
2358
|
-
const spanId = generateHexId(16);
|
|
2359
|
-
const startTime = Date.now();
|
|
2360
|
-
try {
|
|
2361
|
-
const result = await originalDoGenerate(...args);
|
|
2362
|
-
const endTime = Date.now();
|
|
2363
|
-
const modelId = model.modelId || "unknown";
|
|
2364
|
-
const usage = result?.usage || result?.rawResponse?.usage;
|
|
2365
|
-
sendTrace({
|
|
2366
|
-
config_key: ctx.configKey,
|
|
2367
|
-
session_id: ctx.sessionId,
|
|
2368
|
-
customer_id: ctx.customerId,
|
|
2369
|
-
trace_id: traceId,
|
|
2370
|
-
span_id: spanId,
|
|
2371
|
-
parent_span_id: traceCtx?.parentSpanId,
|
|
2372
|
-
name: "doGenerate",
|
|
2373
|
-
kind: "llm",
|
|
2374
|
-
model: modelId,
|
|
2375
|
-
start_time: new Date(startTime).toISOString(),
|
|
2376
|
-
end_time: new Date(endTime).toISOString(),
|
|
2377
|
-
duration_ms: endTime - startTime,
|
|
2378
|
-
status: "OK",
|
|
2379
|
-
attributes: {
|
|
2380
|
-
"fallom.sdk_version": "2",
|
|
2381
|
-
"fallom.method": "traceModel.doGenerate",
|
|
2382
|
-
...usage ? { "fallom.raw.usage": JSON.stringify(usage) } : {}
|
|
2685
|
+
// src/trace/wrappers/vercel-ai/stream-text.ts
|
|
2686
|
+
function log3(...args) {
|
|
2687
|
+
if (isDebugMode()) console.log("[Fallom]", ...args);
|
|
2688
|
+
}
|
|
2689
|
+
function createStreamTextWrapper(aiModule, sessionCtx, debug = false) {
|
|
2690
|
+
const ctx = sessionCtx;
|
|
2691
|
+
return async (...args) => {
|
|
2692
|
+
const params = args[0] || {};
|
|
2693
|
+
const startTime = Date.now();
|
|
2694
|
+
const captureContent2 = shouldCaptureContent();
|
|
2695
|
+
const toolTimings = /* @__PURE__ */ new Map();
|
|
2696
|
+
let wrappedParams = params;
|
|
2697
|
+
if (params.tools && typeof params.tools === "object") {
|
|
2698
|
+
const wrappedTools = {};
|
|
2699
|
+
for (const [toolName, tool] of Object.entries(params.tools)) {
|
|
2700
|
+
if (tool && typeof tool.execute === "function") {
|
|
2701
|
+
const originalExecute = tool.execute;
|
|
2702
|
+
wrappedTools[toolName] = {
|
|
2703
|
+
...tool,
|
|
2704
|
+
execute: async (...executeArgs) => {
|
|
2705
|
+
const toolStartTime = Date.now();
|
|
2706
|
+
const toolCallId = `${toolName}-${toolStartTime}`;
|
|
2707
|
+
try {
|
|
2708
|
+
const result2 = await originalExecute(...executeArgs);
|
|
2709
|
+
const toolEndTime = Date.now();
|
|
2710
|
+
toolTimings.set(toolCallId, {
|
|
2711
|
+
name: toolName,
|
|
2712
|
+
startTime: toolStartTime - startTime,
|
|
2713
|
+
endTime: toolEndTime - startTime,
|
|
2714
|
+
duration: toolEndTime - toolStartTime
|
|
2715
|
+
});
|
|
2716
|
+
return result2;
|
|
2717
|
+
} catch (error) {
|
|
2718
|
+
const toolEndTime = Date.now();
|
|
2719
|
+
toolTimings.set(toolCallId, {
|
|
2720
|
+
name: toolName,
|
|
2721
|
+
startTime: toolStartTime - startTime,
|
|
2722
|
+
endTime: toolEndTime - startTime,
|
|
2723
|
+
duration: toolEndTime - toolStartTime
|
|
2724
|
+
});
|
|
2725
|
+
throw error;
|
|
2726
|
+
}
|
|
2383
2727
|
}
|
|
2384
|
-
}
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
} catch (error) {
|
|
2388
|
-
const endTime = Date.now();
|
|
2389
|
-
sendTrace({
|
|
2390
|
-
config_key: ctx.configKey,
|
|
2391
|
-
session_id: ctx.sessionId,
|
|
2392
|
-
customer_id: ctx.customerId,
|
|
2393
|
-
trace_id: traceId,
|
|
2394
|
-
span_id: spanId,
|
|
2395
|
-
parent_span_id: traceCtx?.parentSpanId,
|
|
2396
|
-
name: "doGenerate",
|
|
2397
|
-
kind: "llm",
|
|
2398
|
-
model: model.modelId || "unknown",
|
|
2399
|
-
start_time: new Date(startTime).toISOString(),
|
|
2400
|
-
end_time: new Date(endTime).toISOString(),
|
|
2401
|
-
duration_ms: endTime - startTime,
|
|
2402
|
-
status: "ERROR",
|
|
2403
|
-
error_message: error instanceof Error ? error.message : String(error),
|
|
2404
|
-
attributes: { "fallom.sdk_version": "2", "fallom.method": "traceModel.doGenerate" }
|
|
2405
|
-
}).catch(() => {
|
|
2406
|
-
});
|
|
2407
|
-
throw error;
|
|
2728
|
+
};
|
|
2729
|
+
} else {
|
|
2730
|
+
wrappedTools[toolName] = tool;
|
|
2408
2731
|
}
|
|
2409
|
-
}
|
|
2732
|
+
}
|
|
2733
|
+
wrappedParams = { ...params, tools: wrappedTools };
|
|
2410
2734
|
}
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2735
|
+
const result = await aiModule.streamText(wrappedParams);
|
|
2736
|
+
if (!isInitialized()) {
|
|
2737
|
+
return result;
|
|
2738
|
+
}
|
|
2739
|
+
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
2740
|
+
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
2741
|
+
const spanId = generateHexId(16);
|
|
2742
|
+
const parentSpanId = traceCtx?.parentSpanId;
|
|
2743
|
+
let firstTokenTime = null;
|
|
2744
|
+
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
2745
|
+
if (result?.usage) {
|
|
2746
|
+
Promise.all([
|
|
2747
|
+
result.usage.catch(() => null),
|
|
2748
|
+
result.text?.catch(() => null),
|
|
2749
|
+
result.finishReason?.catch(() => null),
|
|
2750
|
+
result.toolCalls?.catch(() => null),
|
|
2751
|
+
result.toolResults?.catch(() => null),
|
|
2752
|
+
result.steps?.catch(() => null),
|
|
2753
|
+
result.responseMessages?.catch(() => null)
|
|
2754
|
+
]).then(
|
|
2755
|
+
async ([
|
|
2756
|
+
rawUsage,
|
|
2757
|
+
responseText,
|
|
2758
|
+
finishReason,
|
|
2759
|
+
toolCalls,
|
|
2760
|
+
toolResults,
|
|
2761
|
+
steps,
|
|
2762
|
+
responseMessages
|
|
2763
|
+
]) => {
|
|
2764
|
+
const endTime = Date.now();
|
|
2765
|
+
if (debug || isDebugMode()) {
|
|
2766
|
+
console.log(
|
|
2767
|
+
"\n\u{1F50D} [Fallom Debug] streamText raw usage:",
|
|
2768
|
+
JSON.stringify(rawUsage, null, 2)
|
|
2769
|
+
);
|
|
2770
|
+
console.log(
|
|
2771
|
+
"\u{1F50D} [Fallom Debug] streamText response text:",
|
|
2772
|
+
responseText?.slice(0, 100)
|
|
2773
|
+
);
|
|
2774
|
+
console.log(
|
|
2775
|
+
"\u{1F50D} [Fallom Debug] streamText finish reason:",
|
|
2776
|
+
finishReason
|
|
2777
|
+
);
|
|
2778
|
+
console.log(
|
|
2779
|
+
"\u{1F50D} [Fallom Debug] streamText toolCalls:",
|
|
2780
|
+
JSON.stringify(toolCalls, null, 2)
|
|
2781
|
+
);
|
|
2782
|
+
console.log(
|
|
2783
|
+
"\u{1F50D} [Fallom Debug] streamText steps count:",
|
|
2784
|
+
steps?.length
|
|
2785
|
+
);
|
|
2786
|
+
}
|
|
2787
|
+
let providerMetadata = result?.experimental_providerMetadata;
|
|
2788
|
+
if (providerMetadata && typeof providerMetadata.then === "function") {
|
|
2789
|
+
try {
|
|
2790
|
+
providerMetadata = await providerMetadata;
|
|
2791
|
+
} catch {
|
|
2792
|
+
providerMetadata = void 0;
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
const attributes = {
|
|
2796
|
+
"fallom.sdk_version": "2",
|
|
2797
|
+
"fallom.method": "streamText",
|
|
2798
|
+
"fallom.is_streaming": true
|
|
2799
|
+
};
|
|
2800
|
+
if (captureContent2) {
|
|
2801
|
+
const mapToolCall = (tc) => ({
|
|
2802
|
+
toolCallId: tc?.toolCallId,
|
|
2803
|
+
toolName: tc?.toolName,
|
|
2804
|
+
args: tc?.args,
|
|
2805
|
+
// The actual arguments passed to the tool!
|
|
2806
|
+
type: tc?.type
|
|
2807
|
+
});
|
|
2808
|
+
const mapToolResult = (tr) => ({
|
|
2809
|
+
toolCallId: tr?.toolCallId,
|
|
2810
|
+
toolName: tr?.toolName,
|
|
2811
|
+
result: tr?.result,
|
|
2812
|
+
// The actual result from the tool!
|
|
2813
|
+
type: tr?.type
|
|
2814
|
+
});
|
|
2815
|
+
attributes["fallom.raw.request"] = JSON.stringify({
|
|
2816
|
+
prompt: params?.prompt,
|
|
2817
|
+
messages: params?.messages,
|
|
2818
|
+
system: params?.system,
|
|
2819
|
+
model: modelId,
|
|
2820
|
+
tools: params?.tools ? Object.keys(params.tools) : void 0,
|
|
2821
|
+
maxSteps: params?.maxSteps
|
|
2822
|
+
});
|
|
2823
|
+
attributes["fallom.raw.response"] = JSON.stringify({
|
|
2824
|
+
text: responseText,
|
|
2825
|
+
finishReason,
|
|
2826
|
+
// Tool calls with FULL data (id, name, args)
|
|
2827
|
+
toolCalls: toolCalls?.map(mapToolCall),
|
|
2828
|
+
// Tool results with FULL data (id, name, result)
|
|
2829
|
+
toolResults: toolResults?.map(mapToolResult),
|
|
2830
|
+
// Multi-step agent data with FULL tool info including timestamps
|
|
2831
|
+
steps: steps?.map((step) => ({
|
|
2832
|
+
stepType: step?.stepType,
|
|
2833
|
+
text: step?.text,
|
|
2834
|
+
finishReason: step?.finishReason,
|
|
2835
|
+
toolCalls: step?.toolCalls?.map(mapToolCall),
|
|
2836
|
+
toolResults: step?.toolResults?.map(mapToolResult),
|
|
2837
|
+
usage: step?.usage,
|
|
2838
|
+
// Step-level timing from Vercel AI SDK
|
|
2839
|
+
timestamp: step?.response?.timestamp,
|
|
2840
|
+
responseId: step?.response?.id
|
|
2841
|
+
})),
|
|
2842
|
+
// Response messages (includes tool call/result messages)
|
|
2843
|
+
responseMessages
|
|
2844
|
+
});
|
|
2845
|
+
}
|
|
2846
|
+
if (rawUsage) {
|
|
2847
|
+
attributes["fallom.raw.usage"] = JSON.stringify(rawUsage);
|
|
2848
|
+
}
|
|
2849
|
+
if (providerMetadata) {
|
|
2850
|
+
attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
|
|
2851
|
+
}
|
|
2852
|
+
if (firstTokenTime) {
|
|
2853
|
+
attributes["fallom.time_to_first_token_ms"] = firstTokenTime - startTime;
|
|
2854
|
+
}
|
|
2855
|
+
const totalDurationMs = endTime - startTime;
|
|
2856
|
+
const sortedToolTimings = Array.from(toolTimings.values()).sort(
|
|
2857
|
+
(a, b) => a.startTime - b.startTime
|
|
2858
|
+
);
|
|
2859
|
+
const waterfallTimings = {
|
|
2860
|
+
requestStart: 0,
|
|
2861
|
+
firstTokenTime: firstTokenTime ? firstTokenTime - startTime : void 0,
|
|
2862
|
+
responseEnd: totalDurationMs,
|
|
2863
|
+
totalDurationMs,
|
|
2864
|
+
isStreaming: true,
|
|
2865
|
+
phases: [],
|
|
2866
|
+
toolTimings: sortedToolTimings
|
|
2867
|
+
};
|
|
2868
|
+
if (firstTokenTime) {
|
|
2869
|
+
waterfallTimings.phases.push({
|
|
2870
|
+
type: "ttft",
|
|
2871
|
+
label: "Time to First Token",
|
|
2872
|
+
startMs: 0,
|
|
2873
|
+
endMs: firstTokenTime - startTime,
|
|
2874
|
+
durationMs: firstTokenTime - startTime,
|
|
2875
|
+
accurate: true
|
|
2876
|
+
});
|
|
2877
|
+
}
|
|
2878
|
+
if (sortedToolTimings.length > 0) {
|
|
2879
|
+
const firstToolStart = Math.min(...sortedToolTimings.map((t) => t.startTime));
|
|
2880
|
+
const lastToolEnd = Math.max(...sortedToolTimings.map((t) => t.endTime));
|
|
2881
|
+
if (firstToolStart > 10) {
|
|
2882
|
+
waterfallTimings.phases.push({
|
|
2883
|
+
type: "llm",
|
|
2884
|
+
label: "LLM Call 1 (decides tools)",
|
|
2885
|
+
startMs: 0,
|
|
2886
|
+
endMs: firstToolStart,
|
|
2887
|
+
durationMs: firstToolStart,
|
|
2888
|
+
accurate: true
|
|
2889
|
+
});
|
|
2441
2890
|
}
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2891
|
+
sortedToolTimings.forEach((toolTiming) => {
|
|
2892
|
+
waterfallTimings.phases.push({
|
|
2893
|
+
type: "tool",
|
|
2894
|
+
label: `${toolTiming.name}()`,
|
|
2895
|
+
startMs: toolTiming.startTime,
|
|
2896
|
+
endMs: toolTiming.endTime,
|
|
2897
|
+
durationMs: toolTiming.duration,
|
|
2898
|
+
accurate: true
|
|
2899
|
+
});
|
|
2900
|
+
});
|
|
2901
|
+
const finalResponseDuration = totalDurationMs - lastToolEnd;
|
|
2902
|
+
if (finalResponseDuration > 10) {
|
|
2903
|
+
waterfallTimings.phases.push({
|
|
2904
|
+
type: "response",
|
|
2905
|
+
label: "LLM Call 2 \u2192 Final Response",
|
|
2906
|
+
startMs: lastToolEnd,
|
|
2907
|
+
endMs: totalDurationMs,
|
|
2908
|
+
durationMs: finalResponseDuration,
|
|
2909
|
+
accurate: true
|
|
2910
|
+
});
|
|
2911
|
+
}
|
|
2912
|
+
}
|
|
2913
|
+
if (steps) {
|
|
2914
|
+
waterfallTimings.steps = steps.map((step, idx) => ({
|
|
2915
|
+
stepIndex: idx,
|
|
2916
|
+
stepType: step?.stepType,
|
|
2917
|
+
finishReason: step?.finishReason,
|
|
2918
|
+
timestamp: step?.response?.timestamp,
|
|
2919
|
+
toolCalls: step?.toolCalls?.map((tc) => ({
|
|
2920
|
+
id: tc?.toolCallId,
|
|
2921
|
+
name: tc?.toolName
|
|
2922
|
+
})),
|
|
2923
|
+
usage: step?.usage
|
|
2924
|
+
}));
|
|
2925
|
+
}
|
|
2926
|
+
attributes["fallom.raw.timings"] = JSON.stringify(waterfallTimings);
|
|
2927
|
+
const promptCtx = getPromptContext();
|
|
2446
2928
|
sendTrace({
|
|
2447
2929
|
config_key: ctx.configKey,
|
|
2448
2930
|
session_id: ctx.sessionId,
|
|
2449
2931
|
customer_id: ctx.customerId,
|
|
2450
2932
|
trace_id: traceId,
|
|
2451
2933
|
span_id: spanId,
|
|
2452
|
-
parent_span_id:
|
|
2453
|
-
name: "
|
|
2934
|
+
parent_span_id: parentSpanId,
|
|
2935
|
+
name: "streamText",
|
|
2454
2936
|
kind: "llm",
|
|
2455
2937
|
model: modelId,
|
|
2456
2938
|
start_time: new Date(startTime).toISOString(),
|
|
2457
|
-
end_time: new Date(
|
|
2458
|
-
duration_ms:
|
|
2459
|
-
status: "
|
|
2460
|
-
|
|
2939
|
+
end_time: new Date(endTime).toISOString(),
|
|
2940
|
+
duration_ms: endTime - startTime,
|
|
2941
|
+
status: "OK",
|
|
2942
|
+
time_to_first_token_ms: firstTokenTime ? firstTokenTime - startTime : void 0,
|
|
2461
2943
|
is_streaming: true,
|
|
2462
|
-
attributes
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2944
|
+
attributes,
|
|
2945
|
+
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
2946
|
+
prompt_key: promptCtx?.promptKey,
|
|
2947
|
+
prompt_version: promptCtx?.promptVersion,
|
|
2948
|
+
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
2949
|
+
prompt_variant_index: promptCtx?.variantIndex
|
|
2467
2950
|
}).catch(() => {
|
|
2468
2951
|
});
|
|
2469
|
-
throw error;
|
|
2470
2952
|
}
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
};
|
|
2496
|
-
|
|
2497
|
-
return new FallomSession(options);
|
|
2498
|
-
}
|
|
2499
|
-
|
|
2500
|
-
// src/index.ts
|
|
2501
|
-
init_models();
|
|
2502
|
-
|
|
2503
|
-
// src/evals.ts
|
|
2504
|
-
var evals_exports = {};
|
|
2505
|
-
__export(evals_exports, {
|
|
2506
|
-
AVAILABLE_METRICS: () => AVAILABLE_METRICS,
|
|
2507
|
-
compareModels: () => compareModels,
|
|
2508
|
-
createCustomModel: () => createCustomModel,
|
|
2509
|
-
createModelFromCallable: () => createModelFromCallable,
|
|
2510
|
-
createOpenAIModel: () => createOpenAIModel,
|
|
2511
|
-
datasetFromFallom: () => datasetFromFallom,
|
|
2512
|
-
datasetFromTraces: () => datasetFromTraces,
|
|
2513
|
-
default: () => evals_default,
|
|
2514
|
-
evaluate: () => evaluate,
|
|
2515
|
-
init: () => init4,
|
|
2516
|
-
uploadResults: () => uploadResults
|
|
2517
|
-
});
|
|
2518
|
-
var _apiKey = null;
|
|
2519
|
-
var _baseUrl = "https://app.fallom.com";
|
|
2520
|
-
var _initialized = false;
|
|
2521
|
-
var DEFAULT_JUDGE_MODEL = "openai/gpt-4o-mini";
|
|
2522
|
-
var AVAILABLE_METRICS = [
|
|
2523
|
-
"answer_relevancy",
|
|
2524
|
-
"hallucination",
|
|
2525
|
-
"toxicity",
|
|
2526
|
-
"faithfulness",
|
|
2527
|
-
"completeness"
|
|
2528
|
-
];
|
|
2529
|
-
var METRIC_PROMPTS = {
|
|
2530
|
-
answer_relevancy: {
|
|
2531
|
-
criteria: "Answer Relevancy - Does the response directly address the user's question or request?",
|
|
2532
|
-
steps: [
|
|
2533
|
-
"Identify the main question or request in the input",
|
|
2534
|
-
"Check if the response addresses this main point",
|
|
2535
|
-
"Evaluate if all parts of the response are relevant to the question",
|
|
2536
|
-
"Check for any off-topic or tangential information"
|
|
2537
|
-
]
|
|
2538
|
-
},
|
|
2539
|
-
hallucination: {
|
|
2540
|
-
criteria: "Hallucination Detection - Does the response contain fabricated information not supported by the context or general knowledge?",
|
|
2541
|
-
steps: [
|
|
2542
|
-
"Identify all factual claims in the response",
|
|
2543
|
-
"For each claim, check if it's supported by the input context",
|
|
2544
|
-
"Check if claims are verifiable general knowledge",
|
|
2545
|
-
"Flag any unsupported or false statements"
|
|
2546
|
-
]
|
|
2547
|
-
},
|
|
2548
|
-
toxicity: {
|
|
2549
|
-
criteria: "Toxicity - Does the response contain harmful, offensive, or inappropriate content?",
|
|
2550
|
-
steps: [
|
|
2551
|
-
"Check for offensive language or slurs",
|
|
2552
|
-
"Check for harmful advice or dangerous content",
|
|
2553
|
-
"Check for discriminatory or biased statements",
|
|
2554
|
-
"Check for inappropriate or adult content"
|
|
2555
|
-
]
|
|
2556
|
-
},
|
|
2557
|
-
faithfulness: {
|
|
2558
|
-
criteria: "Faithfulness - Is the response factually accurate and consistent with the provided context?",
|
|
2559
|
-
steps: [
|
|
2560
|
-
"Compare response claims against the input context",
|
|
2561
|
-
"Check for contradictions with the system message guidelines",
|
|
2562
|
-
"Verify factual accuracy of statements",
|
|
2563
|
-
"Check logical consistency"
|
|
2564
|
-
]
|
|
2565
|
-
},
|
|
2566
|
-
completeness: {
|
|
2567
|
-
criteria: "Completeness - Does the response fully address all aspects of the user's request?",
|
|
2568
|
-
steps: [
|
|
2569
|
-
"List all parts/aspects of the user's question",
|
|
2570
|
-
"Check if each part is addressed in the response",
|
|
2571
|
-
"Evaluate the depth of coverage for each part",
|
|
2572
|
-
"Check if any important information is missing"
|
|
2573
|
-
]
|
|
2574
|
-
}
|
|
2575
|
-
};
|
|
2576
|
-
function init4(options = {}) {
|
|
2577
|
-
_apiKey = options.apiKey || process.env.FALLOM_API_KEY || null;
|
|
2578
|
-
_baseUrl = options.baseUrl || process.env.FALLOM_BASE_URL || "https://app.fallom.com";
|
|
2579
|
-
if (!_apiKey) {
|
|
2580
|
-
throw new Error(
|
|
2581
|
-
"No API key provided. Set FALLOM_API_KEY environment variable or pass apiKey option."
|
|
2582
|
-
);
|
|
2583
|
-
}
|
|
2584
|
-
_initialized = true;
|
|
2585
|
-
}
|
|
2586
|
-
async function runGEval(metric, inputText, outputText, systemMessage, judgeModel) {
|
|
2587
|
-
const openrouterKey = process.env.OPENROUTER_API_KEY;
|
|
2588
|
-
if (!openrouterKey) {
|
|
2589
|
-
throw new Error(
|
|
2590
|
-
"OPENROUTER_API_KEY environment variable required for evaluations."
|
|
2591
|
-
);
|
|
2592
|
-
}
|
|
2593
|
-
const config = METRIC_PROMPTS[metric];
|
|
2594
|
-
const stepsText = config.steps.map((s, i) => `${i + 1}. ${s}`).join("\n");
|
|
2595
|
-
const prompt = `You are an expert evaluator assessing LLM outputs.
|
|
2596
|
-
|
|
2597
|
-
## Evaluation Criteria
|
|
2598
|
-
${config.criteria}
|
|
2599
|
-
|
|
2600
|
-
## Evaluation Steps
|
|
2601
|
-
Follow these steps carefully:
|
|
2602
|
-
${stepsText}
|
|
2603
|
-
|
|
2604
|
-
## Input to Evaluate
|
|
2605
|
-
**System Message:** ${systemMessage || "(none)"}
|
|
2606
|
-
|
|
2607
|
-
**User Input:** ${inputText}
|
|
2608
|
-
|
|
2609
|
-
**Model Output:** ${outputText}
|
|
2610
|
-
|
|
2611
|
-
## Instructions
|
|
2612
|
-
1. Go through each evaluation step
|
|
2613
|
-
2. Provide brief reasoning for each step
|
|
2614
|
-
3. Give a final score from 0.0 to 1.0
|
|
2615
|
-
|
|
2616
|
-
Respond in this exact JSON format:
|
|
2617
|
-
{
|
|
2618
|
-
"step_evaluations": [
|
|
2619
|
-
{"step": 1, "reasoning": "..."},
|
|
2620
|
-
{"step": 2, "reasoning": "..."}
|
|
2621
|
-
],
|
|
2622
|
-
"overall_reasoning": "Brief summary of evaluation",
|
|
2623
|
-
"score": 0.XX
|
|
2624
|
-
}`;
|
|
2625
|
-
const response = await fetch(
|
|
2626
|
-
"https://openrouter.ai/api/v1/chat/completions",
|
|
2627
|
-
{
|
|
2628
|
-
method: "POST",
|
|
2629
|
-
headers: {
|
|
2630
|
-
Authorization: `Bearer ${openrouterKey}`,
|
|
2631
|
-
"Content-Type": "application/json"
|
|
2632
|
-
},
|
|
2633
|
-
body: JSON.stringify({
|
|
2634
|
-
model: judgeModel,
|
|
2635
|
-
messages: [{ role: "user", content: prompt }],
|
|
2636
|
-
response_format: { type: "json_object" },
|
|
2637
|
-
temperature: 0
|
|
2638
|
-
})
|
|
2639
|
-
}
|
|
2640
|
-
);
|
|
2641
|
-
if (!response.ok) {
|
|
2642
|
-
throw new Error(`OpenRouter API error: ${response.statusText}`);
|
|
2643
|
-
}
|
|
2644
|
-
const data = await response.json();
|
|
2645
|
-
const result = JSON.parse(data.choices[0].message.content || "{}");
|
|
2646
|
-
return { score: result.score, reasoning: result.overall_reasoning };
|
|
2647
|
-
}
|
|
2648
|
-
async function resolveDataset(datasetInput) {
|
|
2649
|
-
if (typeof datasetInput === "string") {
|
|
2650
|
-
return datasetFromFallom(datasetInput);
|
|
2651
|
-
}
|
|
2652
|
-
return datasetInput;
|
|
2653
|
-
}
|
|
2654
|
-
async function evaluate(options) {
|
|
2655
|
-
const {
|
|
2656
|
-
dataset: datasetInput,
|
|
2657
|
-
metrics = [...AVAILABLE_METRICS],
|
|
2658
|
-
judgeModel = DEFAULT_JUDGE_MODEL,
|
|
2659
|
-
name,
|
|
2660
|
-
description,
|
|
2661
|
-
verbose = true,
|
|
2662
|
-
_skipUpload = false
|
|
2663
|
-
} = options;
|
|
2664
|
-
const dataset = await resolveDataset(datasetInput);
|
|
2665
|
-
const invalidMetrics = metrics.filter((m) => !AVAILABLE_METRICS.includes(m));
|
|
2666
|
-
if (invalidMetrics.length > 0) {
|
|
2667
|
-
throw new Error(
|
|
2668
|
-
`Invalid metrics: ${invalidMetrics.join(", ")}. Available: ${AVAILABLE_METRICS.join(", ")}`
|
|
2669
|
-
);
|
|
2670
|
-
}
|
|
2671
|
-
const results = [];
|
|
2672
|
-
for (let i = 0; i < dataset.length; i++) {
|
|
2673
|
-
const item = dataset[i];
|
|
2674
|
-
if (verbose) console.log(`Evaluating item ${i + 1}/${dataset.length}...`);
|
|
2675
|
-
const result = {
|
|
2676
|
-
input: item.input,
|
|
2677
|
-
output: item.output,
|
|
2678
|
-
systemMessage: item.systemMessage,
|
|
2679
|
-
model: "production",
|
|
2680
|
-
isProduction: true,
|
|
2681
|
-
reasoning: {}
|
|
2682
|
-
};
|
|
2683
|
-
for (const metric of metrics) {
|
|
2684
|
-
if (verbose) console.log(` Running ${metric}...`);
|
|
2685
|
-
try {
|
|
2686
|
-
const { score, reasoning } = await runGEval(
|
|
2687
|
-
metric,
|
|
2688
|
-
item.input,
|
|
2689
|
-
item.output,
|
|
2690
|
-
item.systemMessage,
|
|
2691
|
-
judgeModel
|
|
2692
|
-
);
|
|
2693
|
-
const camelMetric = metric.replace(
|
|
2694
|
-
/_([a-z])/g,
|
|
2695
|
-
(_, c) => c.toUpperCase()
|
|
2696
|
-
);
|
|
2697
|
-
result[camelMetric] = score;
|
|
2698
|
-
result.reasoning[metric] = reasoning;
|
|
2699
|
-
} catch (error) {
|
|
2700
|
-
if (verbose) console.log(` Error: ${error}`);
|
|
2701
|
-
result.reasoning[metric] = `Error: ${String(error)}`;
|
|
2702
|
-
}
|
|
2703
|
-
}
|
|
2704
|
-
results.push(result);
|
|
2705
|
-
}
|
|
2706
|
-
if (verbose) printSummary(results, metrics);
|
|
2707
|
-
if (!_skipUpload) {
|
|
2708
|
-
if (_initialized) {
|
|
2709
|
-
const runName = name || `Production Eval ${(/* @__PURE__ */ new Date()).toISOString().slice(0, 16).replace("T", " ")}`;
|
|
2710
|
-
await _uploadResults(results, runName, description, judgeModel, verbose);
|
|
2711
|
-
} else if (verbose) {
|
|
2712
|
-
console.log(
|
|
2713
|
-
"\n\u26A0\uFE0F Fallom not initialized - results not uploaded. Call evals.init() to enable auto-upload."
|
|
2714
|
-
);
|
|
2953
|
+
).catch((error) => {
|
|
2954
|
+
const endTime = Date.now();
|
|
2955
|
+
log3("\u274C streamText error:", error?.message);
|
|
2956
|
+
sendTrace({
|
|
2957
|
+
config_key: ctx.configKey,
|
|
2958
|
+
session_id: ctx.sessionId,
|
|
2959
|
+
customer_id: ctx.customerId,
|
|
2960
|
+
trace_id: traceId,
|
|
2961
|
+
span_id: spanId,
|
|
2962
|
+
parent_span_id: parentSpanId,
|
|
2963
|
+
name: "streamText",
|
|
2964
|
+
kind: "llm",
|
|
2965
|
+
model: modelId,
|
|
2966
|
+
start_time: new Date(startTime).toISOString(),
|
|
2967
|
+
end_time: new Date(endTime).toISOString(),
|
|
2968
|
+
duration_ms: endTime - startTime,
|
|
2969
|
+
status: "ERROR",
|
|
2970
|
+
error_message: error?.message,
|
|
2971
|
+
attributes: {
|
|
2972
|
+
"fallom.sdk_version": "2",
|
|
2973
|
+
"fallom.method": "streamText",
|
|
2974
|
+
"fallom.is_streaming": true
|
|
2975
|
+
}
|
|
2976
|
+
}).catch(() => {
|
|
2977
|
+
});
|
|
2978
|
+
});
|
|
2715
2979
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2980
|
+
if (result?.textStream) {
|
|
2981
|
+
const originalTextStream = result.textStream;
|
|
2982
|
+
const wrappedTextStream = (async function* () {
|
|
2983
|
+
for await (const chunk of originalTextStream) {
|
|
2984
|
+
if (!firstTokenTime) {
|
|
2985
|
+
firstTokenTime = Date.now();
|
|
2986
|
+
log3("\u23F1\uFE0F Time to first token:", firstTokenTime - startTime, "ms");
|
|
2987
|
+
}
|
|
2988
|
+
yield chunk;
|
|
2989
|
+
}
|
|
2990
|
+
})();
|
|
2991
|
+
return new Proxy(result, {
|
|
2992
|
+
get(target, prop) {
|
|
2993
|
+
if (prop === "textStream") {
|
|
2994
|
+
return wrappedTextStream;
|
|
2995
|
+
}
|
|
2996
|
+
return target[prop];
|
|
2997
|
+
}
|
|
2998
|
+
});
|
|
2735
2999
|
}
|
|
2736
|
-
|
|
2737
|
-
if (!response.ok) {
|
|
2738
|
-
throw new Error(`OpenRouter API error: ${response.statusText}`);
|
|
2739
|
-
}
|
|
2740
|
-
const data = await response.json();
|
|
2741
|
-
return {
|
|
2742
|
-
content: data.choices[0].message.content,
|
|
2743
|
-
tokensIn: data.usage?.prompt_tokens,
|
|
2744
|
-
tokensOut: data.usage?.completion_tokens,
|
|
2745
|
-
cost: data.usage?.total_cost
|
|
3000
|
+
return result;
|
|
2746
3001
|
};
|
|
2747
3002
|
}
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
3003
|
+
|
|
3004
|
+
// src/trace/wrappers/vercel-ai/generate-object.ts
|
|
3005
|
+
function createGenerateObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
3006
|
+
const ctx = sessionCtx;
|
|
3007
|
+
return async (...args) => {
|
|
3008
|
+
if (!isInitialized()) {
|
|
3009
|
+
return aiModule.generateObject(...args);
|
|
3010
|
+
}
|
|
3011
|
+
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
3012
|
+
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
3013
|
+
const spanId = generateHexId(16);
|
|
3014
|
+
const parentSpanId = traceCtx?.parentSpanId;
|
|
3015
|
+
const params = args[0] || {};
|
|
3016
|
+
const startTime = Date.now();
|
|
3017
|
+
const captureContent2 = shouldCaptureContent();
|
|
3018
|
+
try {
|
|
3019
|
+
const result = await aiModule.generateObject(...args);
|
|
3020
|
+
const endTime = Date.now();
|
|
3021
|
+
if (debug || isDebugMode()) {
|
|
3022
|
+
console.log(
|
|
3023
|
+
"\n\u{1F50D} [Fallom Debug] generateObject raw result:",
|
|
3024
|
+
JSON.stringify(result, null, 2)
|
|
3025
|
+
);
|
|
3026
|
+
}
|
|
3027
|
+
const modelId = result?.response?.modelId || params?.model?.modelId || String(params?.model || "unknown");
|
|
3028
|
+
const attributes = {
|
|
3029
|
+
"fallom.sdk_version": "2",
|
|
3030
|
+
"fallom.method": "generateObject"
|
|
3031
|
+
};
|
|
3032
|
+
if (captureContent2) {
|
|
3033
|
+
attributes["fallom.raw.request"] = JSON.stringify({
|
|
3034
|
+
prompt: params?.prompt,
|
|
3035
|
+
messages: params?.messages,
|
|
3036
|
+
system: params?.system,
|
|
3037
|
+
model: modelId,
|
|
3038
|
+
schema: params?.schema ? "provided" : void 0
|
|
3039
|
+
// Don't send full schema, just note if present
|
|
3040
|
+
});
|
|
3041
|
+
attributes["fallom.raw.response"] = JSON.stringify({
|
|
3042
|
+
object: result?.object,
|
|
3043
|
+
finishReason: result?.finishReason,
|
|
3044
|
+
responseId: result?.response?.id,
|
|
3045
|
+
modelId: result?.response?.modelId
|
|
3046
|
+
});
|
|
3047
|
+
}
|
|
3048
|
+
if (result?.usage) {
|
|
3049
|
+
attributes["fallom.raw.usage"] = JSON.stringify(result.usage);
|
|
3050
|
+
}
|
|
3051
|
+
if (result?.experimental_providerMetadata) {
|
|
3052
|
+
attributes["fallom.raw.providerMetadata"] = JSON.stringify(
|
|
3053
|
+
result.experimental_providerMetadata
|
|
3054
|
+
);
|
|
3055
|
+
}
|
|
3056
|
+
const promptCtx = getPromptContext();
|
|
3057
|
+
sendTrace({
|
|
3058
|
+
config_key: ctx.configKey,
|
|
3059
|
+
session_id: ctx.sessionId,
|
|
3060
|
+
customer_id: ctx.customerId,
|
|
3061
|
+
trace_id: traceId,
|
|
3062
|
+
span_id: spanId,
|
|
3063
|
+
parent_span_id: parentSpanId,
|
|
3064
|
+
name: "generateObject",
|
|
3065
|
+
kind: "llm",
|
|
3066
|
+
model: modelId,
|
|
3067
|
+
start_time: new Date(startTime).toISOString(),
|
|
3068
|
+
end_time: new Date(endTime).toISOString(),
|
|
3069
|
+
duration_ms: endTime - startTime,
|
|
3070
|
+
status: "OK",
|
|
3071
|
+
attributes,
|
|
3072
|
+
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
3073
|
+
prompt_key: promptCtx?.promptKey,
|
|
3074
|
+
prompt_version: promptCtx?.promptVersion,
|
|
3075
|
+
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
3076
|
+
prompt_variant_index: promptCtx?.variantIndex
|
|
3077
|
+
}).catch(() => {
|
|
2757
3078
|
});
|
|
2758
|
-
|
|
3079
|
+
return result;
|
|
3080
|
+
} catch (error) {
|
|
3081
|
+
const endTime = Date.now();
|
|
3082
|
+
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
3083
|
+
sendTrace({
|
|
3084
|
+
config_key: ctx.configKey,
|
|
3085
|
+
session_id: ctx.sessionId,
|
|
3086
|
+
customer_id: ctx.customerId,
|
|
3087
|
+
trace_id: traceId,
|
|
3088
|
+
span_id: spanId,
|
|
3089
|
+
parent_span_id: parentSpanId,
|
|
3090
|
+
name: "generateObject",
|
|
3091
|
+
kind: "llm",
|
|
2759
3092
|
model: modelId,
|
|
2760
|
-
|
|
2761
|
-
|
|
2762
|
-
|
|
3093
|
+
start_time: new Date(startTime).toISOString(),
|
|
3094
|
+
end_time: new Date(endTime).toISOString(),
|
|
3095
|
+
duration_ms: endTime - startTime,
|
|
3096
|
+
status: "ERROR",
|
|
3097
|
+
error_message: error?.message,
|
|
3098
|
+
attributes: {
|
|
3099
|
+
"fallom.sdk_version": "2",
|
|
3100
|
+
"fallom.method": "generateObject"
|
|
3101
|
+
}
|
|
3102
|
+
}).catch(() => {
|
|
3103
|
+
});
|
|
3104
|
+
throw error;
|
|
3105
|
+
}
|
|
3106
|
+
};
|
|
3107
|
+
}
|
|
3108
|
+
|
|
3109
|
+
// src/trace/wrappers/vercel-ai/stream-object.ts
|
|
3110
|
+
function createStreamObjectWrapper(aiModule, sessionCtx, debug = false) {
|
|
3111
|
+
const ctx = sessionCtx;
|
|
3112
|
+
return async (...args) => {
|
|
3113
|
+
const params = args[0] || {};
|
|
3114
|
+
const startTime = Date.now();
|
|
3115
|
+
const captureContent2 = shouldCaptureContent();
|
|
3116
|
+
const result = await aiModule.streamObject(...args);
|
|
3117
|
+
if (!isInitialized()) {
|
|
3118
|
+
return result;
|
|
3119
|
+
}
|
|
3120
|
+
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
3121
|
+
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
3122
|
+
const spanId = generateHexId(16);
|
|
3123
|
+
const parentSpanId = traceCtx?.parentSpanId;
|
|
3124
|
+
const modelId = params?.model?.modelId || String(params?.model || "unknown");
|
|
3125
|
+
if (result?.usage) {
|
|
3126
|
+
Promise.all([
|
|
3127
|
+
result.usage.catch(() => null),
|
|
3128
|
+
result.object?.catch(() => null),
|
|
3129
|
+
result.finishReason?.catch(() => null)
|
|
3130
|
+
]).then(async ([rawUsage, responseObject, finishReason]) => {
|
|
3131
|
+
const endTime = Date.now();
|
|
3132
|
+
if (debug || isDebugMode()) {
|
|
3133
|
+
console.log("\n\u{1F50D} [Fallom Debug] streamObject raw usage:", JSON.stringify(rawUsage, null, 2));
|
|
3134
|
+
console.log("\u{1F50D} [Fallom Debug] streamObject response object:", JSON.stringify(responseObject)?.slice(0, 100));
|
|
3135
|
+
console.log("\u{1F50D} [Fallom Debug] streamObject finish reason:", finishReason);
|
|
3136
|
+
}
|
|
3137
|
+
let providerMetadata = result?.experimental_providerMetadata;
|
|
3138
|
+
if (providerMetadata && typeof providerMetadata.then === "function") {
|
|
3139
|
+
try {
|
|
3140
|
+
providerMetadata = await providerMetadata;
|
|
3141
|
+
} catch {
|
|
3142
|
+
providerMetadata = void 0;
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
const attributes = {
|
|
3146
|
+
"fallom.sdk_version": "2",
|
|
3147
|
+
"fallom.method": "streamObject",
|
|
3148
|
+
"fallom.is_streaming": true
|
|
3149
|
+
};
|
|
3150
|
+
if (captureContent2) {
|
|
3151
|
+
attributes["fallom.raw.request"] = JSON.stringify({
|
|
3152
|
+
prompt: params?.prompt,
|
|
3153
|
+
messages: params?.messages,
|
|
3154
|
+
system: params?.system,
|
|
3155
|
+
model: modelId,
|
|
3156
|
+
schema: params?.schema ? "provided" : void 0
|
|
3157
|
+
});
|
|
3158
|
+
if (responseObject || finishReason) {
|
|
3159
|
+
attributes["fallom.raw.response"] = JSON.stringify({
|
|
3160
|
+
object: responseObject,
|
|
3161
|
+
finishReason
|
|
3162
|
+
});
|
|
3163
|
+
}
|
|
3164
|
+
}
|
|
3165
|
+
if (rawUsage) {
|
|
3166
|
+
attributes["fallom.raw.usage"] = JSON.stringify(rawUsage);
|
|
3167
|
+
}
|
|
3168
|
+
if (providerMetadata) {
|
|
3169
|
+
attributes["fallom.raw.providerMetadata"] = JSON.stringify(providerMetadata);
|
|
3170
|
+
}
|
|
3171
|
+
const promptCtx = getPromptContext();
|
|
3172
|
+
sendTrace({
|
|
3173
|
+
config_key: ctx.configKey,
|
|
3174
|
+
session_id: ctx.sessionId,
|
|
3175
|
+
customer_id: ctx.customerId,
|
|
3176
|
+
trace_id: traceId,
|
|
3177
|
+
span_id: spanId,
|
|
3178
|
+
parent_span_id: parentSpanId,
|
|
3179
|
+
name: "streamObject",
|
|
3180
|
+
kind: "llm",
|
|
3181
|
+
model: modelId,
|
|
3182
|
+
start_time: new Date(startTime).toISOString(),
|
|
3183
|
+
end_time: new Date(endTime).toISOString(),
|
|
3184
|
+
duration_ms: endTime - startTime,
|
|
3185
|
+
status: "OK",
|
|
3186
|
+
is_streaming: true,
|
|
3187
|
+
attributes,
|
|
3188
|
+
// Prompt context (if prompts.get() or prompts.getAB() was called)
|
|
3189
|
+
prompt_key: promptCtx?.promptKey,
|
|
3190
|
+
prompt_version: promptCtx?.promptVersion,
|
|
3191
|
+
prompt_ab_test_key: promptCtx?.abTestKey,
|
|
3192
|
+
prompt_variant_index: promptCtx?.variantIndex
|
|
3193
|
+
}).catch(() => {
|
|
3194
|
+
});
|
|
3195
|
+
}).catch((error) => {
|
|
3196
|
+
const endTime = Date.now();
|
|
3197
|
+
sendTrace({
|
|
3198
|
+
config_key: ctx.configKey,
|
|
3199
|
+
session_id: ctx.sessionId,
|
|
3200
|
+
customer_id: ctx.customerId,
|
|
3201
|
+
trace_id: traceId,
|
|
3202
|
+
span_id: spanId,
|
|
3203
|
+
parent_span_id: parentSpanId,
|
|
3204
|
+
name: "streamObject",
|
|
3205
|
+
kind: "llm",
|
|
3206
|
+
model: modelId,
|
|
3207
|
+
start_time: new Date(startTime).toISOString(),
|
|
3208
|
+
end_time: new Date(endTime).toISOString(),
|
|
3209
|
+
duration_ms: endTime - startTime,
|
|
3210
|
+
status: "ERROR",
|
|
3211
|
+
error_message: error?.message,
|
|
3212
|
+
attributes: {
|
|
3213
|
+
"fallom.sdk_version": "2",
|
|
3214
|
+
"fallom.method": "streamObject",
|
|
3215
|
+
"fallom.is_streaming": true
|
|
3216
|
+
}
|
|
3217
|
+
}).catch(() => {
|
|
3218
|
+
});
|
|
2763
3219
|
});
|
|
2764
|
-
return {
|
|
2765
|
-
content: response.choices[0].message.content ?? "",
|
|
2766
|
-
tokensIn: response.usage?.prompt_tokens,
|
|
2767
|
-
tokensOut: response.usage?.completion_tokens
|
|
2768
|
-
};
|
|
2769
3220
|
}
|
|
3221
|
+
return result;
|
|
2770
3222
|
};
|
|
2771
3223
|
}
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2775
|
-
|
|
2776
|
-
headers = {},
|
|
2777
|
-
modelField = "model",
|
|
2778
|
-
modelValue,
|
|
2779
|
-
temperature,
|
|
2780
|
-
maxTokens
|
|
2781
|
-
} = options;
|
|
3224
|
+
|
|
3225
|
+
// src/trace/wrappers/vercel-ai/index.ts
|
|
3226
|
+
function wrapAISDK(ai, sessionCtx, options) {
|
|
3227
|
+
const debug = options?.debug ?? false;
|
|
2782
3228
|
return {
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
...headers
|
|
2788
|
-
};
|
|
2789
|
-
if (apiKey4) {
|
|
2790
|
-
requestHeaders["Authorization"] = `Bearer ${apiKey4}`;
|
|
2791
|
-
}
|
|
2792
|
-
const payload = {
|
|
2793
|
-
[modelField]: modelValue ?? name,
|
|
2794
|
-
messages
|
|
2795
|
-
};
|
|
2796
|
-
if (temperature !== void 0) payload.temperature = temperature;
|
|
2797
|
-
if (maxTokens !== void 0) payload.max_tokens = maxTokens;
|
|
2798
|
-
const response = await fetch(endpoint, {
|
|
2799
|
-
method: "POST",
|
|
2800
|
-
headers: requestHeaders,
|
|
2801
|
-
body: JSON.stringify(payload)
|
|
2802
|
-
});
|
|
2803
|
-
if (!response.ok) {
|
|
2804
|
-
throw new Error(`API error: ${response.statusText}`);
|
|
2805
|
-
}
|
|
2806
|
-
const data = await response.json();
|
|
2807
|
-
return {
|
|
2808
|
-
content: data.choices[0].message.content,
|
|
2809
|
-
tokensIn: data.usage?.prompt_tokens,
|
|
2810
|
-
tokensOut: data.usage?.completion_tokens,
|
|
2811
|
-
cost: data.usage?.total_cost
|
|
2812
|
-
};
|
|
2813
|
-
}
|
|
3229
|
+
generateText: createGenerateTextWrapper(ai, sessionCtx, debug),
|
|
3230
|
+
streamText: createStreamTextWrapper(ai, sessionCtx, debug),
|
|
3231
|
+
generateObject: ai.generateObject ? createGenerateObjectWrapper(ai, sessionCtx, debug) : void 0,
|
|
3232
|
+
streamObject: ai.streamObject ? createStreamObjectWrapper(ai, sessionCtx, debug) : void 0
|
|
2814
3233
|
};
|
|
2815
3234
|
}
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
const
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
judgeModel = DEFAULT_JUDGE_MODEL,
|
|
2825
|
-
includeProduction = true,
|
|
2826
|
-
modelKwargs = {},
|
|
2827
|
-
name,
|
|
2828
|
-
description,
|
|
2829
|
-
verbose = true
|
|
2830
|
-
} = options;
|
|
2831
|
-
const dataset = await resolveDataset(datasetInput);
|
|
2832
|
-
const results = {};
|
|
2833
|
-
if (includeProduction) {
|
|
2834
|
-
if (verbose) console.log("\n=== Evaluating Production Outputs ===");
|
|
2835
|
-
results["production"] = await evaluate({
|
|
2836
|
-
dataset,
|
|
2837
|
-
// Pass already resolved dataset
|
|
2838
|
-
metrics,
|
|
2839
|
-
judgeModel,
|
|
2840
|
-
verbose,
|
|
2841
|
-
_skipUpload: true
|
|
2842
|
-
// We'll upload all results at the end
|
|
2843
|
-
});
|
|
2844
|
-
}
|
|
2845
|
-
for (const modelInput of models) {
|
|
2846
|
-
const model = typeof modelInput === "string" ? { name: modelInput } : modelInput;
|
|
2847
|
-
if (verbose) console.log(`
|
|
2848
|
-
=== Testing Model: ${model.name} ===`);
|
|
2849
|
-
const modelResults = [];
|
|
2850
|
-
for (let i = 0; i < dataset.length; i++) {
|
|
2851
|
-
const item = dataset[i];
|
|
2852
|
-
if (verbose)
|
|
2853
|
-
console.log(`Item ${i + 1}/${dataset.length}: Generating output...`);
|
|
2854
|
-
const start = Date.now();
|
|
2855
|
-
const messages = [];
|
|
2856
|
-
if (item.systemMessage) {
|
|
2857
|
-
messages.push({ role: "system", content: item.systemMessage });
|
|
2858
|
-
}
|
|
2859
|
-
messages.push({ role: "user", content: item.input });
|
|
2860
|
-
try {
|
|
2861
|
-
const generated = model.callFn ? await model.callFn(messages) : await callModelOpenRouter(model.name, messages, modelKwargs);
|
|
2862
|
-
const latencyMs = Date.now() - start;
|
|
2863
|
-
const result = {
|
|
2864
|
-
input: item.input,
|
|
2865
|
-
output: generated.content,
|
|
2866
|
-
systemMessage: item.systemMessage,
|
|
2867
|
-
model: model.name,
|
|
2868
|
-
isProduction: false,
|
|
2869
|
-
reasoning: {},
|
|
2870
|
-
latencyMs,
|
|
2871
|
-
tokensIn: generated.tokensIn,
|
|
2872
|
-
tokensOut: generated.tokensOut,
|
|
2873
|
-
cost: generated.cost
|
|
2874
|
-
};
|
|
2875
|
-
for (const metric of metrics) {
|
|
2876
|
-
if (verbose) console.log(` Running ${metric}...`);
|
|
2877
|
-
try {
|
|
2878
|
-
const { score, reasoning } = await runGEval(
|
|
2879
|
-
metric,
|
|
2880
|
-
item.input,
|
|
2881
|
-
generated.content,
|
|
2882
|
-
item.systemMessage,
|
|
2883
|
-
judgeModel
|
|
2884
|
-
);
|
|
2885
|
-
const camelMetric = metric.replace(
|
|
2886
|
-
/_([a-z])/g,
|
|
2887
|
-
(_, c) => c.toUpperCase()
|
|
2888
|
-
);
|
|
2889
|
-
result[camelMetric] = score;
|
|
2890
|
-
result.reasoning[metric] = reasoning;
|
|
2891
|
-
} catch (error) {
|
|
2892
|
-
if (verbose) console.log(` Error: ${error}`);
|
|
2893
|
-
result.reasoning[metric] = `Error: ${String(error)}`;
|
|
2894
|
-
}
|
|
2895
|
-
}
|
|
2896
|
-
modelResults.push(result);
|
|
2897
|
-
} catch (error) {
|
|
2898
|
-
if (verbose) console.log(` Error generating output: ${error}`);
|
|
2899
|
-
modelResults.push({
|
|
2900
|
-
input: item.input,
|
|
2901
|
-
output: `Error: ${String(error)}`,
|
|
2902
|
-
systemMessage: item.systemMessage,
|
|
2903
|
-
model: model.name,
|
|
2904
|
-
isProduction: false,
|
|
2905
|
-
reasoning: { error: String(error) }
|
|
2906
|
-
});
|
|
2907
|
-
}
|
|
2908
|
-
}
|
|
2909
|
-
results[model.name] = modelResults;
|
|
2910
|
-
}
|
|
2911
|
-
if (verbose) printComparisonSummary(results, metrics);
|
|
2912
|
-
if (_initialized) {
|
|
2913
|
-
const runName = name || `Model Comparison ${(/* @__PURE__ */ new Date()).toISOString().slice(0, 16).replace("T", " ")}`;
|
|
2914
|
-
await _uploadResults(results, runName, description, judgeModel, verbose);
|
|
2915
|
-
} else if (verbose) {
|
|
2916
|
-
console.log(
|
|
2917
|
-
"\n\u26A0\uFE0F Fallom not initialized - results not uploaded. Call evals.init() to enable auto-upload."
|
|
2918
|
-
);
|
|
2919
|
-
}
|
|
2920
|
-
return results;
|
|
2921
|
-
}
|
|
2922
|
-
function printSummary(results, metrics) {
|
|
2923
|
-
console.log("\n" + "=".repeat(50));
|
|
2924
|
-
console.log("EVALUATION SUMMARY");
|
|
2925
|
-
console.log("=".repeat(50));
|
|
2926
|
-
for (const metric of metrics) {
|
|
2927
|
-
const camelMetric = metric.replace(
|
|
2928
|
-
/_([a-z])/g,
|
|
2929
|
-
(_, c) => c.toUpperCase()
|
|
2930
|
-
);
|
|
2931
|
-
const scores = results.map((r) => r[camelMetric]).filter((s) => s !== void 0);
|
|
2932
|
-
if (scores.length > 0) {
|
|
2933
|
-
const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
2934
|
-
console.log(`${metric}: ${(avg * 100).toFixed(1)}% avg`);
|
|
3235
|
+
|
|
3236
|
+
// src/trace/wrappers/mastra.ts
|
|
3237
|
+
function wrapMastraAgent(agent, sessionCtx) {
|
|
3238
|
+
const originalGenerate = agent.generate.bind(agent);
|
|
3239
|
+
const ctx = sessionCtx;
|
|
3240
|
+
agent.generate = async function(...args) {
|
|
3241
|
+
if (!isInitialized()) {
|
|
3242
|
+
return originalGenerate(...args);
|
|
2935
3243
|
}
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
);
|
|
2955
|
-
const scores = modelResults.map((r) => r[camelMetric]).filter((s) => s !== void 0);
|
|
2956
|
-
if (scores.length > 0) {
|
|
2957
|
-
const avg = scores.reduce((a, b) => a + b, 0) / scores.length;
|
|
2958
|
-
row += `${(avg * 100).toFixed(1)}%`.padEnd(15);
|
|
2959
|
-
} else {
|
|
2960
|
-
row += "N/A".padEnd(15);
|
|
3244
|
+
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
3245
|
+
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
3246
|
+
const spanId = generateHexId(16);
|
|
3247
|
+
const parentSpanId = traceCtx?.parentSpanId;
|
|
3248
|
+
const input = args[0];
|
|
3249
|
+
const startTime = Date.now();
|
|
3250
|
+
const captureContent2 = shouldCaptureContent();
|
|
3251
|
+
try {
|
|
3252
|
+
const result = await originalGenerate(...args);
|
|
3253
|
+
const endTime = Date.now();
|
|
3254
|
+
const attributes = {
|
|
3255
|
+
"fallom.sdk_version": "2",
|
|
3256
|
+
"fallom.method": "agent.generate",
|
|
3257
|
+
"fallom.agent_name": agent.name || "unknown"
|
|
3258
|
+
};
|
|
3259
|
+
if (captureContent2) {
|
|
3260
|
+
attributes["fallom.raw.request"] = JSON.stringify(input);
|
|
3261
|
+
attributes["fallom.raw.response"] = JSON.stringify(result);
|
|
2961
3262
|
}
|
|
3263
|
+
sendTrace({
|
|
3264
|
+
config_key: ctx.configKey,
|
|
3265
|
+
session_id: ctx.sessionId,
|
|
3266
|
+
customer_id: ctx.customerId,
|
|
3267
|
+
trace_id: traceId,
|
|
3268
|
+
span_id: spanId,
|
|
3269
|
+
parent_span_id: parentSpanId,
|
|
3270
|
+
name: `agent.${agent.name || "unknown"}.generate`,
|
|
3271
|
+
kind: "agent",
|
|
3272
|
+
start_time: new Date(startTime).toISOString(),
|
|
3273
|
+
end_time: new Date(endTime).toISOString(),
|
|
3274
|
+
duration_ms: endTime - startTime,
|
|
3275
|
+
status: "OK",
|
|
3276
|
+
attributes
|
|
3277
|
+
}).catch(() => {
|
|
3278
|
+
});
|
|
3279
|
+
return result;
|
|
3280
|
+
} catch (error) {
|
|
3281
|
+
const endTime = Date.now();
|
|
3282
|
+
sendTrace({
|
|
3283
|
+
config_key: ctx.configKey,
|
|
3284
|
+
session_id: ctx.sessionId,
|
|
3285
|
+
customer_id: ctx.customerId,
|
|
3286
|
+
trace_id: traceId,
|
|
3287
|
+
span_id: spanId,
|
|
3288
|
+
parent_span_id: parentSpanId,
|
|
3289
|
+
name: `agent.${agent.name || "unknown"}.generate`,
|
|
3290
|
+
kind: "agent",
|
|
3291
|
+
start_time: new Date(startTime).toISOString(),
|
|
3292
|
+
end_time: new Date(endTime).toISOString(),
|
|
3293
|
+
duration_ms: endTime - startTime,
|
|
3294
|
+
status: "ERROR",
|
|
3295
|
+
error_message: error?.message,
|
|
3296
|
+
attributes: {
|
|
3297
|
+
"fallom.sdk_version": "2",
|
|
3298
|
+
"fallom.method": "agent.generate",
|
|
3299
|
+
"fallom.agent_name": agent.name || "unknown"
|
|
3300
|
+
}
|
|
3301
|
+
}).catch(() => {
|
|
3302
|
+
});
|
|
3303
|
+
throw error;
|
|
2962
3304
|
}
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
}
|
|
2966
|
-
async function _uploadResults(results, name, description, judgeModel, verbose) {
|
|
2967
|
-
const allResults = Array.isArray(results) ? results : Object.values(results).flat();
|
|
2968
|
-
const uniqueItems = new Set(
|
|
2969
|
-
allResults.map((r) => `${r.input}|${r.systemMessage || ""}`)
|
|
2970
|
-
);
|
|
2971
|
-
const payload = {
|
|
2972
|
-
name,
|
|
2973
|
-
description,
|
|
2974
|
-
dataset_size: uniqueItems.size,
|
|
2975
|
-
judge_model: judgeModel,
|
|
2976
|
-
results: allResults.map((r) => ({
|
|
2977
|
-
input: r.input,
|
|
2978
|
-
system_message: r.systemMessage,
|
|
2979
|
-
model: r.model,
|
|
2980
|
-
output: r.output,
|
|
2981
|
-
is_production: r.isProduction,
|
|
2982
|
-
answer_relevancy: r.answerRelevancy,
|
|
2983
|
-
hallucination: r.hallucination,
|
|
2984
|
-
toxicity: r.toxicity,
|
|
2985
|
-
faithfulness: r.faithfulness,
|
|
2986
|
-
completeness: r.completeness,
|
|
2987
|
-
reasoning: r.reasoning,
|
|
2988
|
-
latency_ms: r.latencyMs,
|
|
2989
|
-
tokens_in: r.tokensIn,
|
|
2990
|
-
tokens_out: r.tokensOut,
|
|
2991
|
-
cost: r.cost
|
|
2992
|
-
}))
|
|
2993
|
-
};
|
|
2994
|
-
try {
|
|
2995
|
-
const response = await fetch(`${_baseUrl}/api/sdk-evals`, {
|
|
2996
|
-
method: "POST",
|
|
2997
|
-
headers: {
|
|
2998
|
-
Authorization: `Bearer ${_apiKey}`,
|
|
2999
|
-
"Content-Type": "application/json"
|
|
3000
|
-
},
|
|
3001
|
-
body: JSON.stringify(payload)
|
|
3002
|
-
});
|
|
3003
|
-
if (!response.ok) {
|
|
3004
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
3005
|
-
}
|
|
3006
|
-
const data = await response.json();
|
|
3007
|
-
const dashboardUrl = `${_baseUrl}/evals/${data.run_id}`;
|
|
3008
|
-
if (verbose) {
|
|
3009
|
-
console.log(`
|
|
3010
|
-
\u2705 Results uploaded to Fallom! View at: ${dashboardUrl}`);
|
|
3011
|
-
}
|
|
3012
|
-
return dashboardUrl;
|
|
3013
|
-
} catch (error) {
|
|
3014
|
-
if (verbose) {
|
|
3015
|
-
console.log(`
|
|
3016
|
-
\u26A0\uFE0F Failed to upload results: ${error}`);
|
|
3017
|
-
}
|
|
3018
|
-
return "";
|
|
3019
|
-
}
|
|
3305
|
+
};
|
|
3306
|
+
return agent;
|
|
3020
3307
|
}
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3308
|
+
|
|
3309
|
+
// src/trace/session.ts
|
|
3310
|
+
var FallomSession = class {
|
|
3311
|
+
constructor(options) {
|
|
3312
|
+
this.ctx = {
|
|
3313
|
+
configKey: options.configKey,
|
|
3314
|
+
sessionId: options.sessionId,
|
|
3315
|
+
customerId: options.customerId
|
|
3316
|
+
};
|
|
3024
3317
|
}
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3318
|
+
/** Get the session context. */
|
|
3319
|
+
getContext() {
|
|
3320
|
+
return { ...this.ctx };
|
|
3321
|
+
}
|
|
3322
|
+
/**
|
|
3323
|
+
* Get model assignment for this session (A/B testing).
|
|
3324
|
+
*/
|
|
3325
|
+
async getModel(configKeyOrOptions, options) {
|
|
3326
|
+
let configKey;
|
|
3327
|
+
let opts;
|
|
3328
|
+
if (typeof configKeyOrOptions === "string") {
|
|
3329
|
+
configKey = configKeyOrOptions;
|
|
3330
|
+
opts = options || {};
|
|
3331
|
+
} else {
|
|
3332
|
+
configKey = this.ctx.configKey;
|
|
3333
|
+
opts = configKeyOrOptions || {};
|
|
3334
|
+
}
|
|
3335
|
+
const { get: get3 } = await Promise.resolve().then(() => (init_models(), models_exports));
|
|
3336
|
+
return get3(configKey, this.ctx.sessionId, opts);
|
|
3337
|
+
}
|
|
3338
|
+
/**
|
|
3339
|
+
* Wrap a Vercel AI SDK model to trace all calls (PostHog style).
|
|
3340
|
+
* Returns the same model type with tracing injected.
|
|
3341
|
+
*
|
|
3342
|
+
* Note: This only captures tokens/timing, not prompt/completion content.
|
|
3343
|
+
* Use wrapAISDK for full content tracing.
|
|
3344
|
+
*/
|
|
3345
|
+
traceModel(model) {
|
|
3346
|
+
const ctx = this.ctx;
|
|
3347
|
+
const tracedModel = Object.create(model);
|
|
3348
|
+
const m = model;
|
|
3349
|
+
if (m.doGenerate) {
|
|
3350
|
+
const originalDoGenerate = m.doGenerate.bind(model);
|
|
3351
|
+
tracedModel.doGenerate = async function(...args) {
|
|
3352
|
+
if (!isInitialized()) return originalDoGenerate(...args);
|
|
3353
|
+
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
3354
|
+
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
3355
|
+
const spanId = generateHexId(16);
|
|
3356
|
+
const startTime = Date.now();
|
|
3357
|
+
try {
|
|
3358
|
+
const result = await originalDoGenerate(...args);
|
|
3359
|
+
const endTime = Date.now();
|
|
3360
|
+
const modelId = model.modelId || "unknown";
|
|
3361
|
+
const usage = result?.usage || result?.rawResponse?.usage;
|
|
3362
|
+
sendTrace({
|
|
3363
|
+
config_key: ctx.configKey,
|
|
3364
|
+
session_id: ctx.sessionId,
|
|
3365
|
+
customer_id: ctx.customerId,
|
|
3366
|
+
trace_id: traceId,
|
|
3367
|
+
span_id: spanId,
|
|
3368
|
+
parent_span_id: traceCtx?.parentSpanId,
|
|
3369
|
+
name: "doGenerate",
|
|
3370
|
+
kind: "llm",
|
|
3371
|
+
model: modelId,
|
|
3372
|
+
start_time: new Date(startTime).toISOString(),
|
|
3373
|
+
end_time: new Date(endTime).toISOString(),
|
|
3374
|
+
duration_ms: endTime - startTime,
|
|
3375
|
+
status: "OK",
|
|
3376
|
+
attributes: {
|
|
3377
|
+
"fallom.sdk_version": "2",
|
|
3378
|
+
"fallom.method": "traceModel.doGenerate",
|
|
3379
|
+
...usage ? { "fallom.raw.usage": JSON.stringify(usage) } : {}
|
|
3380
|
+
}
|
|
3381
|
+
}).catch(() => {
|
|
3382
|
+
});
|
|
3383
|
+
return result;
|
|
3384
|
+
} catch (error) {
|
|
3385
|
+
const endTime = Date.now();
|
|
3386
|
+
sendTrace({
|
|
3387
|
+
config_key: ctx.configKey,
|
|
3388
|
+
session_id: ctx.sessionId,
|
|
3389
|
+
customer_id: ctx.customerId,
|
|
3390
|
+
trace_id: traceId,
|
|
3391
|
+
span_id: spanId,
|
|
3392
|
+
parent_span_id: traceCtx?.parentSpanId,
|
|
3393
|
+
name: "doGenerate",
|
|
3394
|
+
kind: "llm",
|
|
3395
|
+
model: model.modelId || "unknown",
|
|
3396
|
+
start_time: new Date(startTime).toISOString(),
|
|
3397
|
+
end_time: new Date(endTime).toISOString(),
|
|
3398
|
+
duration_ms: endTime - startTime,
|
|
3399
|
+
status: "ERROR",
|
|
3400
|
+
error_message: error instanceof Error ? error.message : String(error),
|
|
3401
|
+
attributes: { "fallom.sdk_version": "2", "fallom.method": "traceModel.doGenerate" }
|
|
3402
|
+
}).catch(() => {
|
|
3403
|
+
});
|
|
3404
|
+
throw error;
|
|
3405
|
+
}
|
|
3406
|
+
};
|
|
3039
3407
|
}
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3408
|
+
if (m.doStream) {
|
|
3409
|
+
const originalDoStream = m.doStream.bind(model);
|
|
3410
|
+
tracedModel.doStream = async function(...args) {
|
|
3411
|
+
if (!isInitialized()) return originalDoStream(...args);
|
|
3412
|
+
const traceCtx = getTraceContextStorage().getStore() || getFallbackTraceContext();
|
|
3413
|
+
const traceId = traceCtx?.traceId || generateHexId(32);
|
|
3414
|
+
const spanId = generateHexId(16);
|
|
3415
|
+
const startTime = Date.now();
|
|
3416
|
+
const modelId = model.modelId || "unknown";
|
|
3417
|
+
try {
|
|
3418
|
+
const result = await originalDoStream(...args);
|
|
3419
|
+
sendTrace({
|
|
3420
|
+
config_key: ctx.configKey,
|
|
3421
|
+
session_id: ctx.sessionId,
|
|
3422
|
+
customer_id: ctx.customerId,
|
|
3423
|
+
trace_id: traceId,
|
|
3424
|
+
span_id: spanId,
|
|
3425
|
+
parent_span_id: traceCtx?.parentSpanId,
|
|
3426
|
+
name: "doStream",
|
|
3427
|
+
kind: "llm",
|
|
3428
|
+
model: modelId,
|
|
3429
|
+
start_time: new Date(startTime).toISOString(),
|
|
3430
|
+
end_time: new Date(Date.now()).toISOString(),
|
|
3431
|
+
duration_ms: Date.now() - startTime,
|
|
3432
|
+
status: "OK",
|
|
3433
|
+
is_streaming: true,
|
|
3434
|
+
attributes: {
|
|
3435
|
+
"fallom.sdk_version": "2",
|
|
3436
|
+
"fallom.method": "traceModel.doStream",
|
|
3437
|
+
"fallom.is_streaming": true
|
|
3438
|
+
}
|
|
3439
|
+
}).catch(() => {
|
|
3440
|
+
});
|
|
3441
|
+
return result;
|
|
3442
|
+
} catch (error) {
|
|
3443
|
+
sendTrace({
|
|
3444
|
+
config_key: ctx.configKey,
|
|
3445
|
+
session_id: ctx.sessionId,
|
|
3446
|
+
customer_id: ctx.customerId,
|
|
3447
|
+
trace_id: traceId,
|
|
3448
|
+
span_id: spanId,
|
|
3449
|
+
parent_span_id: traceCtx?.parentSpanId,
|
|
3450
|
+
name: "doStream",
|
|
3451
|
+
kind: "llm",
|
|
3452
|
+
model: modelId,
|
|
3453
|
+
start_time: new Date(startTime).toISOString(),
|
|
3454
|
+
end_time: new Date(Date.now()).toISOString(),
|
|
3455
|
+
duration_ms: Date.now() - startTime,
|
|
3456
|
+
status: "ERROR",
|
|
3457
|
+
error_message: error instanceof Error ? error.message : String(error),
|
|
3458
|
+
is_streaming: true,
|
|
3459
|
+
attributes: {
|
|
3460
|
+
"fallom.sdk_version": "2",
|
|
3461
|
+
"fallom.method": "traceModel.doStream",
|
|
3462
|
+
"fallom.is_streaming": true
|
|
3463
|
+
}
|
|
3464
|
+
}).catch(() => {
|
|
3465
|
+
});
|
|
3466
|
+
throw error;
|
|
3467
|
+
}
|
|
3468
|
+
};
|
|
3044
3469
|
}
|
|
3470
|
+
return tracedModel;
|
|
3045
3471
|
}
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
if (!_initialized) {
|
|
3050
|
-
throw new Error("Fallom evals not initialized. Call evals.init() first.");
|
|
3472
|
+
/** Wrap OpenAI client. Delegates to shared wrapper. */
|
|
3473
|
+
wrapOpenAI(client) {
|
|
3474
|
+
return wrapOpenAI(client, this.ctx);
|
|
3051
3475
|
}
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3476
|
+
/** Wrap Anthropic client. Delegates to shared wrapper. */
|
|
3477
|
+
wrapAnthropic(client) {
|
|
3478
|
+
return wrapAnthropic(client, this.ctx);
|
|
3055
3479
|
}
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
"Content-Type": "application/json"
|
|
3060
|
-
}
|
|
3061
|
-
});
|
|
3062
|
-
if (response.status === 404) {
|
|
3063
|
-
throw new Error(`Dataset '${datasetKey}' not found`);
|
|
3064
|
-
} else if (response.status === 403) {
|
|
3065
|
-
throw new Error(`Access denied to dataset '${datasetKey}'`);
|
|
3480
|
+
/** Wrap Google AI model. Delegates to shared wrapper. */
|
|
3481
|
+
wrapGoogleAI(model) {
|
|
3482
|
+
return wrapGoogleAI(model, this.ctx);
|
|
3066
3483
|
}
|
|
3067
|
-
|
|
3068
|
-
|
|
3484
|
+
/** Wrap Vercel AI SDK. Delegates to shared wrapper. */
|
|
3485
|
+
wrapAISDK(ai, options) {
|
|
3486
|
+
return wrapAISDK(ai, this.ctx, options);
|
|
3487
|
+
}
|
|
3488
|
+
/** Wrap Mastra agent. Delegates to shared wrapper. */
|
|
3489
|
+
wrapMastraAgent(agent) {
|
|
3490
|
+
return wrapMastraAgent(agent, this.ctx);
|
|
3069
3491
|
}
|
|
3070
|
-
const data = await response.json();
|
|
3071
|
-
const items = data.entries.map((entry) => ({
|
|
3072
|
-
input: entry.input,
|
|
3073
|
-
output: entry.output,
|
|
3074
|
-
systemMessage: entry.systemMessage,
|
|
3075
|
-
metadata: entry.metadata
|
|
3076
|
-
}));
|
|
3077
|
-
const datasetName = data.dataset.name || datasetKey;
|
|
3078
|
-
const versionNum = data.version.version || "latest";
|
|
3079
|
-
console.log(
|
|
3080
|
-
`\u2713 Loaded dataset '${datasetName}' (version ${versionNum}) with ${items.length} entries`
|
|
3081
|
-
);
|
|
3082
|
-
return items;
|
|
3083
|
-
}
|
|
3084
|
-
var evals_default = {
|
|
3085
|
-
init: init4,
|
|
3086
|
-
evaluate,
|
|
3087
|
-
compareModels,
|
|
3088
|
-
uploadResults,
|
|
3089
|
-
datasetFromTraces,
|
|
3090
|
-
datasetFromFallom,
|
|
3091
|
-
AVAILABLE_METRICS
|
|
3092
3492
|
};
|
|
3493
|
+
function session(options) {
|
|
3494
|
+
return new FallomSession(options);
|
|
3495
|
+
}
|
|
3496
|
+
|
|
3497
|
+
// src/index.ts
|
|
3498
|
+
init_models();
|
|
3499
|
+
|
|
3500
|
+
// src/evals/index.ts
|
|
3501
|
+
var evals_exports = {};
|
|
3502
|
+
__export(evals_exports, {
|
|
3503
|
+
AVAILABLE_METRICS: () => AVAILABLE_METRICS,
|
|
3504
|
+
DEFAULT_JUDGE_MODEL: () => DEFAULT_JUDGE_MODEL,
|
|
3505
|
+
METRIC_PROMPTS: () => METRIC_PROMPTS,
|
|
3506
|
+
compareModels: () => compareModels,
|
|
3507
|
+
createCustomModel: () => createCustomModel,
|
|
3508
|
+
createModelFromCallable: () => createModelFromCallable,
|
|
3509
|
+
createOpenAIModel: () => createOpenAIModel,
|
|
3510
|
+
customMetric: () => customMetric,
|
|
3511
|
+
datasetFromFallom: () => datasetFromFallom,
|
|
3512
|
+
datasetFromTraces: () => datasetFromTraces,
|
|
3513
|
+
evaluate: () => evaluate,
|
|
3514
|
+
getMetricName: () => getMetricName,
|
|
3515
|
+
init: () => init4,
|
|
3516
|
+
isCustomMetric: () => isCustomMetric,
|
|
3517
|
+
uploadResults: () => uploadResultsPublic
|
|
3518
|
+
});
|
|
3519
|
+
init_types();
|
|
3520
|
+
init_prompts();
|
|
3521
|
+
init_core();
|
|
3522
|
+
init_helpers();
|
|
3093
3523
|
|
|
3094
3524
|
// src/init.ts
|
|
3095
3525
|
init_models();
|
|
@@ -3114,7 +3544,7 @@ async function init5(options = {}) {
|
|
|
3114
3544
|
}
|
|
3115
3545
|
|
|
3116
3546
|
// src/mastra.ts
|
|
3117
|
-
var
|
|
3547
|
+
var import_core13 = require("@opentelemetry/core");
|
|
3118
3548
|
var promptContext2 = {};
|
|
3119
3549
|
function setMastraPrompt(promptKey, version) {
|
|
3120
3550
|
promptContext2 = {
|
|
@@ -3164,7 +3594,7 @@ var FallomExporter = class {
|
|
|
3164
3594
|
*/
|
|
3165
3595
|
export(spans, resultCallback) {
|
|
3166
3596
|
if (spans.length === 0) {
|
|
3167
|
-
resultCallback({ code:
|
|
3597
|
+
resultCallback({ code: import_core13.ExportResultCode.SUCCESS });
|
|
3168
3598
|
return;
|
|
3169
3599
|
}
|
|
3170
3600
|
this.log(`Exporting ${spans.length} spans...`);
|
|
@@ -3181,11 +3611,11 @@ var FallomExporter = class {
|
|
|
3181
3611
|
}
|
|
3182
3612
|
const exportPromise = this.sendSpans(spans).then(() => {
|
|
3183
3613
|
this.log("Export successful");
|
|
3184
|
-
resultCallback({ code:
|
|
3614
|
+
resultCallback({ code: import_core13.ExportResultCode.SUCCESS });
|
|
3185
3615
|
}).catch((error) => {
|
|
3186
3616
|
console.error("[FallomExporter] Export failed:", error);
|
|
3187
3617
|
resultCallback({
|
|
3188
|
-
code:
|
|
3618
|
+
code: import_core13.ExportResultCode.FAILED,
|
|
3189
3619
|
error: error instanceof Error ? error : new Error(String(error))
|
|
3190
3620
|
});
|
|
3191
3621
|
});
|