judgeval 0.1.33

package/dist/common/tracer.js

@@ -0,0 +1,1035 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.currentSpanAsyncLocalStorage = exports.currentTraceAsyncLocalStorage = exports.TraceManagerClient = exports.TraceClient = exports.Tracer = exports.wrap = void 0;
+// Core Node.js imports
+const async_hooks_1 = require("async_hooks");
+const uuid_1 = require("uuid");
+// Installed SDKs
+const openai_1 = __importDefault(require("openai"));
+const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
+const together_ai_1 = __importDefault(require("together-ai")); // Use default import
+// Local Imports
+const constants_1 = require("../constants");
+const logger_instance_1 = __importDefault(require("./logger-instance")); // Use the shared winston logger instance
+// --- API Interaction Client ---
+/**
+ * Client for interacting with Judgment trace API endpoints.
+ */
+class TraceManagerClient {
+    constructor(apiKey, organizationId) {
+        if (!apiKey) {
+            throw new Error("TraceManagerClient requires a Judgment API key.");
+        }
+        if (!organizationId) {
+            throw new Error("TraceManagerClient requires a Judgment Organization ID.");
+        }
+        this.apiKey = apiKey;
+        this.organizationId = organizationId;
+    }
+    _fetch(url_1) {
+        return __awaiter(this, arguments, void 0, function* (url, options = {}) {
+            const headers = Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}`, 'X-Organization-Id': this.organizationId }, (options.headers || {}));
+            try {
+                // Use isomorphic fetch (available globally in modern Node.js and browsers)
+                const response = yield fetch(url, Object.assign(Object.assign({}, options), { headers: headers }));
+                if (!response.ok) {
+                    const errorBody = yield response.text();
+                    console.error(`API Error (${response.status}) for ${options.method || 'GET'} ${url}: ${errorBody}`);
+                    throw new Error(`Judgment API request failed: ${response.status} ${response.statusText} - ${errorBody}`);
+                }
+                // Handle cases where the response might be empty (e.g., 204 No Content on DELETE)
+                if (response.status === 204) {
+                    return null; // Indicate success with no content
+                }
+                return yield response.json();
+            }
+            catch (error) {
+                console.error(`Network or fetch error during ${options.method || 'GET'} ${url}:`, error);
+                // Re-throw or handle as appropriate for the application context
+                throw error;
+            }
+        });
+    }
+    fetchTrace(traceId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this._fetch(constants_1.JUDGMENT_TRACES_FETCH_API_URL, {
+                method: 'POST',
+                body: JSON.stringify({ trace_id: traceId }),
+            });
+        });
+    }
+    saveTrace(traceData, emptySave) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const response = yield this._fetch(constants_1.JUDGMENT_TRACES_SAVE_API_URL, {
+                method: 'POST',
+                body: JSON.stringify(traceData),
+            });
+            // Optionally log the UI URL like the Python version
+            if (!emptySave && (response === null || response === void 0 ? void 0 : response.ui_results_url)) {
+                // Use console.info or a dedicated logger for user-facing messages
+                // Note: We can't replicate Rich library's colored link easily in standard console
+                console.info(`
+🔍 View trace: ${response.ui_results_url}
+`);
+            }
+            return response;
+        });
+    }
+    deleteTrace(traceId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Assuming DELETE method is correct based on REST principles for the delete endpoint
+            return this._fetch(constants_1.JUDGMENT_TRACES_DELETE_API_URL, {
+                method: 'DELETE',
+                body: JSON.stringify({ trace_ids: [traceId] }),
+            });
+        });
+    }
+    deleteTraces(traceIds) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return this._fetch(constants_1.JUDGMENT_TRACES_DELETE_API_URL, {
+                method: 'DELETE',
+                body: JSON.stringify({ trace_ids: traceIds }),
+            });
+        });
+    }
+    addTraceToEvalQueue(traceData) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Ensure traceData has the necessary structure for the queue endpoint
+            return this._fetch(constants_1.JUDGMENT_TRACES_ADD_TO_EVAL_QUEUE_API_URL, {
+                method: 'POST',
+                body: JSON.stringify(traceData), // Send the full trace data as per Python impl
+            });
+        });
+    }
+}
+exports.TraceManagerClient = TraceManagerClient;
+// --- Context Management ---
+// Holds the active TraceClient instance for the current async context
+const currentTraceAsyncLocalStorage = new async_hooks_1.AsyncLocalStorage();
+exports.currentTraceAsyncLocalStorage = currentTraceAsyncLocalStorage;
+// Holds the ID of the currently active span within a trace
+const currentSpanAsyncLocalStorage = new async_hooks_1.AsyncLocalStorage();
+exports.currentSpanAsyncLocalStorage = currentSpanAsyncLocalStorage;
+// --- Helper Functions ---
+// Helper function to sanitize names (e.g., replace spaces with underscores)
+function sanitizeName(name) {
+    // Replace spaces with underscores and remove potentially problematic characters
+    // You can adjust the regex further if other characters cause issues.
+    return name.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_.-]/g, '');
+}
+// --- Core Trace Classes ---
+/**
+ * Represents an ongoing trace context.
+ */
+class TraceClient {
+    constructor(config) {
+        var _a, _b, _c, _d, _e;
+        // Internal state
+        this.entries = [];
+        this.spanDepths = {};
+        this.traceManager = null; // Can be null if monitoring disabled
+        this.traceId = config.traceId || (0, uuid_1.v4)();
+        this.originalName = config.name || 'default_trace'; // Store original
+        this.name = sanitizeName(this.originalName); // Use sanitized name internally
+        // If the sanitized name is empty, fallback to a default
+        if (!this.name) {
+            console.warn(`Original trace name "${this.originalName}" sanitized to empty string. Using default_trace_${this.traceId.substring(0, 8)}.`);
+            this.name = `default_trace_${this.traceId.substring(0, 8)}`;
+        }
+        this.projectName = (_a = config.projectName) !== null && _a !== void 0 ? _a : config.tracer.projectName;
+        this.overwrite = (_b = config.overwrite) !== null && _b !== void 0 ? _b : false;
+        this.rules = (_c = config.rules) !== null && _c !== void 0 ? _c : [];
+        // Determine effective monitoring status based on tracer and API keys
+        let effectiveMonitoring = (_d = config.enableMonitoring) !== null && _d !== void 0 ? _d : config.tracer.enableMonitoring;
+        if (effectiveMonitoring && (!config.apiKey || !config.organizationId)) {
+            console.warn(`TraceClient ${this.traceId}: Monitoring requires JUDGMENT_API_KEY and JUDGMENT_ORG_ID. Disabling monitoring for this trace.`);
+            effectiveMonitoring = false;
+        }
+        this.enableMonitoring = effectiveMonitoring;
+        // Evaluations depend on monitoring
+        this.enableEvaluations = effectiveMonitoring && ((_e = config.enableEvaluations) !== null && _e !== void 0 ? _e : config.tracer.enableEvaluations);
+        this.parentTraceId = config.parentTraceId;
+        this.parentName = config.parentName;
+        this.apiKey = config.apiKey;
+        this.organizationId = config.organizationId;
+        this.startTime = Date.now() / 1000;
+        if (this.enableMonitoring) {
+            this.traceManager = new TraceManagerClient(this.apiKey, this.organizationId);
+        }
+    }
+    addEntry(entry) {
+        if (!this.enableMonitoring)
+            return;
+        entry.timestamp = entry.timestamp || Date.now() / 1000;
+        this.entries.push(entry);
+    }
+    recordInput(inputs) {
+        var _a, _b;
+        const spanId = currentSpanAsyncLocalStorage.getStore();
+        if (!spanId || !this.enableMonitoring)
+            return;
+        const enterEntry = this.entries.find(e => e.span_id === spanId && e.type === 'enter');
+        const functionName = (enterEntry === null || enterEntry === void 0 ? void 0 : enterEntry.function) || 'unknown_function';
+        const depth = (_b = (_a = enterEntry === null || enterEntry === void 0 ? void 0 : enterEntry.depth) !== null && _a !== void 0 ? _a : this.spanDepths[spanId]) !== null && _b !== void 0 ? _b : 0;
+        const spanType = (enterEntry === null || enterEntry === void 0 ? void 0 : enterEntry.span_type) || 'span';
+        this.addEntry({
+            type: 'input',
+            span_id: spanId,
+            inputs,
+            function: functionName,
+            depth: depth,
+            span_type: spanType
+        });
+    }
+    recordOutput(output) {
+        var _a, _b, _c;
+        const spanId = currentSpanAsyncLocalStorage.getStore();
+        if (!spanId || !this.enableMonitoring)
+            return;
+        const enterEntry = this.entries.find(e => e.span_id === spanId && e.type === 'enter');
+        const functionName = (enterEntry === null || enterEntry === void 0 ? void 0 : enterEntry.function) || 'unknown_function';
+        const depth = (_b = (_a = enterEntry === null || enterEntry === void 0 ? void 0 : enterEntry.depth) !== null && _a !== void 0 ? _a : this.spanDepths[spanId]) !== null && _b !== void 0 ? _b : 0;
+        const spanType = (enterEntry === null || enterEntry === void 0 ? void 0 : enterEntry.span_type) || 'span';
+        let entryType = 'output';
+        let outputData = output;
+        if (output instanceof Error) {
+            entryType = 'error';
+            outputData = {
+                name: output.name,
+                message: output.message,
+                stack: (_c = output.stack) === null || _c === void 0 ? void 0 : _c.substring(0, 1000)
+            };
+        }
+        else if (output === null || output === void 0 ? void 0 : output.error) {
+            entryType = 'error';
+            outputData = output;
+        }
+        this.addEntry({
+            type: entryType,
+            span_id: spanId,
+            output: outputData,
+            function: functionName,
+            depth: depth,
+            span_type: spanType
+        });
+    }
+    runInSpan(name, options, func) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            if (!this.enableMonitoring) {
+                const result = func();
+                return result instanceof Promise ? yield result : result;
+            }
+            const startTime = Date.now() / 1000;
+            const spanId = (0, uuid_1.v4)();
+            const parentSpanId = currentSpanAsyncLocalStorage.getStore();
+            const spanType = options.spanType || 'span';
+            let currentDepth = 0;
+            if (parentSpanId && this.spanDepths[parentSpanId] !== undefined) {
+                currentDepth = this.spanDepths[parentSpanId] + 1;
+            }
+            this.spanDepths[spanId] = currentDepth;
+            this.addEntry({
+                type: 'enter',
+                function: name,
+                span_id: spanId,
+                depth: currentDepth,
+                timestamp: startTime,
+                span_type: spanType,
+                parent_span_id: parentSpanId
+            });
+            let result;
+            try {
+                result = yield currentSpanAsyncLocalStorage.run(spanId, () => __awaiter(this, void 0, void 0, function* () {
+                    const res = func();
+                    return res instanceof Promise ? yield res : res;
+                }));
+                return result;
+            }
+            catch (error) {
+                this.recordOutput({
+                    error: String(error),
+                    error_details: error instanceof Error
+                        ? { name: error.name, message: error.message, stack: (_a = error.stack) === null || _a === void 0 ? void 0 : _a.substring(0, 500) }
+                        : { detail: String(error) }
+                });
+                throw error;
+            }
+            finally {
+                const endTime = Date.now() / 1000;
+                const duration = endTime - startTime;
+                this.addEntry({
+                    type: 'exit',
+                    function: name,
+                    span_id: spanId,
+                    depth: currentDepth,
+                    timestamp: endTime,
+                    duration: duration,
+                    span_type: spanType
+                });
+                delete this.spanDepths[spanId];
+            }
+        });
+    }
+    getDuration() {
+        return (Date.now() / 1000) - this.startTime;
+    }
+    condenseTrace(rawEntries) {
+        var _a, _b, _c, _d, _e;
+        const spansById = {};
+        for (const entry of rawEntries) {
+            const spanId = entry.span_id;
+            if (!spanId)
+                continue;
+            if (!spansById[spanId]) {
+                spansById[spanId] = {
+                    span_id: spanId,
+                    function: entry.function || 'unknown',
+                    depth: (_a = entry.depth) !== null && _a !== void 0 ? _a : 0,
+                    timestamp: (_b = entry.timestamp) !== null && _b !== void 0 ? _b : 0,
+                    parent_span_id: entry.parent_span_id,
+                    span_type: entry.span_type || 'span',
+                    inputs: null,
+                    output: null,
+                    evaluation_runs: [],
+                    duration: null,
+                    children: []
+                };
+            }
+            const currentSpanData = spansById[spanId];
+            switch (entry.type) {
+                case 'enter':
+                    currentSpanData.function = entry.function || currentSpanData.function;
+                    currentSpanData.depth = (_c = entry.depth) !== null && _c !== void 0 ? _c : currentSpanData.depth;
+                    currentSpanData.timestamp = (_d = entry.timestamp) !== null && _d !== void 0 ? _d : currentSpanData.timestamp;
+                    currentSpanData.parent_span_id = entry.parent_span_id;
+                    currentSpanData.span_type = entry.span_type || currentSpanData.span_type;
+                    currentSpanData.start_time = entry.timestamp;
+                    break;
+                case 'exit':
+                    currentSpanData.duration = (_e = entry.duration) !== null && _e !== void 0 ? _e : currentSpanData.duration;
+                    currentSpanData.end_time = entry.timestamp;
+                    if (currentSpanData.duration === null && currentSpanData.start_time && currentSpanData.end_time) {
+                        currentSpanData.duration = currentSpanData.end_time - currentSpanData.start_time;
+                    }
+                    break;
+                case 'input':
+                    if (currentSpanData.inputs === null && entry.inputs) {
+                        currentSpanData.inputs = entry.inputs;
+                    }
+                    else if (typeof currentSpanData.inputs === 'object' && typeof entry.inputs === 'object') {
+                        currentSpanData.inputs = Object.assign(Object.assign({}, currentSpanData.inputs), entry.inputs);
+                    }
+                    break;
+                case 'output':
+                case 'error':
+                    currentSpanData.output = entry.output;
+                    break;
+                case 'evaluation':
+                    if (entry.evaluation_runs) {
+                        currentSpanData.evaluation_runs.push(...entry.evaluation_runs);
+                    }
+                    break;
+            }
+        }
+        const spansList = Object.values(spansById).map(span => {
+            if (span.duration === null && span.start_time && span.end_time) {
+                span.duration = span.end_time - span.start_time;
+            }
+            delete span.start_time;
+            delete span.end_time;
+            return span;
+        });
+        const childrenMap = {};
+        const roots = [];
+        const spanMap = {};
+        const sortedCondensedList = [];
+        const visited = new Set();
+        for (const span of spansList) {
+            spanMap[span.span_id] = span;
+            const parentId = span.parent_span_id;
+            if (parentId === undefined || parentId === null) {
+                roots.push(span);
+            }
+            else {
+                if (!childrenMap[parentId]) {
+                    childrenMap[parentId] = [];
+                }
+                childrenMap[parentId].push(span);
+            }
+        }
+        roots.sort((a, b) => a.timestamp - b.timestamp);
+        for (const parentId in childrenMap) {
+            childrenMap[parentId].sort((a, b) => a.timestamp - b.timestamp);
+        }
+        function buildFlatListDfs(span) {
+            if (visited.has(span.span_id))
+                return;
+            visited.add(span.span_id);
+            sortedCondensedList.push(span);
+            const children = childrenMap[span.span_id] || [];
+            for (const child of children) {
+                buildFlatListDfs(child);
+            }
+        }
+        for (const rootSpan of roots) {
+            buildFlatListDfs(rootSpan);
+        }
+        for (const span of spansList) {
+            if (!visited.has(span.span_id)) {
+                console.warn(`Orphaned span detected: ${span.span_id}, adding to end of list.`);
+                buildFlatListDfs(span);
+            }
+        }
+        return sortedCondensedList;
+    }
+    save() {
+        return __awaiter(this, arguments, void 0, function* (emptySave = false) {
+            if (!this.enableMonitoring || !this.traceManager) {
+                return null;
+            }
+            const totalDuration = this.getDuration();
+            const condensedEntries = this.condenseTrace(this.entries);
+            const tokenCounts = {
+                prompt_tokens: 0, completion_tokens: 0, total_tokens: 0,
+                prompt_tokens_cost_usd: 0.0, completion_tokens_cost_usd: 0.0, total_cost_usd: 0.0
+            };
+            condensedEntries.forEach(entry => {
+                var _a, _b;
+                if (entry.span_type === 'llm' && ((_a = entry.output) === null || _a === void 0 ? void 0 : _a.usage)) {
+                    const usage = entry.output.usage;
+                    let promptTokens = 0;
+                    let completionTokens = 0;
+                    if (usage.prompt_tokens !== undefined || usage.completion_tokens !== undefined) {
+                        promptTokens = usage.prompt_tokens || 0;
+                        completionTokens = usage.completion_tokens || 0;
+                    }
+                    else if (usage.input_tokens !== undefined || usage.output_tokens !== undefined) {
+                        promptTokens = usage.input_tokens || 0;
+                        completionTokens = usage.output_tokens || 0;
+                        usage.prompt_tokens = promptTokens;
+                        usage.completion_tokens = completionTokens;
+                        delete usage.input_tokens;
+                        delete usage.output_tokens;
+                    }
+                    tokenCounts.prompt_tokens += promptTokens;
+                    tokenCounts.completion_tokens += completionTokens;
+                    tokenCounts.total_tokens += usage.total_tokens || (promptTokens + completionTokens);
+                    const modelName = ((_b = entry.inputs) === null || _b === void 0 ? void 0 : _b.model) || "";
+                    if (modelName) {
+                        try {
+                            const promptCost = 0.0;
+                            const completionCost = 0.0;
+                            const callTotalCost = promptCost + completionCost;
+                            usage.prompt_tokens_cost_usd = promptCost;
+                            usage.completion_tokens_cost_usd = completionCost;
+                            usage.total_cost_usd = callTotalCost;
+                            tokenCounts.prompt_tokens_cost_usd += promptCost;
+                            tokenCounts.completion_tokens_cost_usd += completionCost;
+                            tokenCounts.total_cost_usd += callTotalCost;
+                        }
+                        catch (e) {
+                            console.warn(`Error calculating cost for model '${modelName}':`, e);
+                            usage.prompt_tokens_cost_usd = null;
+                            usage.completion_tokens_cost_usd = null;
+                            usage.total_cost_usd = null;
+                        }
+                    }
+                    else {
+                        usage.prompt_tokens_cost_usd = null;
+                        usage.completion_tokens_cost_usd = null;
+                        usage.total_cost_usd = null;
+                    }
+                }
+            });
+            // Convert rules array to a dictionary (Record<string, Rule>)
+            const rulesDict = {};
+            this.rules.forEach(rule => {
+                var _a;
+                // Use rule_id if available, otherwise fallback to name
+                const key = (_a = rule.rule_id) !== null && _a !== void 0 ? _a : rule.name;
+                rulesDict[key] = rule;
+            });
+            const traceData = {
+                trace_id: this.traceId,
+                name: this.name,
+                project_name: this.projectName,
+                created_at: new Date(this.startTime * 1000).toISOString(),
+                duration: totalDuration,
+                token_counts: tokenCounts,
+                entries: condensedEntries,
+                rules: rulesDict,
+                empty_save: emptySave,
+                overwrite: this.overwrite,
+                parent_trace_id: this.parentTraceId,
+                parent_name: this.parentName
+            };
+            try {
+                yield this.traceManager.saveTrace(traceData, emptySave);
+                logger_instance_1.default.info(`Trace ${this.traceId} saved successfully.`);
+                if (!emptySave && this.enableEvaluations) {
+                    try {
+                        yield this.traceManager.addTraceToEvalQueue(traceData);
+                        logger_instance_1.default.info(`Trace ${this.traceId} added to evaluation queue.`);
+                    }
+                    catch (evalError) {
+                        logger_instance_1.default.warn(`Failed to add trace ${this.traceId} to evaluation queue.`, { error: evalError instanceof Error ? evalError.message : String(evalError) });
+                    }
+                }
+                return { traceId: this.traceId, traceData };
+            }
+            catch (error) {
+                logger_instance_1.default.error(`Failed to save trace ${this.traceId}.`, { error: error instanceof Error ? error.message : String(error) });
+                return null;
+            }
+        });
+    }
+    print() {
+        if (!this.enableMonitoring) {
+            // Keep console.log for direct user output when print() is called
+            console.log("Monitoring was disabled. No trace entries recorded.");
+            return;
+        }
+        if (this.entries.length === 0) {
+            // Keep console.log for direct user output when print() is called
+            console.log("No trace entries recorded.");
+            return;
+        }
+        // Keep console.log for direct user output when print() is called
+        console.log(`\n--- Trace Details: ${this.name} (ID: ${this.traceId}) ---`);
+        this.entries.forEach(entry => {
+            var _a;
+            const indent = "  ".repeat((_a = entry.depth) !== null && _a !== void 0 ? _a : 0);
+            const timeStr = entry.timestamp ? `@ ${new Date(entry.timestamp * 1000).toISOString()}` : '';
+            const shortSpanId = entry.span_id ? `(id: ${entry.span_id.substring(0, 8)}...)` : '';
+            const shortParentId = entry.parent_span_id ? `(parent: ${entry.parent_span_id.substring(0, 8)}...)` : '';
+            try {
+                switch (entry.type) {
+                    case 'enter':
+                        console.log(`${indent}→ ${entry.function || 'unknown'} ${shortSpanId} ${shortParentId} [${entry.span_type || 'span'}] ${timeStr}`);
+                        break;
+                    case 'exit':
+                        const durationStr = entry.duration !== undefined ? `(${entry.duration.toFixed(3)}s)` : '';
+                        // Keep console.log
+                        console.log(`${indent}← ${entry.function || 'unknown'} ${shortSpanId} ${durationStr} ${timeStr}`);
+                        break;
+                    case 'input':
+                        let inputStr = JSON.stringify(entry.inputs);
+                        if (inputStr && inputStr.length > 200) {
+                            inputStr = inputStr.substring(0, 197) + '...';
+                        }
+                        // Keep console.log
+                        console.log(`${indent}  Input (for ${shortSpanId}): ${inputStr || '{}'}`);
+                        break;
+                    case 'output':
+                    case 'error':
+                        let outputStr = JSON.stringify(entry.output);
+                        if (outputStr && outputStr.length > 200) {
+                            outputStr = outputStr.substring(0, 197) + '...';
+                        }
+                        const prefix = entry.type === 'error' ? 'Error' : 'Output';
+                        // Keep console.log
+                        console.log(`${indent}  ${prefix} (for ${shortSpanId}): ${outputStr || 'null'}`);
+                        break;
+                    case 'evaluation':
+                        let evalStr = JSON.stringify(entry.evaluation_runs);
+                        if (evalStr && evalStr.length > 200) {
+                            evalStr = evalStr.substring(0, 197) + '...';
+                        }
+                        // Keep console.log
+                        console.log(`${indent}  Evaluation (for ${shortSpanId}): ${evalStr || '[]'}`);
+                        break;
+                    default:
+                        // Keep console.log
+                        console.log(`${indent}? Unknown entry type: ${JSON.stringify(entry)}`);
+                }
+            }
+            catch (stringifyError) {
+                const errorMessage = stringifyError instanceof Error ? stringifyError.message : String(stringifyError);
+                // Keep console.log
+                console.log(`${indent}! Error formatting entry: ${errorMessage}`);
+                console.log(`${indent}  Raw entry:`, entry);
+            }
+        });
+        // Keep console.log
+        console.log(`--- End Trace: ${this.name} ---`);
+    }
+    delete() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.enableMonitoring || !this.traceManager) {
+                logger_instance_1.default.warn(`Cannot delete trace ${this.traceId}, monitoring disabled or manager missing.`);
+                return null;
+            }
+            try {
+                const result = yield this.traceManager.deleteTrace(this.traceId);
+                logger_instance_1.default.info(`Trace ${this.traceId} deleted successfully.`);
+                return result;
+            }
+            catch (error) {
+                logger_instance_1.default.error(`Failed to delete trace ${this.traceId}.`, { error: error instanceof Error ? error.message : String(error) });
+                throw error; // Re-throw after logging
+            }
+        });
+    }
+    /**
+     * Asynchronously evaluate an example using the provided scorers,
+     * embedding the evaluation request into the trace data.
+     * Ported from the Python SDK's async_evaluate method.
+     *
+     * @param scorers Array of scorers to use for evaluation (currently assumes APIJudgmentScorer)
+     * @param options Evaluation options including input, outputs, and metadata
+     * @returns Promise that resolves when the evaluation entry has been added to the trace
+     */
+    asyncEvaluate(scorers_1) {
+        return __awaiter(this, arguments, void 0, function* (
+        // TODO: Allow JudgevalScorer type if rules are not used?
+        scorers, options = {}) {
+            var _a;
+            if (!this.enableEvaluations) {
+                logger_instance_1.default.warn("Evaluations are disabled. Skipping async evaluation.");
+                return;
+            }
+            if (!scorers || scorers.length === 0) {
+                logger_instance_1.default.warn("No scorers provided. Skipping async evaluation.");
+                return;
+            }
+            const startTime = Date.now() / 1000; // Record start time in seconds
+            // Create example structure matching Python/backend expectations
+            const example = {
+                input: options.input || "",
+                actual_output: options.actualOutput || "",
+                expected_output: options.expectedOutput || "",
+                context: options.context || [],
+                retrieval_context: options.retrievalContext || [],
+                tools_called: options.toolsCalled || [],
+                expected_tools: options.expectedTools || [],
+                additional_metadata: options.additionalMetadata || {},
+                trace_id: this.traceId
+            };
+            try {
+                // Get the current span ID from the context
+                const currentSpanId = currentSpanAsyncLocalStorage.getStore();
+                if (!currentSpanId) {
+                    logger_instance_1.default.warn("No active span found for async evaluation. Evaluation will not be associated with a specific step.");
+                    // Decide if we should proceed or return. For now, proceed without span association.
+                    // return;
+                }
+                // Determine function name and depth (best effort if spanId is missing)
+                let functionName = "unknown_function";
+                let entrySpanType = "evaluation";
+                let currentDepth = 0;
+                if (currentSpanId) {
+                    currentDepth = (_a = this.spanDepths[currentSpanId]) !== null && _a !== void 0 ? _a : 0;
+                    for (let i = this.entries.length - 1; i >= 0; i--) {
+                        const entry = this.entries[i];
+                        if (entry.span_id === currentSpanId && entry.type === 'enter') {
+                            functionName = entry.function || "unknown_function";
+                            // Keep span_type as 'evaluation' for the entry itself
+                            break;
+                        }
+                    }
+                }
+                // --- Create evaluation run name (similar to Python) ---
+                // Capitalize scorer names
+                const scorerNames = scorers.map(scorer => {
+                    var _a;
+                    // Attempt to get score_type, fallback to class name or Unknown
+                    const name = (scorer === null || scorer === void 0 ? void 0 : scorer.scoreType) || ((_a = scorer === null || scorer === void 0 ? void 0 : scorer.constructor) === null || _a === void 0 ? void 0 : _a.name) || "Unknown";
+                    return name.charAt(0).toUpperCase() + name.slice(1);
+                }).join(',');
+                // Use trace name and shortened span ID (or trace ID if no span)
+                const idPart = currentSpanId ? currentSpanId.substring(0, 8) : this.traceId.substring(0, 8);
+                const evalName = `${this.name.charAt(0).toUpperCase() + this.name.slice(1)}-${idPart}-[${scorerNames}]`;
+                // --- End eval name creation ---
+                // Process rules (currently just using this.rules directly)
+                const loadedRules = this.rules; // TODO: Add ScorerWrapper-like processing if needed in TS
+                // Construct the evaluation payload
+                const evalRunPayload = {
+                    organization_id: this.organizationId,
+                    log_results: options.logResults !== false, // Default to true
+                    project_name: this.projectName,
+                    eval_name: evalName,
+                    examples: [example],
+                    scorers: scorers, // Pass scorers directly
+                    model: options.model || "",
+                    metadata: {}, // Matches Python tracer
+                    judgment_api_key: this.apiKey,
+                    override: this.overwrite, // Use trace's overwrite setting
+                    rules: loadedRules // Pass the processed rules
+                };
+                // Add evaluation entry using the helper method
+                this._addEvalRun(evalRunPayload, startTime);
+            }
+            catch (error) {
+                console.error(`Failed during asyncEvaluate execution: ${error instanceof Error ? error.message : String(error)}`);
+                // Decide if we should re-throw or just log
+            }
+        });
+    }
+    /**
+     * Private helper to add an evaluation entry to the trace.
+     * This mirrors the structure of Python's add_eval_run.
+     *
+     * @param evalRunPayload The constructed payload for the evaluation.
+     * @param startTime The start time (in seconds) of the evaluation process.
+     */
+    _addEvalRun(evalRunPayload, startTime) {
+        var _a;
+        const currentSpanId = currentSpanAsyncLocalStorage.getStore();
+        if (!currentSpanId) {
+            // If no span ID, the evaluation entry won't be linked to a specific span
+            // This might happen if asyncEvaluate is called outside a runInSpan context
+            console.warn("Adding evaluation entry without an active span ID.");
+        }
+        let functionName = "unknown_function";
+        let currentDepth = 0; // Default depth if no span
+        if (currentSpanId) {
+            currentDepth = (_a = this.spanDepths[currentSpanId]) !== null && _a !== void 0 ? _a : 0;
+            // Find the function name associated with the current span_id
+            for (let i = this.entries.length - 1; i >= 0; i--) {
+                const entry = this.entries[i];
+                if (entry.span_id === currentSpanId && entry.type === 'enter') {
+                    functionName = entry.function || "unknown_function";
+                    break;
+                }
+            }
+        }
+        const duration = (Date.now() / 1000) - startTime;
+        // Add evaluation entry to the trace
+        this.addEntry({
+            type: "evaluation",
+            function: functionName,
+            span_id: currentSpanId, // May be undefined
+            depth: currentDepth,
+            timestamp: Date.now() / 1000,
+            evaluation_runs: [evalRunPayload], // Embed the payload
+            duration: duration,
+            span_type: "evaluation"
+        });
+    }
+    // OPTIONAL: Add a method to get the original name if needed elsewhere
+    getOriginalName() {
+        return this.originalName;
+    }
+}
+exports.TraceClient = TraceClient;
+/**
+ * Singleton Tracer class. Manages overall tracing configuration and trace creation.
+ */
+class Tracer {
+    constructor(config) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
+        this.initialized = false;
+        const isNode = typeof process !== 'undefined' && process.versions != null && process.versions.node != null;
+        const envApiKey = isNode ? (_a = process.env) === null || _a === void 0 ? void 0 : _a.JUDGMENT_API_KEY : undefined;
+        const envOrgId = isNode ? (_b = process.env) === null || _b === void 0 ? void 0 : _b.JUDGMENT_ORG_ID : undefined;
+        const envProjectName = isNode ? (_c = process.env) === null || _c === void 0 ? void 0 : _c.JUDGMENT_PROJECT_NAME : undefined;
+        const envMonitoring = isNode ? (_d = process.env) === null || _d === void 0 ? void 0 : _d.JUDGMENT_MONITORING : 'true';
+        const envEvaluations = isNode ? (_e = process.env) === null || _e === void 0 ? void 0 : _e.JUDGMENT_EVALUATIONS : 'true';
+        this.apiKey = (_g = (_f = config === null || config === void 0 ? void 0 : config.apiKey) !== null && _f !== void 0 ? _f : envApiKey) !== null && _g !== void 0 ? _g : '';
+        this.organizationId = (_j = (_h = config === null || config === void 0 ? void 0 : config.organizationId) !== null && _h !== void 0 ? _h : envOrgId) !== null && _j !== void 0 ? _j : '';
+        this.projectName = (_l = (_k = config === null || config === void 0 ? void 0 : config.projectName) !== null && _k !== void 0 ? _k : envProjectName) !== null && _l !== void 0 ? _l : 'default_project';
+        this.defaultRules = (_m = config === null || config === void 0 ? void 0 : config.rules) !== null && _m !== void 0 ? _m : [];
+        let effectiveMonitoring = (_o = config === null || config === void 0 ? void 0 : config.enableMonitoring) !== null && _o !== void 0 ? _o : ((envMonitoring === null || envMonitoring === void 0 ? void 0 : envMonitoring.toLowerCase()) !== 'false');
+        if (effectiveMonitoring && (!this.apiKey || !this.organizationId)) {
+            console.warn("JUDGMENT_API_KEY or JUDGMENT_ORG_ID missing. Monitoring disabled.");
+            effectiveMonitoring = false;
+        }
+        this.enableMonitoring = effectiveMonitoring;
+        this.enableEvaluations = effectiveMonitoring && ((_p = config === null || config === void 0 ? void 0 : config.enableEvaluations) !== null && _p !== void 0 ? _p : ((envEvaluations === null || envEvaluations === void 0 ? void 0 : envEvaluations.toLowerCase()) !== 'false'));
+        this.initialized = true;
+    }
+    static getInstance(config) {
+        if (!Tracer.instance) {
+            Tracer.instance = new Tracer(config);
+        }
+        else if (config && !Tracer.instance.initialized) {
+            console.warn("Tracer getInstance called with config after implicit initialization. Re-initializing.");
+            Tracer.instance = new Tracer(config);
+        }
+        else if (config && Tracer.instance.initialized) {
+            if (config.projectName && config.projectName !== Tracer.instance.projectName) {
+                console.warn(`Attempting to re-initialize Tracer with different project_name. Original will be used: '${Tracer.instance.projectName}'.`);
+            }
+            if (config.rules && config.rules.length > 0 && Tracer.instance.defaultRules.length === 0) {
+                try {
+                    console.warn("Setting default rules on Tracer instance after initial creation.");
+                    Tracer.instance.defaultRules = config.rules;
+                }
+                catch (e) {
+                    console.error("Failed to set default rules after tracer initialization:", e);
+                }
+            }
+            else if (config.rules && JSON.stringify(config.rules) !== JSON.stringify(Tracer.instance.defaultRules)) {
+                console.warn("Attempting to change default rules on Tracer after initialization. Original rules will be used.");
+            }
+        }
+        return Tracer.instance;
+    }
+    getCurrentTrace() {
+        return currentTraceAsyncLocalStorage.getStore();
+    }
+    _startTraceInternal(config) {
+        var _a, _b;
+        const parentTrace = this.getCurrentTrace();
+        const traceSpecificRules = (_a = config.rules) !== null && _a !== void 0 ? _a : [];
+        const effectiveRulesMap = new Map();
+        this.defaultRules.forEach(rule => { var _a; return effectiveRulesMap.set((_a = rule.rule_id) !== null && _a !== void 0 ? _a : rule.name, rule); });
+        traceSpecificRules.forEach(rule => { var _a; return effectiveRulesMap.set((_a = rule.rule_id) !== null && _a !== void 0 ? _a : rule.name, rule); });
+        const effectiveRules = Array.from(effectiveRulesMap.values());
+        const traceClient = new TraceClient({
+            tracer: this,
+            name: config.name,
+            projectName: (_b = config.projectName) !== null && _b !== void 0 ? _b : this.projectName,
+            overwrite: config.overwrite,
+            rules: effectiveRules,
+            enableMonitoring: this.enableMonitoring,
+            enableEvaluations: this.enableEvaluations,
+            parentTraceId: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.traceId,
+            parentName: parentTrace === null || parentTrace === void 0 ? void 0 : parentTrace.name,
+            apiKey: this.apiKey,
+            organizationId: this.organizationId,
+        });
+        if (traceClient.enableMonitoring) {
+            traceClient.save(true).catch(err => {
+                console.error(`>>> Tracer: Error saving empty trace for ${traceClient.traceId}:`, err);
+            });
+        }
+        return traceClient;
+    }
+    runInTrace(config, func) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const traceClient = this._startTraceInternal(config);
+            const shouldCreateRootSpan = (_a = config.createRootSpan) !== null && _a !== void 0 ? _a : true;
+            return yield currentTraceAsyncLocalStorage.run(traceClient, () => __awaiter(this, void 0, void 0, function* () {
+                let result;
+                try {
+                    if (shouldCreateRootSpan) {
+                        result = yield traceClient.runInSpan(config.name, { spanType: 'chain' }, () => __awaiter(this, void 0, void 0, function* () {
+                            const funcResult = func(traceClient);
+                            return funcResult instanceof Promise ? yield funcResult : funcResult;
+                        }));
+                    }
+                    else {
+                        const funcResult = func(traceClient);
+                        result = funcResult instanceof Promise ? yield funcResult : funcResult;
+                    }
+                    if (traceClient.enableMonitoring) {
+                        yield traceClient.save(false).catch(saveErr => {
+                            console.error(`Failed to save completed trace '${config.name}' (${traceClient.traceId}):`, saveErr);
+                        });
+                    }
+                    return result;
+                }
+                catch (error) {
+                    console.error(`Error during traced execution of '${config.name}' (${traceClient.traceId}):`, error);
+                    if (traceClient.enableMonitoring) {
+                        yield traceClient.save(false).catch(saveErr => {
+                            console.error(`Failed to save trace '${config.name}' after error:`, saveErr);
+                        });
+                    }
+                    throw error;
+                }
+            }));
+        });
+    }
+    observe(options) {
+        if (!this.enableMonitoring) {
+            return (func) => func;
+        }
+        return (func) => {
+            const spanName = (options === null || options === void 0 ? void 0 : options.name) || func.name || 'anonymous_function';
+            const spanType = (options === null || options === void 0 ? void 0 : options.spanType) || 'span';
+            const wrapper = (...args) => {
+                const currentTrace = this.getCurrentTrace();
+                if (!currentTrace) {
+                    return this.runInTrace({
+                        name: spanName,
+                        createRootSpan: false
+                    }, (traceClient) => __awaiter(this, void 0, void 0, function* () {
+                        const executionLogic = () => {
+                            const result = func(...args);
+                            return result instanceof Promise ? result : Promise.resolve(result);
+                        };
+                        return traceClient.runInSpan(spanName, { spanType }, () => __awaiter(this, void 0, void 0, function* () {
+                            const serializableArgs = args.map(arg => { try {
+                                return JSON.parse(JSON.stringify(arg));
+                            }
+                            catch (_a) {
+                                return String(arg);
+                            } });
+                            traceClient.recordInput({ args: serializableArgs });
+                            try {
+                                const finalResult = yield executionLogic();
+                                traceClient.recordOutput(finalResult);
+                                return finalResult;
+                            }
+                            catch (error) {
+                                console.error(`Error captured by observe decorator (root) in span '${spanName}':`, error);
+                                throw error;
+                            }
+                        }));
+                    }));
+                }
+                else {
+                    const executionLogic = () => {
+                        const result = func(...args);
+                        return result instanceof Promise ? result : Promise.resolve(result);
+                    };
+                    return currentTrace.runInSpan(spanName, { spanType }, () => __awaiter(this, void 0, void 0, function* () {
+                        const serializableArgs = args.map(arg => { try {
+                            return JSON.parse(JSON.stringify(arg));
+                        }
+                        catch (_a) {
+                            return String(arg);
+                        } });
+                        currentTrace.recordInput({ args: serializableArgs });
+                        try {
+                            const finalResult = yield executionLogic();
+                            currentTrace.recordOutput(finalResult);
+                            return finalResult;
+                        }
+                        catch (error) {
+                            console.error(`Error captured by observe decorator (nested) in span '${spanName}':`, error);
+                            throw error;
+                        }
+                    }));
+                }
+            };
+            Object.defineProperty(wrapper, 'name', { value: func.name, configurable: true });
+            return wrapper;
+        };
+    }
+}
+exports.Tracer = Tracer;
+// --- Helper Functions for Wrapping LLM Clients --- //
+function _getClientConfig(client) {
+    var _a, _b, _c, _d, _e;
+    if (client instanceof openai_1.default && typeof ((_b = (_a = client === null || client === void 0 ? void 0 : client.chat) === null || _a === void 0 ? void 0 : _a.completions) === null || _b === void 0 ? void 0 : _b.create) === 'function') {
+        return { spanName: "OPENAI_API_CALL", originalMethod: client.chat.completions.create.bind(client.chat.completions) };
+    }
+    else if (client instanceof sdk_1.default && typeof ((_c = client === null || client === void 0 ? void 0 : client.messages) === null || _c === void 0 ? void 0 : _c.create) === 'function') {
+        return { spanName: "ANTHROPIC_API_CALL", originalMethod: client.messages.create.bind(client.messages) };
+    }
+    // Use type assertion for older Together AI version
+    else if (client instanceof together_ai_1.default && typeof ((_d = client === null || client === void 0 ? void 0 : client.completions) === null || _d === void 0 ? void 0 : _d.create) === 'function') {
+        return { spanName: "TOGETHER_API_CALL", originalMethod: client.completions.create.bind(client.completions) };
+    }
+    logger_instance_1.default.warn("Cannot wrap client: Unsupported type or incompatible SDK structure.", { clientType: (_e = client === null || client === void 0 ? void 0 : client.constructor) === null || _e === void 0 ? void 0 : _e.name });
+    return null;
+}
+function _formatInputData(client, args) {
+    const params = args[0] || {};
+    try {
+        // Handle Together client potentially having different input structure
+        if (client instanceof openai_1.default || client instanceof together_ai_1.default) {
+            return { model: params.model, messages: params.messages, /* other potential params */ };
+        }
+        else if (client instanceof sdk_1.default) {
+            return { model: params.model, messages: params.messages, max_tokens: params.max_tokens, };
+        }
+    }
+    catch (e) {
+        logger_instance_1.default.error("Error formatting LLM input:", { error: e instanceof Error ? e.message : String(e), params });
+        return { raw_params: params };
+    }
+    return { raw_params: params };
+}
+function _formatOutputData(client, response) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+    try {
+        // Separate handling for OpenAI and Together, assuming Together might differ
+        if (client instanceof openai_1.default && ((_b = (_a = response === null || response === void 0 ? void 0 : response.choices) === null || _a === void 0 ? void 0 : _a[0]) === null || _b === void 0 ? void 0 : _b.message)) {
+            const message = response.choices[0].message;
+            return { content: message.content, usage: response.usage, };
+        }
+        else if (client instanceof together_ai_1.default && ((_d = (_c = response === null || response === void 0 ? void 0 : response.choices) === null || _c === void 0 ? void 0 : _c[0]) === null || _d === void 0 ? void 0 : _d.message)) {
+            // Assume Together v0.5.1 has a similar structure for now, but access defensively
+            const message = response.choices[0].message;
+            return { content: message === null || message === void 0 ? void 0 : message.content, usage: response === null || response === void 0 ? void 0 : response.usage, }; // Optional chaining for safety
+        }
+        else if (client instanceof sdk_1.default && ((_e = response === null || response === void 0 ? void 0 : response.content) === null || _e === void 0 ? void 0 : _e[0])) {
+            const textContent = response.content.filter((block) => block.type === 'text').map((block) => block.text).join('');
+            // Reconstruct usage for Anthropic if needed
+            const usage = {
+                input_tokens: (_f = response.usage) === null || _f === void 0 ? void 0 : _f.input_tokens,
+                output_tokens: (_g = response.usage) === null || _g === void 0 ? void 0 : _g.output_tokens,
+                total_tokens: (((_h = response.usage) === null || _h === void 0 ? void 0 : _h.input_tokens) || 0) + (((_j = response.usage) === null || _j === void 0 ? void 0 : _j.output_tokens) || 0)
+            };
+            return { content: textContent, usage: usage, };
+        }
+    }
+    catch (e) {
+        logger_instance_1.default.error("Error formatting LLM output:", { error: e instanceof Error ? e.message : String(e) });
+        return { formatting_error: String(e), raw_response: response };
+    }
+    // Return raw if structure doesn't match known patterns
+    return { raw_response: response };
+}
+// --- The Wrap Function --- //
+function wrap(client) {
+    var _a, _b, _c;
+    const tracer = Tracer.getInstance();
+    if (!tracer.enableMonitoring) {
+        logger_instance_1.default.info("Global monitoring disabled, client wrapping skipped.");
+        return client;
+    }
+    const config = _getClientConfig(client);
+    if (!config) {
+        return client;
+    }
+    const { spanName, originalMethod } = config;
+    const tracedMethod = (...args) => __awaiter(this, void 0, void 0, function* () {
+        const currentTrace = tracer.getCurrentTrace();
+        if (!currentTrace || !currentTrace.enableMonitoring) {
+            return originalMethod(...args);
+        }
+        return yield currentTrace.runInSpan(spanName, { spanType: 'llm' }, () => __awaiter(this, void 0, void 0, function* () {
+            const inputData = _formatInputData(client, args);
+            currentTrace.recordInput(inputData);
+            try {
+                const response = yield originalMethod(...args);
+                const outputData = _formatOutputData(client, response);
+                currentTrace.recordOutput(outputData);
+                return response;
+            }
+            catch (error) {
+                currentTrace.recordOutput(error); // Record error object in output
+                throw error;
+            }
+        }));
+    });
+    // Apply the wrapper
+    if (client instanceof openai_1.default && ((_a = client.chat) === null || _a === void 0 ? void 0 : _a.completions)) {
+        client.chat.completions.create = tracedMethod;
+    }
+    else if (client instanceof sdk_1.default && client.messages) {
+        client.messages.create = tracedMethod;
+    }
+    // Use type assertion for older Together AI version
+    else if (client instanceof together_ai_1.default && client.completions) {
+        client.completions.create = tracedMethod;
+    }
+    else {
+        // Log if we couldn't apply the wrapper despite getting a config
+        logger_instance_1.default.error("Failed to apply wrapper: Could not find method to replace after config check.", { clientType: (_b = client === null || client === void 0 ? void 0 : client.constructor) === null || _b === void 0 ? void 0 : _b.name });
+        return client;
+    }
+    logger_instance_1.default.info(`Successfully wrapped client: ${(_c = client === null || client === void 0 ? void 0 : client.constructor) === null || _c === void 0 ? void 0 : _c.name}`);
+    return client;
+}
+exports.wrap = wrap;
+//# sourceMappingURL=tracer.js.map