@anyway-sh/node-server-sdk 0.22.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README +35 -0
- package/dist/index.d.ts +1957 -0
- package/dist/index.js +4458 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +4382 -0
- package/dist/src/index.d.ts +10 -0
- package/dist/src/lib/associations/associations.d.ts +32 -0
- package/dist/src/lib/client/annotation/base-annotation.d.ts +20 -0
- package/dist/src/lib/client/annotation/user-feedback.d.ts +32 -0
- package/dist/src/lib/client/dataset/attachment-uploader.d.ts +50 -0
- package/dist/src/lib/client/dataset/attachment.d.ts +84 -0
- package/dist/src/lib/client/dataset/base-dataset.d.ts +10 -0
- package/dist/src/lib/client/dataset/column.d.ts +23 -0
- package/dist/src/lib/client/dataset/dataset.d.ts +43 -0
- package/dist/src/lib/client/dataset/datasets.d.ts +14 -0
- package/dist/src/lib/client/dataset/index.d.ts +8 -0
- package/dist/src/lib/client/dataset/row.d.ts +73 -0
- package/dist/src/lib/client/evaluator/evaluator.d.ts +28 -0
- package/dist/src/lib/client/evaluator/index.d.ts +2 -0
- package/dist/src/lib/client/experiment/experiment.d.ts +76 -0
- package/dist/src/lib/client/experiment/index.d.ts +2 -0
- package/dist/src/lib/client/traceloop-client.d.ts +40 -0
- package/dist/src/lib/configuration/index.d.ts +35 -0
- package/dist/src/lib/configuration/validation.d.ts +3 -0
- package/dist/src/lib/errors/index.d.ts +36 -0
- package/dist/src/lib/generated/evaluators/index.d.ts +5 -0
- package/dist/src/lib/generated/evaluators/mbt-evaluators.d.ts +386 -0
- package/dist/src/lib/generated/evaluators/registry.d.ts +12 -0
- package/dist/src/lib/generated/evaluators/types.d.ts +401 -0
- package/dist/src/lib/images/image-uploader.d.ts +15 -0
- package/dist/src/lib/images/index.d.ts +2 -0
- package/dist/src/lib/interfaces/annotations.interface.d.ts +35 -0
- package/dist/src/lib/interfaces/dataset.interface.d.ts +105 -0
- package/dist/src/lib/interfaces/evaluator.interface.d.ts +83 -0
- package/dist/src/lib/interfaces/experiment.interface.d.ts +117 -0
- package/dist/src/lib/interfaces/index.d.ts +8 -0
- package/dist/src/lib/interfaces/initialize-options.interface.d.ts +133 -0
- package/dist/src/lib/interfaces/prompts.interface.d.ts +53 -0
- package/dist/src/lib/interfaces/traceloop-client.interface.d.ts +7 -0
- package/dist/src/lib/node-server-sdk.d.ts +19 -0
- package/dist/src/lib/prompts/fetch.d.ts +3 -0
- package/dist/src/lib/prompts/index.d.ts +3 -0
- package/dist/src/lib/prompts/registry.d.ts +9 -0
- package/dist/src/lib/prompts/template.d.ts +3 -0
- package/dist/src/lib/tracing/ai-sdk-transformations.d.ts +5 -0
- package/dist/src/lib/tracing/association.d.ts +4 -0
- package/dist/src/lib/tracing/baggage-utils.d.ts +2 -0
- package/dist/src/lib/tracing/custom-metric.d.ts +14 -0
- package/dist/src/lib/tracing/decorators.d.ts +22 -0
- package/dist/src/lib/tracing/index.d.ts +14 -0
- package/dist/src/lib/tracing/manual.d.ts +60 -0
- package/dist/src/lib/tracing/sampler.d.ts +7 -0
- package/dist/src/lib/tracing/span-processor.d.ts +48 -0
- package/dist/src/lib/tracing/tracing.d.ts +10 -0
- package/dist/src/lib/utils/response-transformer.d.ts +19 -0
- package/package.json +127 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,4382 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import * as mime from 'mime-types';
|
|
4
|
+
import * as Papa from 'papaparse';
|
|
5
|
+
import { NodeSDK } from '@opentelemetry/sdk-node';
|
|
6
|
+
import { createContextKey, trace, context, diag, DiagConsoleLogger, DiagLogLevel } from '@opentelemetry/api';
|
|
7
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-proto';
|
|
8
|
+
import { TraceExporter } from '@google-cloud/opentelemetry-cloud-trace-exporter';
|
|
9
|
+
import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions';
|
|
10
|
+
import { SpanAttributes, TraceloopSpanKindValues, CONTEXT_KEY_ALLOW_TRACE_CONTENT, Events, EventAttributes } from '@traceloop/ai-semantic-conventions';
|
|
11
|
+
import { AnthropicInstrumentation } from '@traceloop/instrumentation-anthropic';
|
|
12
|
+
import { OpenAIInstrumentation } from '@traceloop/instrumentation-openai';
|
|
13
|
+
import { LlamaIndexInstrumentation } from '@traceloop/instrumentation-llamaindex';
|
|
14
|
+
import { VertexAIInstrumentation, AIPlatformInstrumentation } from '@traceloop/instrumentation-vertexai';
|
|
15
|
+
import { BedrockInstrumentation } from '@traceloop/instrumentation-bedrock';
|
|
16
|
+
import { CohereInstrumentation } from '@traceloop/instrumentation-cohere';
|
|
17
|
+
import { PineconeInstrumentation } from '@traceloop/instrumentation-pinecone';
|
|
18
|
+
import { LangChainInstrumentation } from '@traceloop/instrumentation-langchain';
|
|
19
|
+
import { ChromaDBInstrumentation } from '@traceloop/instrumentation-chromadb';
|
|
20
|
+
import { QdrantInstrumentation } from '@traceloop/instrumentation-qdrant';
|
|
21
|
+
import { TogetherInstrumentation } from '@traceloop/instrumentation-together';
|
|
22
|
+
import { McpInstrumentation } from '@traceloop/instrumentation-mcp';
|
|
23
|
+
import { SimpleSpanProcessor, BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
|
|
24
|
+
import { ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_REQUEST_MODEL, ATTR_GEN_AI_COMPLETION, ATTR_GEN_AI_OUTPUT_MESSAGES, ATTR_GEN_AI_PROMPT, ATTR_GEN_AI_INPUT_MESSAGES, ATTR_GEN_AI_USAGE_INPUT_TOKENS, ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, ATTR_GEN_AI_RESPONSE_FINISH_REASONS, ATTR_GEN_AI_RESPONSE_MODEL, ATTR_GEN_AI_RESPONSE_ID, ATTR_GEN_AI_SYSTEM, ATTR_GEN_AI_PROVIDER_NAME, ATTR_GEN_AI_CONVERSATION_ID, ATTR_GEN_AI_TOOL_NAME, ATTR_GEN_AI_TOOL_CALL_ID, ATTR_GEN_AI_TOOL_CALL_ARGUMENTS, ATTR_GEN_AI_TOOL_CALL_RESULT, ATTR_GEN_AI_AGENT_NAME } from '@opentelemetry/semantic-conventions/incubating';
|
|
25
|
+
import fetch$1 from 'cross-fetch';
|
|
26
|
+
import fetchBuilder from 'fetch-retry';
|
|
27
|
+
import { suppressTracing } from '@opentelemetry/core';
|
|
28
|
+
import { Environment } from 'nunjucks';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The severity of an error.
|
|
32
|
+
*/
|
|
33
|
+
const SEVERITY = {
|
|
34
|
+
Warning: "Warning",
|
|
35
|
+
Error: "Error",
|
|
36
|
+
Critical: "Critical",
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Base class for all Traceloop errors.
|
|
40
|
+
*/
|
|
41
|
+
class TraceloopError extends Error {
|
|
42
|
+
constructor(message, severity = SEVERITY.Error) {
|
|
43
|
+
super(message);
|
|
44
|
+
this.severity = severity;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
class NotInitializedError extends TraceloopError {
|
|
48
|
+
constructor() {
|
|
49
|
+
super(`The Traceloop SDK must be initialized by calling the "initialize" function prior to use.`, SEVERITY.Critical);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
class InitializationError extends TraceloopError {
|
|
53
|
+
constructor(message, cause) {
|
|
54
|
+
super(message !== null && message !== void 0 ? message : "Failed to initialize Traceloop SDK", SEVERITY.Critical);
|
|
55
|
+
this.underlyingCause = cause;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
class ArgumentNotProvidedError extends TraceloopError {
|
|
59
|
+
constructor(argumentName) {
|
|
60
|
+
super(`The "${argumentName}" argument is required and must be a string.`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
class PromptNotFoundError extends TraceloopError {
|
|
64
|
+
constructor(key) {
|
|
65
|
+
super(`The prompt "${key}" was not found in the registry.`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
var version = "0.22.8";
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Base class for handling annotation operations with the Traceloop API.
|
|
73
|
+
* @internal
|
|
74
|
+
*/
|
|
75
|
+
class BaseAnnotation {
|
|
76
|
+
constructor(client, flow) {
|
|
77
|
+
this.client = client;
|
|
78
|
+
this.flow = flow;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Creates a new annotation.
|
|
82
|
+
*
|
|
83
|
+
* @param options - The annotation creation options
|
|
84
|
+
* @returns Promise resolving to the fetch Response
|
|
85
|
+
*/
|
|
86
|
+
async create(options) {
|
|
87
|
+
return await this.client.post(`/v2/annotation-tasks/${options.annotationTask}/annotations`, {
|
|
88
|
+
entity_instance_id: options.entity.id,
|
|
89
|
+
tags: options.tags,
|
|
90
|
+
source: "sdk",
|
|
91
|
+
flow: this.flow,
|
|
92
|
+
actor: {
|
|
93
|
+
type: "service",
|
|
94
|
+
id: this.client.appName,
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Handles user feedback annotations with the Traceloop API.
|
|
102
|
+
*/
|
|
103
|
+
class UserFeedback extends BaseAnnotation {
|
|
104
|
+
constructor(client) {
|
|
105
|
+
super(client, "user_feedback");
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Creates a new annotation for a specific task and entity.
|
|
109
|
+
*
|
|
110
|
+
* @param options - The options for creating an annotation
|
|
111
|
+
* @returns Promise resolving to the fetch Response
|
|
112
|
+
*
|
|
113
|
+
* @example
|
|
114
|
+
* ```typescript
|
|
115
|
+
* await client.annotation.create({
|
|
116
|
+
* annotationTask: 'sample-annotation-task',
|
|
117
|
+
* entity: {
|
|
118
|
+
* id: '123456',
|
|
119
|
+
* },
|
|
120
|
+
* tags: {
|
|
121
|
+
* sentiment: 'positive',
|
|
122
|
+
* score: 0.85,
|
|
123
|
+
* tones: ['happy', 'surprised']
|
|
124
|
+
* }
|
|
125
|
+
* });
|
|
126
|
+
* ```
|
|
127
|
+
*/
|
|
128
|
+
async create(options) {
|
|
129
|
+
return await super.create(options);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Utility functions for transforming API responses from snake_case to camelCase
|
|
135
|
+
*/
|
|
136
|
+
/**
|
|
137
|
+
* Converts a snake_case string to camelCase
|
|
138
|
+
* @param str The snake_case string to convert
|
|
139
|
+
* @returns The camelCase version of the string
|
|
140
|
+
*/
|
|
141
|
+
function snakeToCamel(str) {
|
|
142
|
+
return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Recursively transforms all snake_case keys in an object to camelCase
|
|
146
|
+
* @param obj The object to transform
|
|
147
|
+
* @returns A new object with camelCase keys
|
|
148
|
+
*/
|
|
149
|
+
function transformResponseKeys(obj) {
|
|
150
|
+
if (obj === null || obj === undefined) {
|
|
151
|
+
return obj;
|
|
152
|
+
}
|
|
153
|
+
if (Array.isArray(obj)) {
|
|
154
|
+
return obj.map(transformResponseKeys);
|
|
155
|
+
}
|
|
156
|
+
if (typeof obj === "object" && obj.constructor === Object) {
|
|
157
|
+
const transformed = {};
|
|
158
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
159
|
+
const camelKey = snakeToCamel(key);
|
|
160
|
+
transformed[camelKey] = transformResponseKeys(value);
|
|
161
|
+
}
|
|
162
|
+
return transformed;
|
|
163
|
+
}
|
|
164
|
+
return obj;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Transforms API response data by converting snake_case keys to camelCase
|
|
168
|
+
* This function is designed to be used in the BaseDataset.handleResponse() method
|
|
169
|
+
* to ensure consistent camelCase format throughout the SDK
|
|
170
|
+
*
|
|
171
|
+
* @param data The raw API response data
|
|
172
|
+
* @returns The transformed data with camelCase keys
|
|
173
|
+
*/
|
|
174
|
+
function transformApiResponse(data) {
|
|
175
|
+
return transformResponseKeys(data);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
class BaseDatasetEntity {
|
|
179
|
+
constructor(client) {
|
|
180
|
+
this.client = client;
|
|
181
|
+
}
|
|
182
|
+
async handleResponse(response) {
|
|
183
|
+
if (!response.ok) {
|
|
184
|
+
let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
|
|
185
|
+
try {
|
|
186
|
+
const errorData = await response.json();
|
|
187
|
+
if (errorData.error) {
|
|
188
|
+
errorMessage = errorData.error;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
catch (_a) {
|
|
192
|
+
// Use default HTTP error message if JSON parsing fails
|
|
193
|
+
}
|
|
194
|
+
throw new Error(errorMessage);
|
|
195
|
+
}
|
|
196
|
+
const contentType = response.headers.get("content-type");
|
|
197
|
+
if (contentType && contentType.includes("application/json")) {
|
|
198
|
+
const rawData = await response.json();
|
|
199
|
+
return transformApiResponse(rawData);
|
|
200
|
+
}
|
|
201
|
+
// Handle non-JSON responses (text/csv, etc.)
|
|
202
|
+
const textContent = await response.text();
|
|
203
|
+
return {
|
|
204
|
+
contentType: contentType || "text/plain",
|
|
205
|
+
body: textContent,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
validateDatasetId(id) {
|
|
209
|
+
if (!id || typeof id !== "string" || id.trim().length === 0) {
|
|
210
|
+
throw new Error("Dataset ID is required and must be a non-empty string");
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
validateDatasetSlug(slug) {
|
|
214
|
+
if (!slug || typeof slug !== "string" || slug.trim().length === 0) {
|
|
215
|
+
throw new Error("Dataset slug is required and must be a non-empty string");
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
validateDatasetName(name) {
|
|
219
|
+
if (!name || typeof name !== "string" || name.trim().length === 0) {
|
|
220
|
+
throw new Error("Dataset name is required and must be a non-empty string");
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function detectFileType(contentType) {
|
|
226
|
+
if (contentType.startsWith("image/"))
|
|
227
|
+
return "image";
|
|
228
|
+
if (contentType.startsWith("video/"))
|
|
229
|
+
return "video";
|
|
230
|
+
if (contentType.startsWith("audio/"))
|
|
231
|
+
return "audio";
|
|
232
|
+
return "file";
|
|
233
|
+
}
|
|
234
|
+
function getMimeType(filename) {
|
|
235
|
+
return mime.lookup(filename) || "application/octet-stream";
|
|
236
|
+
}
|
|
237
|
+
class Attachment {
|
|
238
|
+
constructor(options) {
|
|
239
|
+
this.type = "attachment";
|
|
240
|
+
const { filePath, data, filename, contentType, fileType, metadata } = options;
|
|
241
|
+
if (!filePath && !data) {
|
|
242
|
+
throw new Error("Either filePath or data must be provided");
|
|
243
|
+
}
|
|
244
|
+
if (filePath && data) {
|
|
245
|
+
throw new Error("Cannot provide both filePath and data");
|
|
246
|
+
}
|
|
247
|
+
if (data && !filename) {
|
|
248
|
+
throw new Error("filename is required when using data");
|
|
249
|
+
}
|
|
250
|
+
this._filePath = filePath;
|
|
251
|
+
this._data = data;
|
|
252
|
+
this._filename = filename;
|
|
253
|
+
this._contentType = contentType;
|
|
254
|
+
this._fileType = fileType;
|
|
255
|
+
this._metadata = metadata;
|
|
256
|
+
}
|
|
257
|
+
async getData() {
|
|
258
|
+
if (this._data) {
|
|
259
|
+
return Buffer.isBuffer(this._data) ? this._data : Buffer.from(this._data);
|
|
260
|
+
}
|
|
261
|
+
if (this._filePath) {
|
|
262
|
+
return fs.promises.readFile(this._filePath);
|
|
263
|
+
}
|
|
264
|
+
throw new Error("No data source available");
|
|
265
|
+
}
|
|
266
|
+
getFileName() {
|
|
267
|
+
if (this._filename) {
|
|
268
|
+
return this._filename;
|
|
269
|
+
}
|
|
270
|
+
if (this._filePath) {
|
|
271
|
+
return path.basename(this._filePath);
|
|
272
|
+
}
|
|
273
|
+
throw new Error("No filename available");
|
|
274
|
+
}
|
|
275
|
+
getContentType() {
|
|
276
|
+
if (this._contentType) {
|
|
277
|
+
return this._contentType;
|
|
278
|
+
}
|
|
279
|
+
return getMimeType(this.getFileName());
|
|
280
|
+
}
|
|
281
|
+
get fileType() {
|
|
282
|
+
if (this._fileType) {
|
|
283
|
+
return this._fileType;
|
|
284
|
+
}
|
|
285
|
+
return detectFileType(this.getContentType());
|
|
286
|
+
}
|
|
287
|
+
get metadata() {
|
|
288
|
+
return this._metadata;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
class ExternalAttachment {
|
|
292
|
+
constructor(options) {
|
|
293
|
+
this.type = "external";
|
|
294
|
+
const { url, filename, contentType, fileType, metadata } = options;
|
|
295
|
+
if (!url || typeof url !== "string") {
|
|
296
|
+
throw new Error("URL is required and must be a string");
|
|
297
|
+
}
|
|
298
|
+
try {
|
|
299
|
+
new URL(url);
|
|
300
|
+
}
|
|
301
|
+
catch (_a) {
|
|
302
|
+
throw new Error("Invalid URL provided");
|
|
303
|
+
}
|
|
304
|
+
this._url = url;
|
|
305
|
+
this._filename = filename;
|
|
306
|
+
this._contentType = contentType;
|
|
307
|
+
this._fileType = fileType || "file";
|
|
308
|
+
this._metadata = metadata;
|
|
309
|
+
}
|
|
310
|
+
get url() {
|
|
311
|
+
return this._url;
|
|
312
|
+
}
|
|
313
|
+
get filename() {
|
|
314
|
+
return this._filename;
|
|
315
|
+
}
|
|
316
|
+
get contentType() {
|
|
317
|
+
return this._contentType;
|
|
318
|
+
}
|
|
319
|
+
get fileType() {
|
|
320
|
+
return this._fileType;
|
|
321
|
+
}
|
|
322
|
+
get metadata() {
|
|
323
|
+
return this._metadata;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
class AttachmentReference {
|
|
327
|
+
constructor(storageType, storageKey, url, fileType, metadata) {
|
|
328
|
+
this.storageType = storageType;
|
|
329
|
+
this.storageKey = storageKey;
|
|
330
|
+
this.url = url;
|
|
331
|
+
this.fileType = fileType;
|
|
332
|
+
this.metadata = metadata;
|
|
333
|
+
}
|
|
334
|
+
async download(filePath) {
|
|
335
|
+
if (!this.url) {
|
|
336
|
+
throw new Error("Cannot download attachment: no URL available");
|
|
337
|
+
}
|
|
338
|
+
const response = await fetch(this.url);
|
|
339
|
+
if (!response.ok) {
|
|
340
|
+
throw new Error(`Failed to download attachment: ${response.status} ${response.statusText}`);
|
|
341
|
+
}
|
|
342
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
343
|
+
const buffer = Buffer.from(arrayBuffer);
|
|
344
|
+
if (filePath) {
|
|
345
|
+
await fs.promises.writeFile(filePath, buffer);
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
return buffer;
|
|
349
|
+
}
|
|
350
|
+
getUrl() {
|
|
351
|
+
return this.url;
|
|
352
|
+
}
|
|
353
|
+
toJSON() {
|
|
354
|
+
return {
|
|
355
|
+
storageType: this.storageType,
|
|
356
|
+
storageKey: this.storageKey,
|
|
357
|
+
url: this.url,
|
|
358
|
+
fileType: this.fileType,
|
|
359
|
+
metadata: this.metadata,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
function isAttachment(value) {
|
|
364
|
+
return value instanceof Attachment || (value === null || value === void 0 ? void 0 : value.type) === "attachment";
|
|
365
|
+
}
|
|
366
|
+
function isExternalAttachment(value) {
|
|
367
|
+
return (value instanceof ExternalAttachment || (value === null || value === void 0 ? void 0 : value.type) === "external");
|
|
368
|
+
}
|
|
369
|
+
function isAttachmentReference(value) {
|
|
370
|
+
return (value instanceof AttachmentReference ||
|
|
371
|
+
((value === null || value === void 0 ? void 0 : value.storageType) !== undefined &&
|
|
372
|
+
(value === null || value === void 0 ? void 0 : value.storageKey) !== undefined));
|
|
373
|
+
}
|
|
374
|
+
function isAnyAttachment(value) {
|
|
375
|
+
return isAttachment(value) || isExternalAttachment(value);
|
|
376
|
+
}
|
|
377
|
+
const attachment = {
|
|
378
|
+
file: (filePath, options) => {
|
|
379
|
+
return new Attachment(Object.assign({ filePath }, options));
|
|
380
|
+
},
|
|
381
|
+
buffer: (data, filename, options) => {
|
|
382
|
+
return new Attachment(Object.assign({ data,
|
|
383
|
+
filename }, options));
|
|
384
|
+
},
|
|
385
|
+
url: (url, options) => {
|
|
386
|
+
return new ExternalAttachment(Object.assign({ url }, options));
|
|
387
|
+
},
|
|
388
|
+
};
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Handles attachment upload and registration operations
|
|
392
|
+
*/
|
|
393
|
+
class AttachmentUploader {
|
|
394
|
+
constructor(client) {
|
|
395
|
+
this.client = client;
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Requests a presigned upload URL from the API
|
|
399
|
+
*/
|
|
400
|
+
async getUploadUrl(datasetSlug, rowId, columnSlug, fileName, contentType, fileType, metadata) {
|
|
401
|
+
const response = await this.client.post(`/v2/datasets/${datasetSlug}/rows/${rowId}/cells/${columnSlug}/upload-url`, {
|
|
402
|
+
type: fileType,
|
|
403
|
+
file_name: fileName,
|
|
404
|
+
content_type: contentType,
|
|
405
|
+
metadata: metadata,
|
|
406
|
+
});
|
|
407
|
+
if (!response.ok) {
|
|
408
|
+
let errorMessage = `Failed to get upload URL: ${response.status} ${response.statusText}`;
|
|
409
|
+
try {
|
|
410
|
+
const errorData = await response.json();
|
|
411
|
+
if (errorData.error) {
|
|
412
|
+
errorMessage = errorData.error;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
catch (_a) {
|
|
416
|
+
// Use default error message
|
|
417
|
+
}
|
|
418
|
+
throw new Error(errorMessage);
|
|
419
|
+
}
|
|
420
|
+
const data = await response.json();
|
|
421
|
+
return {
|
|
422
|
+
uploadUrl: data.upload_url,
|
|
423
|
+
storageKey: data.storage_key,
|
|
424
|
+
expiresAt: data.expires_at,
|
|
425
|
+
method: data.method || "PUT",
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Uploads file data directly to S3 using the presigned URL
|
|
430
|
+
*/
|
|
431
|
+
async uploadToS3(uploadUrl, data, contentType) {
|
|
432
|
+
const response = await fetch(uploadUrl, {
|
|
433
|
+
method: "PUT",
|
|
434
|
+
headers: {
|
|
435
|
+
"Content-Type": contentType,
|
|
436
|
+
},
|
|
437
|
+
body: new Uint8Array(data),
|
|
438
|
+
});
|
|
439
|
+
if (!response.ok) {
|
|
440
|
+
throw new Error(`Failed to upload to S3: ${response.status} ${response.statusText}`);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Confirms the upload status with the API
|
|
445
|
+
*/
|
|
446
|
+
async confirmUpload(datasetSlug, rowId, columnSlug, status, metadata) {
|
|
447
|
+
const response = await this.client.put(`/v2/datasets/${datasetSlug}/rows/${rowId}/cells/${columnSlug}/upload-status`, {
|
|
448
|
+
status,
|
|
449
|
+
metadata,
|
|
450
|
+
});
|
|
451
|
+
if (!response.ok) {
|
|
452
|
+
let errorMessage = `Failed to confirm upload: ${response.status} ${response.statusText}`;
|
|
453
|
+
try {
|
|
454
|
+
const errorData = await response.json();
|
|
455
|
+
if (errorData.error) {
|
|
456
|
+
errorMessage = errorData.error;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
catch (_a) {
|
|
460
|
+
// Use default error message
|
|
461
|
+
}
|
|
462
|
+
throw new Error(errorMessage);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Registers an external URL as an attachment
|
|
467
|
+
*/
|
|
468
|
+
async registerExternalUrl(datasetSlug, rowId, columnSlug, url, fileType, metadata) {
|
|
469
|
+
const response = await this.client.post(`/v2/datasets/${datasetSlug}/rows/${rowId}/cells/${columnSlug}/external-url`, {
|
|
470
|
+
type: fileType,
|
|
471
|
+
url,
|
|
472
|
+
metadata,
|
|
473
|
+
});
|
|
474
|
+
if (!response.ok) {
|
|
475
|
+
let errorMessage = `Failed to register external URL: ${response.status} ${response.statusText}`;
|
|
476
|
+
try {
|
|
477
|
+
const errorData = await response.json();
|
|
478
|
+
if (errorData.error) {
|
|
479
|
+
errorMessage = errorData.error;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
catch (_a) {
|
|
483
|
+
// Use default error message
|
|
484
|
+
}
|
|
485
|
+
throw new Error(errorMessage);
|
|
486
|
+
}
|
|
487
|
+
const data = await response.json();
|
|
488
|
+
return new AttachmentReference("external", data.storage_key || "", url, fileType, metadata);
|
|
489
|
+
}
|
|
490
|
+
/**
|
|
491
|
+
* Uploads an Attachment through the full flow:
|
|
492
|
+
* 1. Get presigned URL
|
|
493
|
+
* 2. Upload to S3
|
|
494
|
+
* 3. Confirm upload
|
|
495
|
+
*/
|
|
496
|
+
async uploadAttachment(datasetSlug, rowId, columnSlug, attachment) {
|
|
497
|
+
const fileName = attachment.getFileName();
|
|
498
|
+
const contentType = attachment.getContentType();
|
|
499
|
+
const fileType = attachment.fileType;
|
|
500
|
+
const data = await attachment.getData();
|
|
501
|
+
// Step 1: Get presigned upload URL
|
|
502
|
+
const uploadInfo = await this.getUploadUrl(datasetSlug, rowId, columnSlug, fileName, contentType, fileType, attachment.metadata);
|
|
503
|
+
// Step 2: Upload to S3
|
|
504
|
+
try {
|
|
505
|
+
await this.uploadToS3(uploadInfo.uploadUrl, data, contentType);
|
|
506
|
+
}
|
|
507
|
+
catch (error) {
|
|
508
|
+
// Confirm failure and re-throw
|
|
509
|
+
try {
|
|
510
|
+
await this.confirmUpload(datasetSlug, rowId, columnSlug, "failed", attachment.metadata);
|
|
511
|
+
}
|
|
512
|
+
catch (_a) {
|
|
513
|
+
// Ignore confirmation errors
|
|
514
|
+
}
|
|
515
|
+
throw error;
|
|
516
|
+
}
|
|
517
|
+
// Step 3: Confirm success
|
|
518
|
+
await this.confirmUpload(datasetSlug, rowId, columnSlug, "success", attachment.metadata);
|
|
519
|
+
return new AttachmentReference("internal", uploadInfo.storageKey, undefined, fileType, attachment.metadata);
|
|
520
|
+
}
|
|
521
|
+
/**
|
|
522
|
+
* Processes an ExternalAttachment by registering the URL
|
|
523
|
+
*/
|
|
524
|
+
async processExternalAttachment(datasetSlug, rowId, columnSlug, attachment) {
|
|
525
|
+
return this.registerExternalUrl(datasetSlug, rowId, columnSlug, attachment.url, attachment.fileType, attachment.metadata);
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* Processes any attachment type (Attachment or ExternalAttachment)
|
|
529
|
+
*/
|
|
530
|
+
async processAnyAttachment(datasetSlug, rowId, columnSlug, attachmentObj) {
|
|
531
|
+
if (attachmentObj instanceof Attachment) {
|
|
532
|
+
return this.uploadAttachment(datasetSlug, rowId, columnSlug, attachmentObj);
|
|
533
|
+
}
|
|
534
|
+
else {
|
|
535
|
+
return this.processExternalAttachment(datasetSlug, rowId, columnSlug, attachmentObj);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
class Row extends BaseDatasetEntity {
|
|
541
|
+
constructor(client, data) {
|
|
542
|
+
super(client);
|
|
543
|
+
this._deleted = false;
|
|
544
|
+
this._data = data;
|
|
545
|
+
}
|
|
546
|
+
get id() {
|
|
547
|
+
return this._data.id;
|
|
548
|
+
}
|
|
549
|
+
get datasetId() {
|
|
550
|
+
return this._data.datasetId;
|
|
551
|
+
}
|
|
552
|
+
get datasetSlug() {
|
|
553
|
+
return this._data.datasetSlug;
|
|
554
|
+
}
|
|
555
|
+
get data() {
|
|
556
|
+
return Object.assign({}, this._data.data);
|
|
557
|
+
}
|
|
558
|
+
get createdAt() {
|
|
559
|
+
return this._data.createdAt;
|
|
560
|
+
}
|
|
561
|
+
get updatedAt() {
|
|
562
|
+
return this._data.updatedAt;
|
|
563
|
+
}
|
|
564
|
+
get deleted() {
|
|
565
|
+
return this._deleted;
|
|
566
|
+
}
|
|
567
|
+
getValue(columnName) {
|
|
568
|
+
const value = this._data.data[columnName];
|
|
569
|
+
return value !== undefined ? value : null;
|
|
570
|
+
}
|
|
571
|
+
hasColumn(columnName) {
|
|
572
|
+
return columnName in this._data.data;
|
|
573
|
+
}
|
|
574
|
+
getColumns() {
|
|
575
|
+
return Object.keys(this._data.data);
|
|
576
|
+
}
|
|
577
|
+
async update(options) {
|
|
578
|
+
if (this._deleted) {
|
|
579
|
+
throw new Error("Cannot update a deleted row");
|
|
580
|
+
}
|
|
581
|
+
if (!options.data || typeof options.data !== "object") {
|
|
582
|
+
throw new Error("Update data must be a valid object");
|
|
583
|
+
}
|
|
584
|
+
const updatedData = Object.assign(Object.assign({}, this._data.data), options.data);
|
|
585
|
+
const response = await this.client.put(`/v2/datasets/${this.datasetSlug}/rows/${this.id}`, {
|
|
586
|
+
Values: updatedData,
|
|
587
|
+
});
|
|
588
|
+
const result = await this.handleResponse(response);
|
|
589
|
+
if (result && result.id) {
|
|
590
|
+
this._data = result;
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
async partialUpdate(updates) {
|
|
594
|
+
if (this._deleted) {
|
|
595
|
+
throw new Error("Cannot update a deleted row");
|
|
596
|
+
}
|
|
597
|
+
if (!updates || typeof updates !== "object") {
|
|
598
|
+
throw new Error("Updates must be a valid object");
|
|
599
|
+
}
|
|
600
|
+
const response = await this.client.put(`/v2/datasets/${this.datasetSlug}/rows/${this.id}`, {
|
|
601
|
+
Values: updates,
|
|
602
|
+
});
|
|
603
|
+
const result = await this.handleResponse(response);
|
|
604
|
+
if (result && result.id) {
|
|
605
|
+
this._data = result;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
async delete() {
|
|
609
|
+
if (this._deleted) {
|
|
610
|
+
throw new Error("Row is already deleted");
|
|
611
|
+
}
|
|
612
|
+
const response = await this.client.delete(`/v2/datasets/${this.datasetSlug}/rows/${this.id}`);
|
|
613
|
+
await this.handleResponse(response);
|
|
614
|
+
this._deleted = true;
|
|
615
|
+
}
|
|
616
|
+
toJSON() {
|
|
617
|
+
return Object.assign({}, this._data.data);
|
|
618
|
+
}
|
|
619
|
+
toCSVRow(columns, delimiter = ",") {
|
|
620
|
+
const columnsToUse = columns || this.getColumns();
|
|
621
|
+
const values = columnsToUse.map((column) => {
|
|
622
|
+
const value = this._data.data[column];
|
|
623
|
+
if (value === null || value === undefined) {
|
|
624
|
+
return "";
|
|
625
|
+
}
|
|
626
|
+
const stringValue = String(value);
|
|
627
|
+
if (stringValue.includes(delimiter) ||
|
|
628
|
+
stringValue.includes('"') ||
|
|
629
|
+
stringValue.includes("\n")) {
|
|
630
|
+
return `"${stringValue.replace(/"/g, '""')}"`;
|
|
631
|
+
}
|
|
632
|
+
return stringValue;
|
|
633
|
+
});
|
|
634
|
+
return values.join(delimiter);
|
|
635
|
+
}
|
|
636
|
+
validate(columnValidators) {
|
|
637
|
+
const errors = [];
|
|
638
|
+
if (columnValidators) {
|
|
639
|
+
Object.keys(columnValidators).forEach((columnName) => {
|
|
640
|
+
const validator = columnValidators[columnName];
|
|
641
|
+
const value = this._data.data[columnName];
|
|
642
|
+
if (!validator(value)) {
|
|
643
|
+
errors.push(`Invalid value for column '${columnName}': ${value}`);
|
|
644
|
+
}
|
|
645
|
+
});
|
|
646
|
+
}
|
|
647
|
+
return {
|
|
648
|
+
valid: errors.length === 0,
|
|
649
|
+
errors,
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
clone() {
|
|
653
|
+
const clonedData = Object.assign(Object.assign({}, this._data), { data: Object.assign({}, this._data.data) });
|
|
654
|
+
return new Row(this.client, clonedData);
|
|
655
|
+
}
|
|
656
|
+
/**
|
|
657
|
+
* Gets an attachment reference from a column
|
|
658
|
+
* @param columnName The name of the column containing the attachment
|
|
659
|
+
* @returns AttachmentReference if the column contains an attachment, null otherwise
|
|
660
|
+
*/
|
|
661
|
+
getAttachment(columnName) {
|
|
662
|
+
const value = this._data.data[columnName];
|
|
663
|
+
if (!value || typeof value !== "object") {
|
|
664
|
+
return null;
|
|
665
|
+
}
|
|
666
|
+
// Value is now guaranteed to be an object
|
|
667
|
+
const objValue = value;
|
|
668
|
+
// Check if value is already an AttachmentReference
|
|
669
|
+
if (objValue instanceof AttachmentReference) {
|
|
670
|
+
return objValue;
|
|
671
|
+
}
|
|
672
|
+
// Check if value is a serialized attachment reference
|
|
673
|
+
if (isAttachmentReference(objValue)) {
|
|
674
|
+
const ref = objValue;
|
|
675
|
+
return new AttachmentReference(ref.storageType, ref.storageKey, ref.url, ref.fileType, ref.metadata);
|
|
676
|
+
}
|
|
677
|
+
return null;
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* Checks if a column contains an attachment
|
|
681
|
+
* @param columnName The name of the column to check
|
|
682
|
+
*/
|
|
683
|
+
hasAttachment(columnName) {
|
|
684
|
+
return this.getAttachment(columnName) !== null;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Sets/uploads an attachment to a column
|
|
688
|
+
* @param columnSlug The slug of the column to set the attachment in
|
|
689
|
+
* @param attachment The attachment to upload (Attachment or ExternalAttachment)
|
|
690
|
+
* @returns The created AttachmentReference
|
|
691
|
+
*
|
|
692
|
+
* @example
|
|
693
|
+
* // Upload from file
|
|
694
|
+
* await row.setAttachment("image", new Attachment({ filePath: "./photo.jpg" }));
|
|
695
|
+
*
|
|
696
|
+
* @example
|
|
697
|
+
* // Set external URL
|
|
698
|
+
* await row.setAttachment("document", new ExternalAttachment({ url: "https://example.com/doc.pdf" }));
|
|
699
|
+
*/
|
|
700
|
+
async setAttachment(columnSlug, attachment) {
|
|
701
|
+
if (this._deleted) {
|
|
702
|
+
throw new Error("Cannot set attachment on a deleted row");
|
|
703
|
+
}
|
|
704
|
+
const uploader = new AttachmentUploader(this.client);
|
|
705
|
+
const reference = await uploader.processAnyAttachment(this.datasetSlug, this.id, columnSlug, attachment);
|
|
706
|
+
// Update internal data
|
|
707
|
+
this._data.data[columnSlug] = reference.toJSON();
|
|
708
|
+
return reference;
|
|
709
|
+
}
|
|
710
|
+
/**
|
|
711
|
+
* Downloads an attachment from a column
|
|
712
|
+
* @param columnName The name of the column containing the attachment
|
|
713
|
+
* @param outputPath Optional file path to save the downloaded file
|
|
714
|
+
* @returns Buffer if no outputPath provided, void if saved to file
|
|
715
|
+
*
|
|
716
|
+
* @example
|
|
717
|
+
* // Get as buffer
|
|
718
|
+
* const data = await row.downloadAttachment("image");
|
|
719
|
+
*
|
|
720
|
+
* @example
|
|
721
|
+
* // Save to file
|
|
722
|
+
* await row.downloadAttachment("image", "./downloaded-image.png");
|
|
723
|
+
*/
|
|
724
|
+
async downloadAttachment(columnName, outputPath) {
|
|
725
|
+
if (this._deleted) {
|
|
726
|
+
throw new Error("Cannot download attachment from a deleted row");
|
|
727
|
+
}
|
|
728
|
+
const attachment = this.getAttachment(columnName);
|
|
729
|
+
if (!attachment) {
|
|
730
|
+
throw new Error(`No attachment found in column '${columnName}'`);
|
|
731
|
+
}
|
|
732
|
+
return attachment.download(outputPath);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
class Column extends BaseDatasetEntity {
|
|
737
|
+
constructor(client, data) {
|
|
738
|
+
super(client);
|
|
739
|
+
this._deleted = false;
|
|
740
|
+
this._data = data;
|
|
741
|
+
}
|
|
742
|
+
get slug() {
|
|
743
|
+
return this._data.slug;
|
|
744
|
+
}
|
|
745
|
+
get name() {
|
|
746
|
+
return this._data.name;
|
|
747
|
+
}
|
|
748
|
+
get type() {
|
|
749
|
+
return this._data.type;
|
|
750
|
+
}
|
|
751
|
+
get required() {
|
|
752
|
+
return this._data.required || false;
|
|
753
|
+
}
|
|
754
|
+
get description() {
|
|
755
|
+
return this._data.description;
|
|
756
|
+
}
|
|
757
|
+
get datasetId() {
|
|
758
|
+
return this._data.datasetId;
|
|
759
|
+
}
|
|
760
|
+
get datasetSlug() {
|
|
761
|
+
return this._data.datasetSlug;
|
|
762
|
+
}
|
|
763
|
+
get createdAt() {
|
|
764
|
+
return this._data.createdAt;
|
|
765
|
+
}
|
|
766
|
+
get updatedAt() {
|
|
767
|
+
return this._data.updatedAt;
|
|
768
|
+
}
|
|
769
|
+
get deleted() {
|
|
770
|
+
return this._deleted;
|
|
771
|
+
}
|
|
772
|
+
async update(options) {
|
|
773
|
+
if (this._deleted) {
|
|
774
|
+
throw new Error("Cannot update a deleted column");
|
|
775
|
+
}
|
|
776
|
+
if (options.name && typeof options.name !== "string") {
|
|
777
|
+
throw new Error("Column name must be a string");
|
|
778
|
+
}
|
|
779
|
+
if (options.type &&
|
|
780
|
+
!["string", "number", "boolean", "date", "file"].includes(options.type)) {
|
|
781
|
+
throw new Error("Column type must be one of: string, number, boolean, date, file");
|
|
782
|
+
}
|
|
783
|
+
const response = await this.client.put(`/v2/datasets/${this.datasetSlug}/columns/${this.slug}`, options);
|
|
784
|
+
const data = await this.handleResponse(response);
|
|
785
|
+
// API returns dataset data, extract column info if available
|
|
786
|
+
if (data.columns && data.columns[this.slug]) {
|
|
787
|
+
const columnData = data.columns[this.slug];
|
|
788
|
+
// Update only the fields that changed, preserve datasetSlug and other metadata
|
|
789
|
+
this._data = Object.assign(Object.assign({}, this._data), { name: columnData.name || this._data.name, type: columnData.type || this._data.type, description: columnData.description || this._data.description, updatedAt: data.updatedAt || this._data.updatedAt });
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
async delete() {
|
|
793
|
+
if (this._deleted) {
|
|
794
|
+
throw new Error("Column is already deleted");
|
|
795
|
+
}
|
|
796
|
+
const response = await this.client.delete(`/v2/datasets/${this.datasetSlug}/columns/${this.slug}`);
|
|
797
|
+
await this.handleResponse(response);
|
|
798
|
+
this._deleted = true;
|
|
799
|
+
}
|
|
800
|
+
validateValue(value) {
|
|
801
|
+
if (this.required && (value === null || value === undefined)) {
|
|
802
|
+
return false;
|
|
803
|
+
}
|
|
804
|
+
if (value === null || value === undefined) {
|
|
805
|
+
return true;
|
|
806
|
+
}
|
|
807
|
+
switch (this.type) {
|
|
808
|
+
case "string":
|
|
809
|
+
return typeof value === "string";
|
|
810
|
+
case "number":
|
|
811
|
+
return typeof value === "number" && !isNaN(value) && isFinite(value);
|
|
812
|
+
case "boolean":
|
|
813
|
+
return typeof value === "boolean";
|
|
814
|
+
default:
|
|
815
|
+
return false;
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
convertValue(value) {
|
|
819
|
+
if (value === null || value === undefined) {
|
|
820
|
+
return null;
|
|
821
|
+
}
|
|
822
|
+
switch (this.type) {
|
|
823
|
+
case "string":
|
|
824
|
+
return String(value);
|
|
825
|
+
case "number": {
|
|
826
|
+
const numValue = Number(value);
|
|
827
|
+
return isNaN(numValue) ? null : numValue;
|
|
828
|
+
}
|
|
829
|
+
case "boolean":
|
|
830
|
+
if (typeof value === "boolean")
|
|
831
|
+
return value;
|
|
832
|
+
if (typeof value === "string") {
|
|
833
|
+
const lower = value.toLowerCase();
|
|
834
|
+
if (lower === "true" || lower === "1")
|
|
835
|
+
return true;
|
|
836
|
+
if (lower === "false" || lower === "0")
|
|
837
|
+
return false;
|
|
838
|
+
}
|
|
839
|
+
return Boolean(value);
|
|
840
|
+
default:
|
|
841
|
+
return value;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
class Dataset extends BaseDatasetEntity {
|
|
847
|
+
constructor(client, data) {
|
|
848
|
+
super(client);
|
|
849
|
+
this._deleted = false;
|
|
850
|
+
this._data = data;
|
|
851
|
+
this._attachmentUploader = new AttachmentUploader(client);
|
|
852
|
+
}
|
|
853
|
+
get id() {
|
|
854
|
+
return this._data.id;
|
|
855
|
+
}
|
|
856
|
+
get slug() {
|
|
857
|
+
return this._data.slug;
|
|
858
|
+
}
|
|
859
|
+
get name() {
|
|
860
|
+
return this._data.name;
|
|
861
|
+
}
|
|
862
|
+
get description() {
|
|
863
|
+
return this._data.description;
|
|
864
|
+
}
|
|
865
|
+
get version() {
|
|
866
|
+
return this._data.version;
|
|
867
|
+
}
|
|
868
|
+
get published() {
|
|
869
|
+
return this._data.published || false;
|
|
870
|
+
}
|
|
871
|
+
get createdAt() {
|
|
872
|
+
return this._data.createdAt || "";
|
|
873
|
+
}
|
|
874
|
+
get updatedAt() {
|
|
875
|
+
return this._data.updatedAt || "";
|
|
876
|
+
}
|
|
877
|
+
get deleted() {
|
|
878
|
+
return this._deleted;
|
|
879
|
+
}
|
|
880
|
+
async update(options) {
|
|
881
|
+
if (this._deleted) {
|
|
882
|
+
throw new Error("Cannot update a deleted dataset");
|
|
883
|
+
}
|
|
884
|
+
if (options.name) {
|
|
885
|
+
this.validateDatasetName(options.name);
|
|
886
|
+
}
|
|
887
|
+
const response = await this.client.put(`/v2/datasets/${this.slug}`, options);
|
|
888
|
+
await this.handleResponse(response);
|
|
889
|
+
}
|
|
890
|
+
async delete() {
|
|
891
|
+
if (this._deleted) {
|
|
892
|
+
throw new Error("Dataset is already deleted");
|
|
893
|
+
}
|
|
894
|
+
const response = await this.client.delete(`/v2/datasets/${this.slug}`);
|
|
895
|
+
await this.handleResponse(response);
|
|
896
|
+
this._deleted = true;
|
|
897
|
+
}
|
|
898
|
+
async publish(options = {}) {
|
|
899
|
+
if (this._deleted) {
|
|
900
|
+
throw new Error("Cannot publish a deleted dataset");
|
|
901
|
+
}
|
|
902
|
+
const response = await this.client.post(`/v2/datasets/${this.slug}/publish`, options);
|
|
903
|
+
const data = await this.handleResponse(response);
|
|
904
|
+
this._data = data;
|
|
905
|
+
}
|
|
906
|
+
async addColumn(columns) {
|
|
907
|
+
if (this._deleted) {
|
|
908
|
+
throw new Error("Cannot add columns to a deleted dataset");
|
|
909
|
+
}
|
|
910
|
+
if (!Array.isArray(columns) || columns.length === 0) {
|
|
911
|
+
throw new Error("Columns must be a non-empty array");
|
|
912
|
+
}
|
|
913
|
+
const results = [];
|
|
914
|
+
for (const column of columns) {
|
|
915
|
+
if (!column.name || typeof column.name !== "string") {
|
|
916
|
+
throw new Error("Column name is required and must be a string");
|
|
917
|
+
}
|
|
918
|
+
const response = await this.client.post(`/v2/datasets/${this.slug}/columns`, column);
|
|
919
|
+
const data = await this.handleResponse(response);
|
|
920
|
+
if (!data || !data.slug) {
|
|
921
|
+
throw new Error("Failed to create column: Invalid API response");
|
|
922
|
+
}
|
|
923
|
+
const columnResponse = {
|
|
924
|
+
slug: data.slug,
|
|
925
|
+
datasetId: this._data.id,
|
|
926
|
+
datasetSlug: this.slug,
|
|
927
|
+
name: data.name,
|
|
928
|
+
type: data.type,
|
|
929
|
+
required: data.required,
|
|
930
|
+
description: data.description,
|
|
931
|
+
createdAt: data.createdAt,
|
|
932
|
+
updatedAt: data.updatedAt,
|
|
933
|
+
};
|
|
934
|
+
results.push(new Column(this.client, columnResponse));
|
|
935
|
+
}
|
|
936
|
+
return results;
|
|
937
|
+
}
|
|
938
|
+
async getColumns() {
|
|
939
|
+
if (this._deleted) {
|
|
940
|
+
throw new Error("Cannot get columns from a deleted dataset");
|
|
941
|
+
}
|
|
942
|
+
if (!this._data.columns) {
|
|
943
|
+
return [];
|
|
944
|
+
}
|
|
945
|
+
const columns = [];
|
|
946
|
+
for (const [columnSlug, columnData] of Object.entries(this._data.columns)) {
|
|
947
|
+
const col = columnData;
|
|
948
|
+
const columnResponse = {
|
|
949
|
+
slug: columnSlug,
|
|
950
|
+
datasetId: this._data.id,
|
|
951
|
+
datasetSlug: this.slug,
|
|
952
|
+
name: col.name,
|
|
953
|
+
type: col.type,
|
|
954
|
+
required: col.required === true,
|
|
955
|
+
description: col.description,
|
|
956
|
+
createdAt: this._data.createdAt || this.createdAt,
|
|
957
|
+
updatedAt: this._data.updatedAt || this.updatedAt,
|
|
958
|
+
};
|
|
959
|
+
columns.push(new Column(this.client, columnResponse));
|
|
960
|
+
}
|
|
961
|
+
return columns;
|
|
962
|
+
}
|
|
963
|
+
async addRow(rowData) {
|
|
964
|
+
if (this._deleted) {
|
|
965
|
+
throw new Error("Cannot add row to a deleted dataset");
|
|
966
|
+
}
|
|
967
|
+
if (!rowData || typeof rowData !== "object") {
|
|
968
|
+
throw new Error("Row data must be a valid object");
|
|
969
|
+
}
|
|
970
|
+
const rows = await this.addRows([rowData]);
|
|
971
|
+
if (rows.length === 0) {
|
|
972
|
+
throw new Error("Failed to add row");
|
|
973
|
+
}
|
|
974
|
+
return rows[0];
|
|
975
|
+
}
|
|
976
|
+
async addRows(rows) {
|
|
977
|
+
if (this._deleted) {
|
|
978
|
+
throw new Error("Cannot add rows to a deleted dataset");
|
|
979
|
+
}
|
|
980
|
+
if (!Array.isArray(rows)) {
|
|
981
|
+
throw new Error("Rows must be an array");
|
|
982
|
+
}
|
|
983
|
+
const columns = await this.getColumns();
|
|
984
|
+
const columnMap = new Map();
|
|
985
|
+
columns.forEach((col) => {
|
|
986
|
+
columnMap.set(col.name, col.slug);
|
|
987
|
+
});
|
|
988
|
+
// Phase 1: Extract attachments and prepare clean rows
|
|
989
|
+
const { cleanRows, attachmentMap } = this.extractAttachments(rows, columnMap);
|
|
990
|
+
// Phase 2: Create rows with regular data (attachments replaced with null)
|
|
991
|
+
const transformedRows = cleanRows.map((row) => {
|
|
992
|
+
const transformedRow = {};
|
|
993
|
+
Object.keys(row).forEach((columnName) => {
|
|
994
|
+
const columnSlug = columnMap.get(columnName);
|
|
995
|
+
if (columnSlug) {
|
|
996
|
+
transformedRow[columnSlug] = row[columnName];
|
|
997
|
+
}
|
|
998
|
+
});
|
|
999
|
+
return transformedRow;
|
|
1000
|
+
});
|
|
1001
|
+
const payload = {
|
|
1002
|
+
Rows: transformedRows,
|
|
1003
|
+
};
|
|
1004
|
+
const response = await this.client.post(`/v2/datasets/${this.slug}/rows`, payload);
|
|
1005
|
+
const result = await this.handleResponse(response);
|
|
1006
|
+
const createdRows = [];
|
|
1007
|
+
if (result.rows) {
|
|
1008
|
+
for (const row of result.rows) {
|
|
1009
|
+
const rowResponse = {
|
|
1010
|
+
id: row.id,
|
|
1011
|
+
datasetId: this._data.id,
|
|
1012
|
+
datasetSlug: this.slug,
|
|
1013
|
+
data: this.transformValuesBackToNames(row.values, columnMap),
|
|
1014
|
+
createdAt: row.created_at,
|
|
1015
|
+
updatedAt: row.updated_at,
|
|
1016
|
+
};
|
|
1017
|
+
createdRows.push(new Row(this.client, rowResponse));
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
// Phase 3: Process attachments for created rows
|
|
1021
|
+
if (attachmentMap.size > 0) {
|
|
1022
|
+
await this.processAttachments(createdRows, attachmentMap, columnMap);
|
|
1023
|
+
}
|
|
1024
|
+
return createdRows;
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Extracts attachments from rows and returns clean rows with null values
|
|
1028
|
+
*/
|
|
1029
|
+
extractAttachments(rows, columnMap) {
|
|
1030
|
+
const attachmentMap = new Map();
|
|
1031
|
+
const cleanRows = [];
|
|
1032
|
+
rows.forEach((row, rowIndex) => {
|
|
1033
|
+
const cleanRow = {};
|
|
1034
|
+
const rowAttachments = new Map();
|
|
1035
|
+
Object.keys(row).forEach((columnName) => {
|
|
1036
|
+
const value = row[columnName];
|
|
1037
|
+
const columnSlug = columnMap.get(columnName);
|
|
1038
|
+
if (isAnyAttachment(value) && columnSlug) {
|
|
1039
|
+
// Store attachment for later processing
|
|
1040
|
+
rowAttachments.set(columnSlug, value);
|
|
1041
|
+
// Replace with null in the clean row
|
|
1042
|
+
cleanRow[columnName] = null;
|
|
1043
|
+
}
|
|
1044
|
+
else {
|
|
1045
|
+
cleanRow[columnName] = value;
|
|
1046
|
+
}
|
|
1047
|
+
});
|
|
1048
|
+
cleanRows.push(cleanRow);
|
|
1049
|
+
if (rowAttachments.size > 0) {
|
|
1050
|
+
attachmentMap.set(rowIndex, rowAttachments);
|
|
1051
|
+
}
|
|
1052
|
+
});
|
|
1053
|
+
return { cleanRows, attachmentMap };
|
|
1054
|
+
}
|
|
1055
|
+
/**
|
|
1056
|
+
* Processes attachments for created rows
|
|
1057
|
+
*/
|
|
1058
|
+
async processAttachments(rows, attachmentMap, columnMap) {
|
|
1059
|
+
// Create reverse map for slug to name lookup
|
|
1060
|
+
const reverseColumnMap = new Map();
|
|
1061
|
+
columnMap.forEach((slug, name) => {
|
|
1062
|
+
reverseColumnMap.set(slug, name);
|
|
1063
|
+
});
|
|
1064
|
+
for (const [rowIndex, rowAttachments] of attachmentMap) {
|
|
1065
|
+
const row = rows[rowIndex];
|
|
1066
|
+
if (!row)
|
|
1067
|
+
continue;
|
|
1068
|
+
for (const [columnSlug, attachment] of rowAttachments) {
|
|
1069
|
+
try {
|
|
1070
|
+
const reference = await this._attachmentUploader.processAnyAttachment(this.slug, row.id, columnSlug, attachment);
|
|
1071
|
+
// Update the row's internal data with the attachment reference
|
|
1072
|
+
const columnName = reverseColumnMap.get(columnSlug);
|
|
1073
|
+
if (columnName) {
|
|
1074
|
+
row._data.data[columnName] = reference.toJSON();
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
catch (error) {
|
|
1078
|
+
// Log warning but don't fail the entire operation
|
|
1079
|
+
console.warn(`Failed to process attachment for row ${row.id}, column ${columnSlug}:`, error);
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
transformValuesBackToNames(values, columnMap) {
|
|
1085
|
+
const result = {};
|
|
1086
|
+
const reverseMap = new Map();
|
|
1087
|
+
columnMap.forEach((slug, name) => {
|
|
1088
|
+
reverseMap.set(slug, name);
|
|
1089
|
+
});
|
|
1090
|
+
Object.keys(values).forEach((columnSlug) => {
|
|
1091
|
+
const columnName = reverseMap.get(columnSlug);
|
|
1092
|
+
if (columnName) {
|
|
1093
|
+
result[columnName] = values[columnSlug];
|
|
1094
|
+
}
|
|
1095
|
+
});
|
|
1096
|
+
return result;
|
|
1097
|
+
}
|
|
1098
|
+
async getRows(limit = 100, offset = 0) {
|
|
1099
|
+
if (this._deleted) {
|
|
1100
|
+
throw new Error("Cannot get rows from a deleted dataset");
|
|
1101
|
+
}
|
|
1102
|
+
const response = await this.client.get(`/v2/datasets/${this.slug}/rows?limit=${limit}&offset=${offset}`);
|
|
1103
|
+
const data = await this.handleResponse(response);
|
|
1104
|
+
const rows = data.rows || [];
|
|
1105
|
+
return rows.map((row) => {
|
|
1106
|
+
const rowResponse = {
|
|
1107
|
+
id: row.id,
|
|
1108
|
+
datasetId: this._data.id,
|
|
1109
|
+
datasetSlug: this.slug,
|
|
1110
|
+
data: row.values || row.data || {},
|
|
1111
|
+
createdAt: row.created_at,
|
|
1112
|
+
updatedAt: row.updated_at,
|
|
1113
|
+
};
|
|
1114
|
+
return new Row(this.client, rowResponse);
|
|
1115
|
+
});
|
|
1116
|
+
}
|
|
1117
|
+
async fromCSV(csvContent, options = {}) {
|
|
1118
|
+
if (this._deleted) {
|
|
1119
|
+
throw new Error("Cannot import CSV to a deleted dataset");
|
|
1120
|
+
}
|
|
1121
|
+
const { hasHeader = true, delimiter = "," } = options;
|
|
1122
|
+
if (!csvContent || typeof csvContent !== "string") {
|
|
1123
|
+
throw new Error("CSV content must be a valid string");
|
|
1124
|
+
}
|
|
1125
|
+
const rows = this.parseCSV(csvContent, delimiter, hasHeader);
|
|
1126
|
+
if (rows.length === 0) {
|
|
1127
|
+
throw new Error("No data found in CSV");
|
|
1128
|
+
}
|
|
1129
|
+
const batchSize = 100;
|
|
1130
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
1131
|
+
const batch = rows.slice(i, i + batchSize);
|
|
1132
|
+
await this.addRows(batch);
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
async getVersions() {
|
|
1136
|
+
if (this._deleted) {
|
|
1137
|
+
throw new Error("Cannot get versions of a deleted dataset");
|
|
1138
|
+
}
|
|
1139
|
+
const response = await this.client.get(`/v2/datasets/${this.slug}/versions`);
|
|
1140
|
+
return await this.handleResponse(response);
|
|
1141
|
+
}
|
|
1142
|
+
async getVersion(version) {
|
|
1143
|
+
if (this._deleted) {
|
|
1144
|
+
throw new Error("Cannot get version of a deleted dataset");
|
|
1145
|
+
}
|
|
1146
|
+
const versionsData = await this.getVersions();
|
|
1147
|
+
return versionsData.versions.find((v) => v.version === version) || null;
|
|
1148
|
+
}
|
|
1149
|
+
parseCSV(csvContent, delimiter, hasHeader) {
|
|
1150
|
+
const parseResult = Papa.parse(csvContent, {
|
|
1151
|
+
delimiter,
|
|
1152
|
+
header: hasHeader,
|
|
1153
|
+
skipEmptyLines: true,
|
|
1154
|
+
transformHeader: (header) => header.trim(),
|
|
1155
|
+
transform: (value) => this.parseValue(value.trim()),
|
|
1156
|
+
});
|
|
1157
|
+
if (parseResult.errors.length > 0) {
|
|
1158
|
+
throw new Error(`CSV parsing failed: ${parseResult.errors[0].message}`);
|
|
1159
|
+
}
|
|
1160
|
+
return parseResult.data;
|
|
1161
|
+
}
|
|
1162
|
+
parseValue(value) {
|
|
1163
|
+
if (value === "" || value.toLowerCase() === "null") {
|
|
1164
|
+
return null;
|
|
1165
|
+
}
|
|
1166
|
+
if (value.toLowerCase() === "true") {
|
|
1167
|
+
return true;
|
|
1168
|
+
}
|
|
1169
|
+
if (value.toLowerCase() === "false") {
|
|
1170
|
+
return false;
|
|
1171
|
+
}
|
|
1172
|
+
const numValue = Number(value);
|
|
1173
|
+
if (!isNaN(numValue) && isFinite(numValue)) {
|
|
1174
|
+
return numValue;
|
|
1175
|
+
}
|
|
1176
|
+
return value;
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
class Datasets extends BaseDatasetEntity {
|
|
1181
|
+
constructor(client) {
|
|
1182
|
+
super(client);
|
|
1183
|
+
}
|
|
1184
|
+
async create(options) {
|
|
1185
|
+
this.validateDatasetName(options.name);
|
|
1186
|
+
const response = await this.client.post("/v2/datasets", options);
|
|
1187
|
+
const data = await this.handleResponse(response);
|
|
1188
|
+
return new Dataset(this.client, data);
|
|
1189
|
+
}
|
|
1190
|
+
async get(slug) {
|
|
1191
|
+
this.validateDatasetSlug(slug);
|
|
1192
|
+
const response = await this.client.get(`/v2/datasets/${slug}`);
|
|
1193
|
+
const data = await this.handleResponse(response);
|
|
1194
|
+
return new Dataset(this.client, data);
|
|
1195
|
+
}
|
|
1196
|
+
async list() {
|
|
1197
|
+
const response = await this.client.get(`/v2/datasets`);
|
|
1198
|
+
const data = await this.handleResponse(response);
|
|
1199
|
+
if (!data || !data.datasets) {
|
|
1200
|
+
return {
|
|
1201
|
+
datasets: [],
|
|
1202
|
+
total: 0,
|
|
1203
|
+
};
|
|
1204
|
+
}
|
|
1205
|
+
const datasets = data.datasets.map((datasetData) => new Dataset(this.client, datasetData));
|
|
1206
|
+
return Object.assign(Object.assign({}, data), { datasets });
|
|
1207
|
+
}
|
|
1208
|
+
async delete(slug) {
|
|
1209
|
+
this.validateDatasetSlug(slug);
|
|
1210
|
+
const response = await this.client.delete(`/v2/datasets/${slug}`);
|
|
1211
|
+
await this.handleResponse(response);
|
|
1212
|
+
}
|
|
1213
|
+
async getVersionCSV(slug, version) {
|
|
1214
|
+
this.validateDatasetSlug(slug);
|
|
1215
|
+
if (!version || typeof version !== "string") {
|
|
1216
|
+
throw new Error("Version must be a non-empty string");
|
|
1217
|
+
}
|
|
1218
|
+
const response = await this.client.get(`/v2/datasets/${slug}/versions/${version}`);
|
|
1219
|
+
const csvData = await this.handleResponse(response);
|
|
1220
|
+
if (typeof csvData !== "string") {
|
|
1221
|
+
throw new Error("Expected CSV data as string from API");
|
|
1222
|
+
}
|
|
1223
|
+
return csvData;
|
|
1224
|
+
}
|
|
1225
|
+
async getVersionAsJsonl(slug, version) {
|
|
1226
|
+
if (!version || version === "") {
|
|
1227
|
+
throw new Error("Version is required");
|
|
1228
|
+
}
|
|
1229
|
+
const url = `/v2/datasets/${slug}/versions/${version}/jsonl`;
|
|
1230
|
+
const response = await this.client.get(url);
|
|
1231
|
+
if (!response.ok) {
|
|
1232
|
+
throw new Error(`Failed to fetch JSONL data: ${response.status} ${response.statusText}`);
|
|
1233
|
+
}
|
|
1234
|
+
const contentType = response.headers.get("content-type");
|
|
1235
|
+
if (contentType && contentType.includes("application/json")) {
|
|
1236
|
+
// If server returns JSON, handle it appropriately
|
|
1237
|
+
const jsonData = await response.json();
|
|
1238
|
+
if (jsonData.error) {
|
|
1239
|
+
throw new Error(jsonData.error);
|
|
1240
|
+
}
|
|
1241
|
+
// Convert JSON response to JSONL format if needed
|
|
1242
|
+
if (Array.isArray(jsonData)) {
|
|
1243
|
+
return jsonData.map((item) => JSON.stringify(item)).join("\n");
|
|
1244
|
+
}
|
|
1245
|
+
return JSON.stringify(jsonData);
|
|
1246
|
+
}
|
|
1247
|
+
// Expect JSONL format (text/plain or application/jsonl)
|
|
1248
|
+
return await response.text();
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
class Evaluator extends BaseDatasetEntity {
|
|
1253
|
+
constructor(client) {
|
|
1254
|
+
super(client);
|
|
1255
|
+
}
|
|
1256
|
+
/**
|
|
1257
|
+
* Run evaluators on experiment task results and wait for completion
|
|
1258
|
+
*/
|
|
1259
|
+
async runExperimentEvaluator(options) {
|
|
1260
|
+
const { experimentId, experimentRunId, taskId, taskResult, evaluator, waitForResults = true, } = options;
|
|
1261
|
+
this.validateEvaluatorOptions(options);
|
|
1262
|
+
const triggerResponse = await this.triggerExperimentEvaluator({
|
|
1263
|
+
experimentId,
|
|
1264
|
+
experimentRunId,
|
|
1265
|
+
taskId,
|
|
1266
|
+
evaluator,
|
|
1267
|
+
taskResult,
|
|
1268
|
+
});
|
|
1269
|
+
if (!waitForResults) {
|
|
1270
|
+
return [
|
|
1271
|
+
{
|
|
1272
|
+
executionId: triggerResponse.executionId,
|
|
1273
|
+
result: { status: "running", startedAt: new Date().toISOString() },
|
|
1274
|
+
},
|
|
1275
|
+
];
|
|
1276
|
+
}
|
|
1277
|
+
return this.waitForResult(triggerResponse.executionId, triggerResponse.streamUrl);
|
|
1278
|
+
}
|
|
1279
|
+
/**
|
|
1280
|
+
* Trigger evaluator execution without waiting for results
|
|
1281
|
+
*/
|
|
1282
|
+
async triggerExperimentEvaluator(request) {
|
|
1283
|
+
const { experimentId, experimentRunId, taskId, evaluator, taskResult } = request;
|
|
1284
|
+
if (!experimentId || !taskResult) {
|
|
1285
|
+
throw new Error("experimentId, evaluator, and taskResult are required");
|
|
1286
|
+
}
|
|
1287
|
+
// Handle string, EvaluatorWithVersion, and EvaluatorWithConfig types
|
|
1288
|
+
const evaluatorName = typeof evaluator === "string" ? evaluator : evaluator.name;
|
|
1289
|
+
const evaluatorVersion = typeof evaluator === "string" ? undefined : evaluator.version;
|
|
1290
|
+
// Extract config if present (EvaluatorWithConfig type)
|
|
1291
|
+
const evaluatorConfig = typeof evaluator === "object" && "config" in evaluator
|
|
1292
|
+
? evaluator.config
|
|
1293
|
+
: undefined;
|
|
1294
|
+
if (!evaluatorName) {
|
|
1295
|
+
throw new Error("evaluator name is required");
|
|
1296
|
+
}
|
|
1297
|
+
const inputSchemaMapping = this.createInputSchemaMapping(taskResult);
|
|
1298
|
+
const payload = {
|
|
1299
|
+
experiment_id: experimentId,
|
|
1300
|
+
experiment_run_id: experimentRunId,
|
|
1301
|
+
evaluator_version: evaluatorVersion,
|
|
1302
|
+
task_id: taskId,
|
|
1303
|
+
input_schema_mapping: inputSchemaMapping,
|
|
1304
|
+
};
|
|
1305
|
+
// Add evaluator config if present
|
|
1306
|
+
if (evaluatorConfig && Object.keys(evaluatorConfig).length > 0) {
|
|
1307
|
+
payload.evaluator_config = evaluatorConfig;
|
|
1308
|
+
}
|
|
1309
|
+
const response = await this.client.post(`/v2/evaluators/slug/${evaluatorName}/execute`, payload);
|
|
1310
|
+
const data = await this.handleResponse(response);
|
|
1311
|
+
return {
|
|
1312
|
+
executionId: data.executionId,
|
|
1313
|
+
streamUrl: data.streamUrl,
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
/**
|
|
1317
|
+
* Wait for execution result via stream URL (actually JSON endpoint)
|
|
1318
|
+
*/
|
|
1319
|
+
async waitForResult(executionId, streamUrl) {
|
|
1320
|
+
if (!executionId || !streamUrl) {
|
|
1321
|
+
throw new Error("Execution ID and stream URL are required");
|
|
1322
|
+
}
|
|
1323
|
+
const fullStreamUrl = `${this.client["baseUrl"]}/v2${streamUrl}`;
|
|
1324
|
+
try {
|
|
1325
|
+
const response = await fetch(fullStreamUrl, {
|
|
1326
|
+
headers: {
|
|
1327
|
+
Authorization: `Bearer ${this.client["apiKey"]}`,
|
|
1328
|
+
Accept: "application/json",
|
|
1329
|
+
"Cache-Control": "no-cache",
|
|
1330
|
+
},
|
|
1331
|
+
});
|
|
1332
|
+
if (!response.ok) {
|
|
1333
|
+
const errorText = await response.text();
|
|
1334
|
+
throw new Error(`Failed to get results: ${response.status}, body: ${errorText}`);
|
|
1335
|
+
}
|
|
1336
|
+
const responseText = await response.text();
|
|
1337
|
+
const responseData = JSON.parse(responseText);
|
|
1338
|
+
// Check execution ID match
|
|
1339
|
+
if (responseData.execution_id &&
|
|
1340
|
+
responseData.execution_id !== executionId) {
|
|
1341
|
+
throw new Error(`Execution ID mismatch: ${responseData.execution_id} !== ${executionId}`);
|
|
1342
|
+
}
|
|
1343
|
+
// Convert to ExecutionResponse format
|
|
1344
|
+
const executionResponse = {
|
|
1345
|
+
executionId: responseData.execution_id,
|
|
1346
|
+
result: responseData.result,
|
|
1347
|
+
};
|
|
1348
|
+
return [executionResponse];
|
|
1349
|
+
}
|
|
1350
|
+
catch (error) {
|
|
1351
|
+
throw new Error(`Failed to wait for result: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
/**
|
|
1355
|
+
* Validate evaluator run options
|
|
1356
|
+
*/
|
|
1357
|
+
validateEvaluatorOptions(options) {
|
|
1358
|
+
const { experimentId, evaluator, taskResult } = options;
|
|
1359
|
+
if (!experimentId ||
|
|
1360
|
+
typeof experimentId !== "string" ||
|
|
1361
|
+
experimentId.trim().length === 0) {
|
|
1362
|
+
throw new Error("Experiment ID is required and must be a non-empty string");
|
|
1363
|
+
}
|
|
1364
|
+
if (!evaluator) {
|
|
1365
|
+
throw new Error("At least one evaluator must be specified");
|
|
1366
|
+
}
|
|
1367
|
+
if (!taskResult) {
|
|
1368
|
+
throw new Error("At least one task result must be provided");
|
|
1369
|
+
}
|
|
1370
|
+
// Validate evaluator based on its type
|
|
1371
|
+
if (typeof evaluator === "string") {
|
|
1372
|
+
if (!evaluator.trim()) {
|
|
1373
|
+
throw new Error("Evaluator name cannot be empty");
|
|
1374
|
+
}
|
|
1375
|
+
}
|
|
1376
|
+
else {
|
|
1377
|
+
if (!evaluator.name ||
|
|
1378
|
+
typeof evaluator.name !== "string" ||
|
|
1379
|
+
!evaluator.name.trim()) {
|
|
1380
|
+
throw new Error("Evaluator must have a valid name");
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
// Validate each task result
|
|
1384
|
+
if (!taskResult || typeof taskResult !== "object") {
|
|
1385
|
+
throw new Error(`Task result must be a valid object`);
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
/**
|
|
1389
|
+
* Create InputSchemaMapping from input object
|
|
1390
|
+
*/
|
|
1391
|
+
createInputSchemaMapping(input) {
|
|
1392
|
+
const mapping = {};
|
|
1393
|
+
for (const [key, value] of Object.entries(input)) {
|
|
1394
|
+
mapping[key] = { source: String(value) };
|
|
1395
|
+
}
|
|
1396
|
+
return mapping;
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
class Experiment {
|
|
1401
|
+
constructor(client) {
|
|
1402
|
+
this.client = client;
|
|
1403
|
+
this.evaluator = new Evaluator(client);
|
|
1404
|
+
this.datasets = new Datasets(client);
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* Generate a unique experiment slug
|
|
1408
|
+
*/
|
|
1409
|
+
generateExperimentSlug() {
|
|
1410
|
+
const timestamp = Date.now().toString(36);
|
|
1411
|
+
const random = Math.random().toString(36).substring(2, 7);
|
|
1412
|
+
return `exp-${timestamp}${random}`.substring(0, 15);
|
|
1413
|
+
}
|
|
1414
|
+
async handleResponse(response) {
|
|
1415
|
+
if (!response.ok) {
|
|
1416
|
+
let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
|
|
1417
|
+
try {
|
|
1418
|
+
const errorData = await response.json();
|
|
1419
|
+
if (errorData.error) {
|
|
1420
|
+
errorMessage = errorData.error;
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
catch (_a) {
|
|
1424
|
+
// Use default HTTP error message if JSON parsing fails
|
|
1425
|
+
}
|
|
1426
|
+
throw new Error(errorMessage);
|
|
1427
|
+
}
|
|
1428
|
+
const contentType = (response.headers.get("content-type") || "").toLowerCase();
|
|
1429
|
+
if (contentType.includes("text/csv") ||
|
|
1430
|
+
contentType.includes("application/x-ndjson")) {
|
|
1431
|
+
return await response.text();
|
|
1432
|
+
}
|
|
1433
|
+
else {
|
|
1434
|
+
const rawData = await response.json();
|
|
1435
|
+
return transformApiResponse(rawData);
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
/**
|
|
1439
|
+
* Run an experiment with the given task function and options
|
|
1440
|
+
*/
|
|
1441
|
+
async run(task, options = {}) {
|
|
1442
|
+
// Check if running in GitHub Actions
|
|
1443
|
+
if (process.env.GITHUB_ACTIONS === "true") {
|
|
1444
|
+
return await this.runInGithub(task, options);
|
|
1445
|
+
}
|
|
1446
|
+
return await this.runLocally(task, options);
|
|
1447
|
+
}
|
|
1448
|
+
/**
|
|
1449
|
+
* Run an experiment locally (not in GitHub Actions)
|
|
1450
|
+
*/
|
|
1451
|
+
async runLocally(task, options = {}) {
|
|
1452
|
+
const { datasetSlug, datasetVersion, evaluators = [], waitForResults = true, } = options;
|
|
1453
|
+
// When experimentSlug is not provided a random one is generated
|
|
1454
|
+
let { experimentSlug } = options;
|
|
1455
|
+
if (!experimentSlug) {
|
|
1456
|
+
experimentSlug =
|
|
1457
|
+
this.client.experimentSlug || this.generateExperimentSlug();
|
|
1458
|
+
}
|
|
1459
|
+
this.validateRunOptions(task, options);
|
|
1460
|
+
try {
|
|
1461
|
+
const evaluatorSlugs = evaluators.map((evaluator) => typeof evaluator === "string" ? evaluator : evaluator.name);
|
|
1462
|
+
const experimentResponse = await this.initializeExperiment({
|
|
1463
|
+
slug: experimentSlug,
|
|
1464
|
+
datasetSlug,
|
|
1465
|
+
datasetVersion,
|
|
1466
|
+
evaluatorSlugs,
|
|
1467
|
+
});
|
|
1468
|
+
const rows = await this.getDatasetRows(datasetSlug, datasetVersion);
|
|
1469
|
+
const taskResults = [];
|
|
1470
|
+
const taskErrors = [];
|
|
1471
|
+
const evaluationResults = [];
|
|
1472
|
+
for (const row of rows) {
|
|
1473
|
+
const taskOutput = await task(row);
|
|
1474
|
+
// Create TaskResponse object
|
|
1475
|
+
const taskResponse = {
|
|
1476
|
+
input: row,
|
|
1477
|
+
output: taskOutput,
|
|
1478
|
+
metadata: {
|
|
1479
|
+
rowId: row.id,
|
|
1480
|
+
timestamp: Date.now(),
|
|
1481
|
+
},
|
|
1482
|
+
timestamp: Date.now(),
|
|
1483
|
+
};
|
|
1484
|
+
taskResults.push(taskResponse);
|
|
1485
|
+
const response = await this.createTask(experimentSlug, experimentResponse.run.id, row, taskOutput);
|
|
1486
|
+
const taskId = response.id;
|
|
1487
|
+
if (evaluators.length > 0) {
|
|
1488
|
+
for (const evaluator of evaluators) {
|
|
1489
|
+
const singleEvaluationResult = await this.evaluator.runExperimentEvaluator({
|
|
1490
|
+
experimentId: experimentResponse.experiment.id,
|
|
1491
|
+
experimentRunId: experimentResponse.run.id,
|
|
1492
|
+
taskId,
|
|
1493
|
+
evaluator,
|
|
1494
|
+
taskResult: taskOutput,
|
|
1495
|
+
waitForResults,
|
|
1496
|
+
timeout: 120000, // 2 minutes default
|
|
1497
|
+
});
|
|
1498
|
+
evaluationResults.push(...singleEvaluationResult);
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
const evalResults = evaluationResults.map((evaluation) => evaluation.result);
|
|
1503
|
+
// Track last experiment slug and run ID for export methods
|
|
1504
|
+
this._lastExperimentSlug = experimentSlug;
|
|
1505
|
+
this._lastRunId = experimentResponse.run.id;
|
|
1506
|
+
return {
|
|
1507
|
+
taskResults: taskResults,
|
|
1508
|
+
errors: taskErrors,
|
|
1509
|
+
experimentId: experimentResponse.experiment.id,
|
|
1510
|
+
runId: experimentResponse.run.id,
|
|
1511
|
+
evaluations: evalResults,
|
|
1512
|
+
};
|
|
1513
|
+
}
|
|
1514
|
+
catch (error) {
|
|
1515
|
+
throw new Error(`Experiment execution failed: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
/**
|
|
1519
|
+
* Create a task for the experiment
|
|
1520
|
+
*/
|
|
1521
|
+
async createTask(experimentSlug, experimentRunId, taskInput, taskOutput) {
|
|
1522
|
+
const body = {
|
|
1523
|
+
input: taskInput,
|
|
1524
|
+
output: taskOutput,
|
|
1525
|
+
};
|
|
1526
|
+
const response = await this.client.post(`/v2/experiments/${experimentSlug}/runs/${experimentRunId}/task`, body);
|
|
1527
|
+
if (!response.ok) {
|
|
1528
|
+
throw new Error(`Failed to create task for experiment '${experimentSlug}'`);
|
|
1529
|
+
}
|
|
1530
|
+
const data = await this.handleResponse(response);
|
|
1531
|
+
return {
|
|
1532
|
+
id: data.id,
|
|
1533
|
+
};
|
|
1534
|
+
}
|
|
1535
|
+
/**
|
|
1536
|
+
* Initialize a new experiment
|
|
1537
|
+
*/
|
|
1538
|
+
async initializeExperiment(request) {
|
|
1539
|
+
if (request.aux) {
|
|
1540
|
+
request.experimentRunMetadata = Object.assign(Object.assign({}, request.experimentRunMetadata), { aux: request.aux });
|
|
1541
|
+
}
|
|
1542
|
+
if (request.relatedRef) {
|
|
1543
|
+
request.experimentRunMetadata = Object.assign(Object.assign({}, request.experimentRunMetadata), { related_ref: request.relatedRef });
|
|
1544
|
+
}
|
|
1545
|
+
const payload = {
|
|
1546
|
+
slug: request.slug,
|
|
1547
|
+
dataset_slug: request.datasetSlug,
|
|
1548
|
+
dataset_version: request.datasetVersion,
|
|
1549
|
+
evaluator_slugs: request.evaluatorSlugs,
|
|
1550
|
+
experiment_metadata: request.experimentMetadata,
|
|
1551
|
+
experiment_run_metadata: request.experimentRunMetadata,
|
|
1552
|
+
};
|
|
1553
|
+
const response = await this.client.put("/v2/experiments/initialize", payload);
|
|
1554
|
+
const data = await this.handleResponse(response);
|
|
1555
|
+
return data;
|
|
1556
|
+
}
|
|
1557
|
+
/**
|
|
1558
|
+
* Parse JSONL string into list of {col_name: col_value} dictionaries
|
|
1559
|
+
* Skips the first line (columns definition)
|
|
1560
|
+
*/
|
|
1561
|
+
parseJsonlToRows(jsonlData) {
|
|
1562
|
+
const rows = [];
|
|
1563
|
+
const lines = jsonlData.trim().split("\n");
|
|
1564
|
+
// Skip the first line (columns definition)
|
|
1565
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1566
|
+
const line = lines[i].trim();
|
|
1567
|
+
if (line) {
|
|
1568
|
+
try {
|
|
1569
|
+
const rowData = JSON.parse(line);
|
|
1570
|
+
rows.push(rowData);
|
|
1571
|
+
}
|
|
1572
|
+
catch (_a) {
|
|
1573
|
+
// Skip invalid JSON lines
|
|
1574
|
+
continue;
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
return rows;
|
|
1579
|
+
}
|
|
1580
|
+
/**
|
|
1581
|
+
* Get dataset rows for experiment execution
|
|
1582
|
+
*/
|
|
1583
|
+
async getDatasetRows(datasetSlug, datasetVersion) {
|
|
1584
|
+
if (!datasetSlug) {
|
|
1585
|
+
throw new Error("Dataset slug is required for experiment execution");
|
|
1586
|
+
}
|
|
1587
|
+
const dataset = await this.datasets.getVersionAsJsonl(datasetSlug, datasetVersion || "");
|
|
1588
|
+
const rows = this.parseJsonlToRows(dataset);
|
|
1589
|
+
return rows;
|
|
1590
|
+
}
|
|
1591
|
+
/**
|
|
1592
|
+
* Validate experiment run options
|
|
1593
|
+
*/
|
|
1594
|
+
validateRunOptions(task, options) {
|
|
1595
|
+
if (!task || typeof task !== "function") {
|
|
1596
|
+
throw new Error("Task function is required and must be a function");
|
|
1597
|
+
}
|
|
1598
|
+
if (options.evaluators) {
|
|
1599
|
+
options.evaluators.forEach((evaluator, index) => {
|
|
1600
|
+
if (typeof evaluator === "string") {
|
|
1601
|
+
if (!evaluator.trim()) {
|
|
1602
|
+
throw new Error(`Evaluator at index ${index} cannot be an empty string`);
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
else {
|
|
1606
|
+
if (!evaluator || typeof evaluator !== "object") {
|
|
1607
|
+
throw new Error(`Evaluator at index ${index} must be a string or object with name and version`);
|
|
1608
|
+
}
|
|
1609
|
+
if (!evaluator.name ||
|
|
1610
|
+
typeof evaluator.name !== "string" ||
|
|
1611
|
+
!evaluator.name.trim()) {
|
|
1612
|
+
throw new Error(`Evaluator at index ${index} must have a valid non-empty name`);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
/**
|
|
1619
|
+
* Extract GitHub Actions context from environment variables
|
|
1620
|
+
*/
|
|
1621
|
+
getGithubContext() {
|
|
1622
|
+
const repository = process.env.GITHUB_REPOSITORY;
|
|
1623
|
+
const ref = process.env.GITHUB_REF;
|
|
1624
|
+
const sha = process.env.GITHUB_SHA;
|
|
1625
|
+
const actor = process.env.GITHUB_ACTOR;
|
|
1626
|
+
if (!repository || !ref || !sha || !actor) {
|
|
1627
|
+
throw new Error("Missing required GitHub environment variables: GITHUB_REPOSITORY, GITHUB_REF, GITHUB_SHA, or GITHUB_ACTOR");
|
|
1628
|
+
}
|
|
1629
|
+
// Extract PR number from ref (e.g., refs/pull/123/merge -> 123)
|
|
1630
|
+
const prMatch = ref.match(/refs\/pull\/(\d+)\//);
|
|
1631
|
+
const prNumber = prMatch ? prMatch[1] : null;
|
|
1632
|
+
if (!prNumber) {
|
|
1633
|
+
throw new Error(`This method can only be run on pull request events. Current ref: ${ref}`);
|
|
1634
|
+
}
|
|
1635
|
+
const prUrl = `https://github.com/${repository}/pull/${prNumber}`;
|
|
1636
|
+
return {
|
|
1637
|
+
repository,
|
|
1638
|
+
prUrl,
|
|
1639
|
+
commitHash: sha,
|
|
1640
|
+
actor,
|
|
1641
|
+
};
|
|
1642
|
+
}
|
|
1643
|
+
/**
|
|
1644
|
+
* Execute tasks locally and capture results
|
|
1645
|
+
*/
|
|
1646
|
+
async executeTasksLocally(task, rows) {
|
|
1647
|
+
return await Promise.all(rows.map(async (row) => {
|
|
1648
|
+
try {
|
|
1649
|
+
const output = await task(row);
|
|
1650
|
+
return {
|
|
1651
|
+
input: row,
|
|
1652
|
+
output: output,
|
|
1653
|
+
metadata: {
|
|
1654
|
+
rowId: row.id,
|
|
1655
|
+
timestamp: Date.now(),
|
|
1656
|
+
},
|
|
1657
|
+
};
|
|
1658
|
+
}
|
|
1659
|
+
catch (error) {
|
|
1660
|
+
return {
|
|
1661
|
+
input: row,
|
|
1662
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1663
|
+
metadata: {
|
|
1664
|
+
rowId: row.id,
|
|
1665
|
+
timestamp: Date.now(),
|
|
1666
|
+
},
|
|
1667
|
+
};
|
|
1668
|
+
}
|
|
1669
|
+
}));
|
|
1670
|
+
}
|
|
1671
|
+
/**
|
|
1672
|
+
* Run an experiment in GitHub Actions environment
|
|
1673
|
+
* This method executes tasks locally and submits results to the backend for evaluation
|
|
1674
|
+
*/
|
|
1675
|
+
async runInGithub(task, options) {
|
|
1676
|
+
const { datasetSlug, datasetVersion, evaluators = [], experimentMetadata, experimentRunMetadata, relatedRef, aux, } = options;
|
|
1677
|
+
// Generate or use provided experiment slug
|
|
1678
|
+
let { experimentSlug } = options;
|
|
1679
|
+
if (!experimentSlug) {
|
|
1680
|
+
experimentSlug =
|
|
1681
|
+
this.client.experimentSlug || this.generateExperimentSlug();
|
|
1682
|
+
}
|
|
1683
|
+
if (!task || typeof task !== "function") {
|
|
1684
|
+
throw new Error("Task function is required and must be a function");
|
|
1685
|
+
}
|
|
1686
|
+
try {
|
|
1687
|
+
const githubContext = this.getGithubContext();
|
|
1688
|
+
const rows = await this.getDatasetRows(datasetSlug, datasetVersion);
|
|
1689
|
+
const taskResults = await this.executeTasksLocally(task, rows);
|
|
1690
|
+
// Prepare evaluator slugs
|
|
1691
|
+
const evaluatorSlugs = evaluators.map((evaluator) => typeof evaluator === "string" ? evaluator : evaluator.name);
|
|
1692
|
+
const mergedExperimentMetadata = Object.assign(Object.assign({}, (experimentMetadata || {})), { created_from: "github" });
|
|
1693
|
+
const mergedExperimentRunMetadata = Object.assign(Object.assign(Object.assign({}, (experimentRunMetadata || {})), (relatedRef && { related_ref: relatedRef })), (aux && { aux: aux }));
|
|
1694
|
+
// Submit to backend
|
|
1695
|
+
const payload = {
|
|
1696
|
+
experiment_slug: experimentSlug,
|
|
1697
|
+
dataset_slug: datasetSlug,
|
|
1698
|
+
dataset_version: datasetVersion,
|
|
1699
|
+
evaluator_slugs: evaluatorSlugs,
|
|
1700
|
+
task_results: taskResults,
|
|
1701
|
+
github_context: {
|
|
1702
|
+
repository: githubContext.repository,
|
|
1703
|
+
pr_url: githubContext.prUrl,
|
|
1704
|
+
commit_hash: githubContext.commitHash,
|
|
1705
|
+
actor: githubContext.actor,
|
|
1706
|
+
},
|
|
1707
|
+
experiment_metadata: mergedExperimentMetadata,
|
|
1708
|
+
experiment_run_metadata: mergedExperimentRunMetadata,
|
|
1709
|
+
};
|
|
1710
|
+
const response = await this.client.post("/v2/experiments/run-in-github", payload);
|
|
1711
|
+
const data = await this.handleResponse(response);
|
|
1712
|
+
// Track last experiment slug and run ID for export methods
|
|
1713
|
+
this._lastExperimentSlug = data.experimentSlug;
|
|
1714
|
+
this._lastRunId = data.runId;
|
|
1715
|
+
return data;
|
|
1716
|
+
}
|
|
1717
|
+
catch (error) {
|
|
1718
|
+
throw new Error(`GitHub experiment execution failed: ${error instanceof Error ? error.message : "Unknown error"}`);
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
/**
|
|
1722
|
+
* Resolve export parameters by falling back to last used values
|
|
1723
|
+
*/
|
|
1724
|
+
resolveExportParams(experimentSlug, runId) {
|
|
1725
|
+
const slug = experimentSlug || this._lastExperimentSlug;
|
|
1726
|
+
const rid = runId || this._lastRunId;
|
|
1727
|
+
if (!slug) {
|
|
1728
|
+
throw new Error("experiment_slug is required");
|
|
1729
|
+
}
|
|
1730
|
+
if (!rid) {
|
|
1731
|
+
throw new Error("run_id is required");
|
|
1732
|
+
}
|
|
1733
|
+
return { slug, runId: rid };
|
|
1734
|
+
}
|
|
1735
|
+
/**
|
|
1736
|
+
* Export experiment results as CSV string
|
|
1737
|
+
* @param experimentSlug - Optional experiment slug (uses last run if not provided)
|
|
1738
|
+
* @param runId - Optional run ID (uses last run if not provided)
|
|
1739
|
+
* @returns CSV string of experiment results
|
|
1740
|
+
*/
|
|
1741
|
+
async toCsvString(experimentSlug, runId) {
|
|
1742
|
+
const { slug, runId: rid } = this.resolveExportParams(experimentSlug, runId);
|
|
1743
|
+
const response = await this.client.get(`/v2/experiments/${slug}/runs/${rid}/export/csv`);
|
|
1744
|
+
if (!response.ok) {
|
|
1745
|
+
throw new Error(`Failed to export CSV for experiment '${slug}' run '${rid}'`);
|
|
1746
|
+
}
|
|
1747
|
+
const result = await this.handleResponse(response);
|
|
1748
|
+
if (result === null || result === undefined) {
|
|
1749
|
+
throw new Error(`Failed to export CSV for experiment '${slug}' run '${rid}'`);
|
|
1750
|
+
}
|
|
1751
|
+
return String(result);
|
|
1752
|
+
}
|
|
1753
|
+
/**
|
|
1754
|
+
* Export experiment results as JSON string
|
|
1755
|
+
* @param experimentSlug - Optional experiment slug (uses last run if not provided)
|
|
1756
|
+
* @param runId - Optional run ID (uses last run if not provided)
|
|
1757
|
+
* @returns JSON string of experiment results
|
|
1758
|
+
*/
|
|
1759
|
+
async toJsonString(experimentSlug, runId) {
|
|
1760
|
+
const { slug, runId: rid } = this.resolveExportParams(experimentSlug, runId);
|
|
1761
|
+
const response = await this.client.get(`/v2/experiments/${slug}/runs/${rid}/export/json`);
|
|
1762
|
+
if (!response.ok) {
|
|
1763
|
+
throw new Error(`Failed to export JSON for experiment '${slug}' run '${rid}'`);
|
|
1764
|
+
}
|
|
1765
|
+
const result = await this.handleResponse(response);
|
|
1766
|
+
if (result === null || result === undefined) {
|
|
1767
|
+
throw new Error(`Failed to export JSON for experiment '${slug}' run '${rid}'`);
|
|
1768
|
+
}
|
|
1769
|
+
// If result is already a string, return it; otherwise stringify it
|
|
1770
|
+
return typeof result === "string" ? result : JSON.stringify(result);
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
/**
|
|
1775
|
+
* The main client for interacting with Traceloop's API.
|
|
1776
|
+
* This client can be used either directly or through the singleton pattern via configuration.
|
|
1777
|
+
*
|
|
1778
|
+
* @example
|
|
1779
|
+
* // Direct usage
|
|
1780
|
+
* const client = new TraceloopClient('your-api-key');
|
|
1781
|
+
*
|
|
1782
|
+
* @example
|
|
1783
|
+
* // Through configuration (recommended)
|
|
1784
|
+
* initialize({ apiKey: 'your-api-key', appName: 'your-app' });
|
|
1785
|
+
* const client = getClient();
|
|
1786
|
+
*/
|
|
1787
|
+
class TraceloopClient {
|
|
1788
|
+
/**
|
|
1789
|
+
* Creates a new instance of the TraceloopClient.
|
|
1790
|
+
*
|
|
1791
|
+
* @param options - Configuration options for the client
|
|
1792
|
+
*/
|
|
1793
|
+
constructor(options) {
|
|
1794
|
+
this.version = version;
|
|
1795
|
+
this.apiKey = options.apiKey;
|
|
1796
|
+
this.appName = options.appName;
|
|
1797
|
+
this.baseUrl =
|
|
1798
|
+
options.baseUrl ||
|
|
1799
|
+
process.env.ANYWAY_BASE_URL ||
|
|
1800
|
+
"https://api.traceloop.com";
|
|
1801
|
+
this.experimentSlug = options.experimentSlug;
|
|
1802
|
+
this.userFeedback = new UserFeedback(this);
|
|
1803
|
+
this.datasets = new Datasets(this);
|
|
1804
|
+
this.experiment = new Experiment(this);
|
|
1805
|
+
this.evaluator = new Evaluator(this);
|
|
1806
|
+
}
|
|
1807
|
+
async post(path, body) {
|
|
1808
|
+
return await fetch(`${this.baseUrl}${path}`, {
|
|
1809
|
+
method: "POST",
|
|
1810
|
+
headers: {
|
|
1811
|
+
"Content-Type": "application/json",
|
|
1812
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
1813
|
+
"X-Traceloop-SDK-Version": this.version,
|
|
1814
|
+
},
|
|
1815
|
+
body: JSON.stringify(body),
|
|
1816
|
+
});
|
|
1817
|
+
}
|
|
1818
|
+
async get(path) {
|
|
1819
|
+
return await fetch(`${this.baseUrl}${path}`, {
|
|
1820
|
+
method: "GET",
|
|
1821
|
+
headers: {
|
|
1822
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
1823
|
+
"X-Traceloop-SDK-Version": this.version,
|
|
1824
|
+
},
|
|
1825
|
+
});
|
|
1826
|
+
}
|
|
1827
|
+
async put(path, body) {
|
|
1828
|
+
return await fetch(`${this.baseUrl}${path}`, {
|
|
1829
|
+
method: "PUT",
|
|
1830
|
+
headers: {
|
|
1831
|
+
"Content-Type": "application/json",
|
|
1832
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
1833
|
+
"X-Traceloop-SDK-Version": this.version,
|
|
1834
|
+
},
|
|
1835
|
+
body: JSON.stringify(body),
|
|
1836
|
+
});
|
|
1837
|
+
}
|
|
1838
|
+
async delete(path) {
|
|
1839
|
+
return await fetch(`${this.baseUrl}${path}`, {
|
|
1840
|
+
method: "DELETE",
|
|
1841
|
+
headers: {
|
|
1842
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
1843
|
+
"X-Traceloop-SDK-Version": this.version,
|
|
1844
|
+
},
|
|
1845
|
+
});
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
// Auto-generated - DO NOT EDIT
|
|
1850
|
+
// Regenerate with: pnpm generate:evaluator-models
|
|
1851
|
+
const EVALUATOR_SLUGS = [
|
|
1852
|
+
'agent-efficiency',
|
|
1853
|
+
'agent-flow-quality',
|
|
1854
|
+
'agent-goal-accuracy',
|
|
1855
|
+
'agent-goal-completeness',
|
|
1856
|
+
'agent-tool-error-detector',
|
|
1857
|
+
'agent-tool-trajectory',
|
|
1858
|
+
'answer-completeness',
|
|
1859
|
+
'answer-correctness',
|
|
1860
|
+
'answer-relevancy',
|
|
1861
|
+
'char-count',
|
|
1862
|
+
'char-count-ratio',
|
|
1863
|
+
'context-relevance',
|
|
1864
|
+
'conversation-quality',
|
|
1865
|
+
'faithfulness',
|
|
1866
|
+
'html-comparison',
|
|
1867
|
+
'instruction-adherence',
|
|
1868
|
+
'intent-change',
|
|
1869
|
+
'json-validator',
|
|
1870
|
+
'perplexity',
|
|
1871
|
+
'pii-detector',
|
|
1872
|
+
'placeholder-regex',
|
|
1873
|
+
'profanity-detector',
|
|
1874
|
+
'prompt-injection',
|
|
1875
|
+
'prompt-perplexity',
|
|
1876
|
+
'regex-validator',
|
|
1877
|
+
'secrets-detector',
|
|
1878
|
+
'semantic-similarity',
|
|
1879
|
+
'sexism-detector',
|
|
1880
|
+
'sql-validator',
|
|
1881
|
+
'tone-detection',
|
|
1882
|
+
'topic-adherence',
|
|
1883
|
+
'toxicity-detector',
|
|
1884
|
+
'uncertainty-detector',
|
|
1885
|
+
'word-count',
|
|
1886
|
+
'word-count-ratio',
|
|
1887
|
+
];
|
|
1888
|
+
const EVALUATOR_SCHEMAS = {
|
|
1889
|
+
'agent-efficiency': {
|
|
1890
|
+
slug: 'agent-efficiency',
|
|
1891
|
+
requiredInputFields: ['trajectory_completions', 'trajectory_prompts'],
|
|
1892
|
+
optionalConfigFields: [],
|
|
1893
|
+
description: "Evaluate agent efficiency - detect redundant calls, unnecessary follow-ups\n\n**Request Body:**\n- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory\n- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory",
|
|
1894
|
+
},
|
|
1895
|
+
'agent-flow-quality': {
|
|
1896
|
+
slug: 'agent-flow-quality',
|
|
1897
|
+
requiredInputFields: ['trajectory_completions', 'trajectory_prompts'],
|
|
1898
|
+
optionalConfigFields: ['conditions', 'threshold'],
|
|
1899
|
+
description: "Validate agent trajectory against user-defined conditions\n\n**Request Body:**\n- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory\n- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory\n- `config.conditions` (array of strings, required): Array of evaluation conditions/rules to validate against\n- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)",
|
|
1900
|
+
},
|
|
1901
|
+
'agent-goal-accuracy': {
|
|
1902
|
+
slug: 'agent-goal-accuracy',
|
|
1903
|
+
requiredInputFields: ['completion', 'question', 'reference'],
|
|
1904
|
+
optionalConfigFields: [],
|
|
1905
|
+
description: "Evaluate agent goal accuracy\n\n**Request Body:**\n- `input.question` (string, required): The original question or goal\n- `input.completion` (string, required): The agent's completion/response\n- `input.reference` (string, required): The expected reference answer",
|
|
1906
|
+
},
|
|
1907
|
+
'agent-goal-completeness': {
|
|
1908
|
+
slug: 'agent-goal-completeness',
|
|
1909
|
+
requiredInputFields: ['trajectory_completions', 'trajectory_prompts'],
|
|
1910
|
+
optionalConfigFields: ['threshold'],
|
|
1911
|
+
description: "Measure if agent accomplished all user goals\n\n**Request Body:**\n- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory\n- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory\n- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)",
|
|
1912
|
+
},
|
|
1913
|
+
'agent-tool-error-detector': {
|
|
1914
|
+
slug: 'agent-tool-error-detector',
|
|
1915
|
+
requiredInputFields: ['tool_input', 'tool_output'],
|
|
1916
|
+
optionalConfigFields: [],
|
|
1917
|
+
description: "Detect errors or failures during tool execution\n\n**Request Body:**\n- `input.tool_input` (string, required): JSON string of the tool input\n- `input.tool_output` (string, required): JSON string of the tool output",
|
|
1918
|
+
},
|
|
1919
|
+
'agent-tool-trajectory': {
|
|
1920
|
+
slug: 'agent-tool-trajectory',
|
|
1921
|
+
requiredInputFields: ['executed_tool_calls', 'expected_tool_calls'],
|
|
1922
|
+
optionalConfigFields: ['input_params_sensitive', 'mismatch_sensitive', 'order_sensitive', 'threshold'],
|
|
1923
|
+
description: "Compare actual tool calls against expected reference tool calls\n\n**Request Body:**\n- `input.executed_tool_calls` (string, required): JSON array of actual tool calls made by the agent\n- `input.expected_tool_calls` (string, required): JSON array of expected/reference tool calls\n- `config.threshold` (float, optional): Score threshold for pass/fail determination (default: 0.5)\n- `config.mismatch_sensitive` (bool, optional): Whether tool calls must match exactly (default: false)\n- `config.order_sensitive` (bool, optional): Whether order of tool calls matters (default: false)\n- `config.input_params_sensitive` (bool, optional): Whether to compare input parameters (default: true)",
|
|
1924
|
+
},
|
|
1925
|
+
'answer-completeness': {
|
|
1926
|
+
slug: 'answer-completeness',
|
|
1927
|
+
requiredInputFields: ['completion', 'context', 'question'],
|
|
1928
|
+
optionalConfigFields: [],
|
|
1929
|
+
description: "Evaluate whether the answer is complete and contains all the necessary information\n\n**Request Body:**\n- `input.question` (string, required): The original question\n- `input.completion` (string, required): The completion to evaluate for completeness\n- `input.context` (string, required): The context that provides the complete information",
|
|
1930
|
+
},
|
|
1931
|
+
'answer-correctness': {
|
|
1932
|
+
slug: 'answer-correctness',
|
|
1933
|
+
requiredInputFields: ['completion', 'ground_truth', 'question'],
|
|
1934
|
+
optionalConfigFields: [],
|
|
1935
|
+
description: "Evaluate factual accuracy by comparing answers against ground truth\n\n**Request Body:**\n- `input.question` (string, required): The original question\n- `input.completion` (string, required): The completion to evaluate\n- `input.ground_truth` (string, required): The expected correct answer",
|
|
1936
|
+
},
|
|
1937
|
+
'answer-relevancy': {
|
|
1938
|
+
slug: 'answer-relevancy',
|
|
1939
|
+
requiredInputFields: ['answer', 'question'],
|
|
1940
|
+
optionalConfigFields: [],
|
|
1941
|
+
description: "Check if an answer is relevant to a question\n\n**Request Body:**\n- `input.answer` (string, required): The answer to evaluate for relevancy\n- `input.question` (string, required): The question that the answer should be relevant to",
|
|
1942
|
+
},
|
|
1943
|
+
'char-count': {
|
|
1944
|
+
slug: 'char-count',
|
|
1945
|
+
requiredInputFields: ['text'],
|
|
1946
|
+
optionalConfigFields: [],
|
|
1947
|
+
description: "Count the number of characters in text\n\n**Request Body:**\n- `input.text` (string, required): The text to count characters in",
|
|
1948
|
+
},
|
|
1949
|
+
'char-count-ratio': {
|
|
1950
|
+
slug: 'char-count-ratio',
|
|
1951
|
+
requiredInputFields: ['denominator_text', 'numerator_text'],
|
|
1952
|
+
optionalConfigFields: [],
|
|
1953
|
+
description: "Calculate the ratio of characters between two texts\n\n**Request Body:**\n- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)\n- `input.denominator_text` (string, required): The denominator text (divides the numerator)",
|
|
1954
|
+
},
|
|
1955
|
+
'context-relevance': {
|
|
1956
|
+
slug: 'context-relevance',
|
|
1957
|
+
requiredInputFields: ['context', 'query'],
|
|
1958
|
+
optionalConfigFields: ['model'],
|
|
1959
|
+
description: "Evaluate whether retrieved context contains sufficient information to answer the query\n\n**Request Body:**\n- `input.query` (string, required): The query/question to evaluate context relevance for\n- `input.context` (string, required): The context to evaluate for relevance to the query\n- `config.model` (string, optional): Model to use for evaluation (default: gpt-4o)",
|
|
1960
|
+
},
|
|
1961
|
+
'conversation-quality': {
|
|
1962
|
+
slug: 'conversation-quality',
|
|
1963
|
+
requiredInputFields: ['completions', 'prompts'],
|
|
1964
|
+
optionalConfigFields: [],
|
|
1965
|
+
description: "Evaluate conversation quality based on tone, clarity, flow, responsiveness, and transparency\n\n**Request Body:**\n- `input.prompts` (string, required): JSON array of prompts in the conversation\n- `input.completions` (string, required): JSON array of completions in the conversation",
|
|
1966
|
+
},
|
|
1967
|
+
'faithfulness': {
|
|
1968
|
+
slug: 'faithfulness',
|
|
1969
|
+
requiredInputFields: ['completion', 'context', 'question'],
|
|
1970
|
+
optionalConfigFields: [],
|
|
1971
|
+
description: "Check if a completion is faithful to the provided context\n\n**Request Body:**\n- `input.completion` (string, required): The LLM completion to check for faithfulness\n- `input.context` (string, required): The context that the completion should be faithful to\n- `input.question` (string, required): The original question asked",
|
|
1972
|
+
},
|
|
1973
|
+
'html-comparison': {
|
|
1974
|
+
slug: 'html-comparison',
|
|
1975
|
+
requiredInputFields: ['html1', 'html2'],
|
|
1976
|
+
optionalConfigFields: [],
|
|
1977
|
+
description: "Compare two HTML documents for structural and content similarity\n\n**Request Body:**\n- `input.html1` (string, required): The first HTML document to compare\n- `input.html2` (string, required): The second HTML document to compare",
|
|
1978
|
+
},
|
|
1979
|
+
'instruction-adherence': {
|
|
1980
|
+
slug: 'instruction-adherence',
|
|
1981
|
+
requiredInputFields: ['instructions', 'response'],
|
|
1982
|
+
optionalConfigFields: [],
|
|
1983
|
+
description: "Evaluate how well responses follow given instructions\n\n**Request Body:**\n- `input.instructions` (string, required): The instructions that should be followed\n- `input.response` (string, required): The response to evaluate for instruction adherence",
|
|
1984
|
+
},
|
|
1985
|
+
'intent-change': {
|
|
1986
|
+
slug: 'intent-change',
|
|
1987
|
+
requiredInputFields: ['completions', 'prompts'],
|
|
1988
|
+
optionalConfigFields: [],
|
|
1989
|
+
description: "Detect changes in user intent between prompts and completions\n\n**Request Body:**\n- `input.prompts` (string, required): JSON array of prompts in the conversation\n- `input.completions` (string, required): JSON array of completions in the conversation",
|
|
1990
|
+
},
|
|
1991
|
+
'json-validator': {
|
|
1992
|
+
slug: 'json-validator',
|
|
1993
|
+
requiredInputFields: ['text'],
|
|
1994
|
+
optionalConfigFields: ['enable_schema_validation', 'schema_string'],
|
|
1995
|
+
description: "Validate JSON syntax\n\n**Request Body:**\n- `input.text` (string, required): The text to validate as JSON\n- `config.enable_schema_validation` (bool, optional): Enable JSON schema validation\n- `config.schema_string` (string, optional): JSON schema to validate against",
|
|
1996
|
+
},
|
|
1997
|
+
'perplexity': {
|
|
1998
|
+
slug: 'perplexity',
|
|
1999
|
+
requiredInputFields: ['logprobs'],
|
|
2000
|
+
optionalConfigFields: [],
|
|
2001
|
+
description: "Measure text perplexity from logprobs\n\n**Request Body:**\n- `input.logprobs` (string, required): JSON array of log probabilities from the model",
|
|
2002
|
+
},
|
|
2003
|
+
'pii-detector': {
|
|
2004
|
+
slug: 'pii-detector',
|
|
2005
|
+
requiredInputFields: ['text'],
|
|
2006
|
+
optionalConfigFields: ['probability_threshold'],
|
|
2007
|
+
description: "Detect personally identifiable information in text\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for personally identifiable information\n- `config.probability_threshold` (float, optional): Detection threshold (default: 0.8)",
|
|
2008
|
+
},
|
|
2009
|
+
'placeholder-regex': {
|
|
2010
|
+
slug: 'placeholder-regex',
|
|
2011
|
+
requiredInputFields: ['placeholder_value', 'text'],
|
|
2012
|
+
optionalConfigFields: ['case_sensitive', 'dot_include_nl', 'multi_line', 'should_match'],
|
|
2013
|
+
description: "Validate text against a placeholder regex pattern\n\n**Request Body:**\n- `input.placeholder_value` (string, required): The regex pattern to match against\n- `input.text` (string, required): The text to validate against the regex pattern\n- `config.should_match` (bool, optional): Whether the text should match the regex\n- `config.case_sensitive` (bool, optional): Case-sensitive matching\n- `config.dot_include_nl` (bool, optional): Dot matches newlines\n- `config.multi_line` (bool, optional): Multi-line mode",
|
|
2014
|
+
},
|
|
2015
|
+
'profanity-detector': {
|
|
2016
|
+
slug: 'profanity-detector',
|
|
2017
|
+
requiredInputFields: ['text'],
|
|
2018
|
+
optionalConfigFields: [],
|
|
2019
|
+
description: "Detect profanity in text\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for profanity",
|
|
2020
|
+
},
|
|
2021
|
+
'prompt-injection': {
|
|
2022
|
+
slug: 'prompt-injection',
|
|
2023
|
+
requiredInputFields: ['prompt'],
|
|
2024
|
+
optionalConfigFields: ['threshold'],
|
|
2025
|
+
description: "Detect prompt injection attempts\n\n**Request Body:**\n- `input.prompt` (string, required): The prompt to check for injection attempts\n- `config.threshold` (float, optional): Detection threshold (default: 0.5)",
|
|
2026
|
+
},
|
|
2027
|
+
'prompt-perplexity': {
|
|
2028
|
+
slug: 'prompt-perplexity',
|
|
2029
|
+
requiredInputFields: ['prompt'],
|
|
2030
|
+
optionalConfigFields: [],
|
|
2031
|
+
description: "Measure prompt perplexity to detect potential injection attempts\n\n**Request Body:**\n- `input.prompt` (string, required): The prompt to calculate perplexity for",
|
|
2032
|
+
},
|
|
2033
|
+
'regex-validator': {
|
|
2034
|
+
slug: 'regex-validator',
|
|
2035
|
+
requiredInputFields: ['text'],
|
|
2036
|
+
optionalConfigFields: ['case_sensitive', 'dot_include_nl', 'multi_line', 'regex', 'should_match'],
|
|
2037
|
+
description: "Validate text against a regex pattern\n\n**Request Body:**\n- `input.text` (string, required): The text to validate against a regex pattern\n- `config.regex` (string, optional): The regex pattern to match against\n- `config.should_match` (bool, optional): Whether the text should match the regex\n- `config.case_sensitive` (bool, optional): Case-sensitive matching\n- `config.dot_include_nl` (bool, optional): Dot matches newlines\n- `config.multi_line` (bool, optional): Multi-line mode",
|
|
2038
|
+
},
|
|
2039
|
+
'secrets-detector': {
|
|
2040
|
+
slug: 'secrets-detector',
|
|
2041
|
+
requiredInputFields: ['text'],
|
|
2042
|
+
optionalConfigFields: [],
|
|
2043
|
+
description: "Detect secrets and credentials in text\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for secrets (API keys, passwords, etc.)",
|
|
2044
|
+
},
|
|
2045
|
+
'semantic-similarity': {
|
|
2046
|
+
slug: 'semantic-similarity',
|
|
2047
|
+
requiredInputFields: ['completion', 'reference'],
|
|
2048
|
+
optionalConfigFields: [],
|
|
2049
|
+
description: "Calculate semantic similarity between completion and reference\n\n**Request Body:**\n- `input.completion` (string, required): The completion text to compare\n- `input.reference` (string, required): The reference text to compare against",
|
|
2050
|
+
},
|
|
2051
|
+
'sexism-detector': {
|
|
2052
|
+
slug: 'sexism-detector',
|
|
2053
|
+
requiredInputFields: ['text'],
|
|
2054
|
+
optionalConfigFields: ['threshold'],
|
|
2055
|
+
description: "Detect sexist language and bias\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for sexist content\n- `config.threshold` (float, optional): Detection threshold (default: 0.5)",
|
|
2056
|
+
},
|
|
2057
|
+
'sql-validator': {
|
|
2058
|
+
slug: 'sql-validator',
|
|
2059
|
+
requiredInputFields: ['text'],
|
|
2060
|
+
optionalConfigFields: [],
|
|
2061
|
+
description: "Validate SQL query syntax\n\n**Request Body:**\n- `input.text` (string, required): The text to validate as SQL",
|
|
2062
|
+
},
|
|
2063
|
+
'tone-detection': {
|
|
2064
|
+
slug: 'tone-detection',
|
|
2065
|
+
requiredInputFields: ['text'],
|
|
2066
|
+
optionalConfigFields: [],
|
|
2067
|
+
description: "Detect the tone of the text\n\n**Request Body:**\n- `input.text` (string, required): The text to detect the tone of",
|
|
2068
|
+
},
|
|
2069
|
+
'topic-adherence': {
|
|
2070
|
+
slug: 'topic-adherence',
|
|
2071
|
+
requiredInputFields: ['completion', 'question', 'reference_topics'],
|
|
2072
|
+
optionalConfigFields: [],
|
|
2073
|
+
description: "Evaluate topic adherence\n\n**Request Body:**\n- `input.question` (string, required): The original question\n- `input.completion` (string, required): The completion to evaluate\n- `input.reference_topics` (string, required): Comma-separated list of expected topics",
|
|
2074
|
+
},
|
|
2075
|
+
'toxicity-detector': {
|
|
2076
|
+
slug: 'toxicity-detector',
|
|
2077
|
+
requiredInputFields: ['text'],
|
|
2078
|
+
optionalConfigFields: ['threshold'],
|
|
2079
|
+
description: "Detect toxic or harmful language\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for toxic content\n- `config.threshold` (float, optional): Detection threshold (default: 0.5)",
|
|
2080
|
+
},
|
|
2081
|
+
'uncertainty-detector': {
|
|
2082
|
+
slug: 'uncertainty-detector',
|
|
2083
|
+
requiredInputFields: ['prompt'],
|
|
2084
|
+
optionalConfigFields: [],
|
|
2085
|
+
description: "Detect uncertainty in the text\n\n**Request Body:**\n- `input.prompt` (string, required): The text to detect uncertainty in",
|
|
2086
|
+
},
|
|
2087
|
+
'word-count': {
|
|
2088
|
+
slug: 'word-count',
|
|
2089
|
+
requiredInputFields: ['text'],
|
|
2090
|
+
optionalConfigFields: [],
|
|
2091
|
+
description: "Count the number of words in text\n\n**Request Body:**\n- `input.text` (string, required): The text to count words in",
|
|
2092
|
+
},
|
|
2093
|
+
'word-count-ratio': {
|
|
2094
|
+
slug: 'word-count-ratio',
|
|
2095
|
+
requiredInputFields: ['denominator_text', 'numerator_text'],
|
|
2096
|
+
optionalConfigFields: [],
|
|
2097
|
+
description: "Calculate the ratio of words between two texts\n\n**Request Body:**\n- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)\n- `input.denominator_text` (string, required): The denominator text (divides the numerator)",
|
|
2098
|
+
},
|
|
2099
|
+
};
|
|
2100
|
+
function isValidEvaluatorSlug(slug) {
|
|
2101
|
+
return slug in EVALUATOR_SCHEMAS;
|
|
2102
|
+
}
|
|
2103
|
+
|
|
2104
|
+
// Auto-generated - DO NOT EDIT
|
|
2105
|
+
// Regenerate with: pnpm generate:evaluator-models
|
|
2106
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
2107
|
+
// Utility functions
|
|
2108
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
2109
|
+
/**
|
|
2110
|
+
* Create an evaluator configuration object.
|
|
2111
|
+
*/
|
|
2112
|
+
function createEvaluator(slug, options) {
|
|
2113
|
+
return {
|
|
2114
|
+
name: slug,
|
|
2115
|
+
version: options === null || options === void 0 ? void 0 : options.version,
|
|
2116
|
+
config: options === null || options === void 0 ? void 0 : options.config,
|
|
2117
|
+
};
|
|
2118
|
+
}
|
|
2119
|
+
/**
|
|
2120
|
+
* Validate that required input fields are present in task output.
|
|
2121
|
+
*/
|
|
2122
|
+
function validateEvaluatorInput(slug, taskOutput) {
|
|
2123
|
+
const schema = EVALUATOR_SCHEMAS[slug];
|
|
2124
|
+
if (!schema) {
|
|
2125
|
+
return { valid: false, missingFields: [] };
|
|
2126
|
+
}
|
|
2127
|
+
const missingFields = schema.requiredInputFields.filter((field) => !(field in taskOutput) || taskOutput[field] === undefined);
|
|
2128
|
+
return {
|
|
2129
|
+
valid: missingFields.length === 0,
|
|
2130
|
+
missingFields,
|
|
2131
|
+
};
|
|
2132
|
+
}
|
|
2133
|
+
/**
|
|
2134
|
+
* Get all available evaluator slugs.
|
|
2135
|
+
*/
|
|
2136
|
+
function getAvailableEvaluatorSlugs() {
|
|
2137
|
+
return [...EVALUATOR_SLUGS];
|
|
2138
|
+
}
|
|
2139
|
+
/**
|
|
2140
|
+
* Get schema information for an evaluator.
|
|
2141
|
+
*/
|
|
2142
|
+
function getEvaluatorSchemaInfo(slug) {
|
|
2143
|
+
return EVALUATOR_SCHEMAS[slug];
|
|
2144
|
+
}
|
|
2145
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
2146
|
+
// Factory class
|
|
2147
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
2148
|
+
/**
|
|
2149
|
+
* Factory class for creating type-safe MBT evaluator configurations.
|
|
2150
|
+
*
|
|
2151
|
+
* @example
|
|
2152
|
+
* ```typescript
|
|
2153
|
+
* import { EvaluatorMadeByTraceloop } from '@anyway-sh/node-server-sdk';
|
|
2154
|
+
*
|
|
2155
|
+
* const evaluators = [
|
|
2156
|
+
* EvaluatorMadeByTraceloop.piiDetector({ probability_threshold: 0.8 }),
|
|
2157
|
+
* EvaluatorMadeByTraceloop.faithfulness(),
|
|
2158
|
+
* ];
|
|
2159
|
+
* ```
|
|
2160
|
+
*/
|
|
2161
|
+
class EvaluatorMadeByTraceloop {
|
|
2162
|
+
static create(slug, options) {
|
|
2163
|
+
return createEvaluator(slug, options);
|
|
2164
|
+
}
|
|
2165
|
+
static getAvailableSlugs() {
|
|
2166
|
+
return getAvailableEvaluatorSlugs();
|
|
2167
|
+
}
|
|
2168
|
+
static isValidSlug(slug) {
|
|
2169
|
+
return isValidEvaluatorSlug(slug);
|
|
2170
|
+
}
|
|
2171
|
+
/**
|
|
2172
|
+
* Evaluate agent efficiency - detect redundant calls, unnecessary follow-ups
|
|
2173
|
+
|
|
2174
|
+
**Request Body:**
|
|
2175
|
+
- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
|
|
2176
|
+
- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
|
|
2177
|
+
* Required task output fields: trajectory_completions, trajectory_prompts
|
|
2178
|
+
*/
|
|
2179
|
+
static agentEfficiency() {
|
|
2180
|
+
return createEvaluator('agent-efficiency');
|
|
2181
|
+
}
|
|
2182
|
+
/**
|
|
2183
|
+
* Validate agent trajectory against user-defined conditions
|
|
2184
|
+
|
|
2185
|
+
**Request Body:**
|
|
2186
|
+
- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
|
|
2187
|
+
- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
|
|
2188
|
+
- `config.conditions` (array of strings, required): Array of evaluation conditions/rules to validate against
|
|
2189
|
+
- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)
|
|
2190
|
+
* Required task output fields: trajectory_completions, trajectory_prompts
|
|
2191
|
+
*/
|
|
2192
|
+
static agentFlowQuality(config) {
|
|
2193
|
+
return createEvaluator('agent-flow-quality', { config: config });
|
|
2194
|
+
}
|
|
2195
|
+
/**
|
|
2196
|
+
* Evaluate agent goal accuracy
|
|
2197
|
+
|
|
2198
|
+
**Request Body:**
|
|
2199
|
+
- `input.question` (string, required): The original question or goal
|
|
2200
|
+
- `input.completion` (string, required): The agent's completion/response
|
|
2201
|
+
- `input.reference` (string, required): The expected reference answer
|
|
2202
|
+
* Required task output fields: completion, question, reference
|
|
2203
|
+
*/
|
|
2204
|
+
static agentGoalAccuracy() {
|
|
2205
|
+
return createEvaluator('agent-goal-accuracy');
|
|
2206
|
+
}
|
|
2207
|
+
/**
|
|
2208
|
+
* Measure if agent accomplished all user goals
|
|
2209
|
+
|
|
2210
|
+
**Request Body:**
|
|
2211
|
+
- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory
|
|
2212
|
+
- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory
|
|
2213
|
+
- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)
|
|
2214
|
+
* Required task output fields: trajectory_completions, trajectory_prompts
|
|
2215
|
+
*/
|
|
2216
|
+
static agentGoalCompleteness(config) {
|
|
2217
|
+
return createEvaluator('agent-goal-completeness', { config: config });
|
|
2218
|
+
}
|
|
2219
|
+
/**
|
|
2220
|
+
* Detect errors or failures during tool execution
|
|
2221
|
+
|
|
2222
|
+
**Request Body:**
|
|
2223
|
+
- `input.tool_input` (string, required): JSON string of the tool input
|
|
2224
|
+
- `input.tool_output` (string, required): JSON string of the tool output
|
|
2225
|
+
* Required task output fields: tool_input, tool_output
|
|
2226
|
+
*/
|
|
2227
|
+
static agentToolErrorDetector() {
|
|
2228
|
+
return createEvaluator('agent-tool-error-detector');
|
|
2229
|
+
}
|
|
2230
|
+
/**
|
|
2231
|
+
* Compare actual tool calls against expected reference tool calls
|
|
2232
|
+
|
|
2233
|
+
**Request Body:**
|
|
2234
|
+
- `input.executed_tool_calls` (string, required): JSON array of actual tool calls made by the agent
|
|
2235
|
+
- `input.expected_tool_calls` (string, required): JSON array of expected/reference tool calls
|
|
2236
|
+
- `config.threshold` (float, optional): Score threshold for pass/fail determination (default: 0.5)
|
|
2237
|
+
- `config.mismatch_sensitive` (bool, optional): Whether tool calls must match exactly (default: false)
|
|
2238
|
+
- `config.order_sensitive` (bool, optional): Whether order of tool calls matters (default: false)
|
|
2239
|
+
- `config.input_params_sensitive` (bool, optional): Whether to compare input parameters (default: true)
|
|
2240
|
+
* Required task output fields: executed_tool_calls, expected_tool_calls
|
|
2241
|
+
*/
|
|
2242
|
+
static agentToolTrajectory(config) {
|
|
2243
|
+
return createEvaluator('agent-tool-trajectory', { config: config });
|
|
2244
|
+
}
|
|
2245
|
+
/**
|
|
2246
|
+
* Evaluate whether the answer is complete and contains all the necessary information
|
|
2247
|
+
|
|
2248
|
+
**Request Body:**
|
|
2249
|
+
- `input.question` (string, required): The original question
|
|
2250
|
+
- `input.completion` (string, required): The completion to evaluate for completeness
|
|
2251
|
+
- `input.context` (string, required): The context that provides the complete information
|
|
2252
|
+
* Required task output fields: completion, context, question
|
|
2253
|
+
*/
|
|
2254
|
+
static answerCompleteness() {
|
|
2255
|
+
return createEvaluator('answer-completeness');
|
|
2256
|
+
}
|
|
2257
|
+
/**
|
|
2258
|
+
* Evaluate factual accuracy by comparing answers against ground truth
|
|
2259
|
+
|
|
2260
|
+
**Request Body:**
|
|
2261
|
+
- `input.question` (string, required): The original question
|
|
2262
|
+
- `input.completion` (string, required): The completion to evaluate
|
|
2263
|
+
- `input.ground_truth` (string, required): The expected correct answer
|
|
2264
|
+
* Required task output fields: completion, ground_truth, question
|
|
2265
|
+
*/
|
|
2266
|
+
static answerCorrectness() {
|
|
2267
|
+
return createEvaluator('answer-correctness');
|
|
2268
|
+
}
|
|
2269
|
+
/**
|
|
2270
|
+
* Check if an answer is relevant to a question
|
|
2271
|
+
|
|
2272
|
+
**Request Body:**
|
|
2273
|
+
- `input.answer` (string, required): The answer to evaluate for relevancy
|
|
2274
|
+
- `input.question` (string, required): The question that the answer should be relevant to
|
|
2275
|
+
* Required task output fields: answer, question
|
|
2276
|
+
*/
|
|
2277
|
+
static answerRelevancy() {
|
|
2278
|
+
return createEvaluator('answer-relevancy');
|
|
2279
|
+
}
|
|
2280
|
+
/**
|
|
2281
|
+
* Count the number of characters in text
|
|
2282
|
+
|
|
2283
|
+
**Request Body:**
|
|
2284
|
+
- `input.text` (string, required): The text to count characters in
|
|
2285
|
+
* Required task output fields: text
|
|
2286
|
+
*/
|
|
2287
|
+
static charCount() {
|
|
2288
|
+
return createEvaluator('char-count');
|
|
2289
|
+
}
|
|
2290
|
+
/**
|
|
2291
|
+
* Calculate the ratio of characters between two texts
|
|
2292
|
+
|
|
2293
|
+
**Request Body:**
|
|
2294
|
+
- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)
|
|
2295
|
+
- `input.denominator_text` (string, required): The denominator text (divides the numerator)
|
|
2296
|
+
* Required task output fields: denominator_text, numerator_text
|
|
2297
|
+
*/
|
|
2298
|
+
static charCountRatio() {
|
|
2299
|
+
return createEvaluator('char-count-ratio');
|
|
2300
|
+
}
|
|
2301
|
+
/**
|
|
2302
|
+
* Evaluate whether retrieved context contains sufficient information to answer the query
|
|
2303
|
+
|
|
2304
|
+
**Request Body:**
|
|
2305
|
+
- `input.query` (string, required): The query/question to evaluate context relevance for
|
|
2306
|
+
- `input.context` (string, required): The context to evaluate for relevance to the query
|
|
2307
|
+
- `config.model` (string, optional): Model to use for evaluation (default: gpt-4o)
|
|
2308
|
+
* Required task output fields: context, query
|
|
2309
|
+
*/
|
|
2310
|
+
static contextRelevance(config) {
|
|
2311
|
+
return createEvaluator('context-relevance', { config: config });
|
|
2312
|
+
}
|
|
2313
|
+
/**
|
|
2314
|
+
* Evaluate conversation quality based on tone, clarity, flow, responsiveness, and transparency
|
|
2315
|
+
|
|
2316
|
+
**Request Body:**
|
|
2317
|
+
- `input.prompts` (string, required): JSON array of prompts in the conversation
|
|
2318
|
+
- `input.completions` (string, required): JSON array of completions in the conversation
|
|
2319
|
+
* Required task output fields: completions, prompts
|
|
2320
|
+
*/
|
|
2321
|
+
static conversationQuality() {
|
|
2322
|
+
return createEvaluator('conversation-quality');
|
|
2323
|
+
}
|
|
2324
|
+
/**
|
|
2325
|
+
* Check if a completion is faithful to the provided context
|
|
2326
|
+
|
|
2327
|
+
**Request Body:**
|
|
2328
|
+
- `input.completion` (string, required): The LLM completion to check for faithfulness
|
|
2329
|
+
- `input.context` (string, required): The context that the completion should be faithful to
|
|
2330
|
+
- `input.question` (string, required): The original question asked
|
|
2331
|
+
* Required task output fields: completion, context, question
|
|
2332
|
+
*/
|
|
2333
|
+
static faithfulness() {
|
|
2334
|
+
return createEvaluator('faithfulness');
|
|
2335
|
+
}
|
|
2336
|
+
/**
|
|
2337
|
+
* Compare two HTML documents for structural and content similarity
|
|
2338
|
+
|
|
2339
|
+
**Request Body:**
|
|
2340
|
+
- `input.html1` (string, required): The first HTML document to compare
|
|
2341
|
+
- `input.html2` (string, required): The second HTML document to compare
|
|
2342
|
+
* Required task output fields: html1, html2
|
|
2343
|
+
*/
|
|
2344
|
+
static htmlComparison() {
|
|
2345
|
+
return createEvaluator('html-comparison');
|
|
2346
|
+
}
|
|
2347
|
+
/**
|
|
2348
|
+
* Evaluate how well responses follow given instructions
|
|
2349
|
+
|
|
2350
|
+
**Request Body:**
|
|
2351
|
+
- `input.instructions` (string, required): The instructions that should be followed
|
|
2352
|
+
- `input.response` (string, required): The response to evaluate for instruction adherence
|
|
2353
|
+
* Required task output fields: instructions, response
|
|
2354
|
+
*/
|
|
2355
|
+
static instructionAdherence() {
|
|
2356
|
+
return createEvaluator('instruction-adherence');
|
|
2357
|
+
}
|
|
2358
|
+
/**
|
|
2359
|
+
* Detect changes in user intent between prompts and completions
|
|
2360
|
+
|
|
2361
|
+
**Request Body:**
|
|
2362
|
+
- `input.prompts` (string, required): JSON array of prompts in the conversation
|
|
2363
|
+
- `input.completions` (string, required): JSON array of completions in the conversation
|
|
2364
|
+
* Required task output fields: completions, prompts
|
|
2365
|
+
*/
|
|
2366
|
+
static intentChange() {
|
|
2367
|
+
return createEvaluator('intent-change');
|
|
2368
|
+
}
|
|
2369
|
+
/**
|
|
2370
|
+
* Validate JSON syntax
|
|
2371
|
+
|
|
2372
|
+
**Request Body:**
|
|
2373
|
+
- `input.text` (string, required): The text to validate as JSON
|
|
2374
|
+
- `config.enable_schema_validation` (bool, optional): Enable JSON schema validation
|
|
2375
|
+
- `config.schema_string` (string, optional): JSON schema to validate against
|
|
2376
|
+
* Required task output fields: text
|
|
2377
|
+
*/
|
|
2378
|
+
static jsonValidator(config) {
|
|
2379
|
+
return createEvaluator('json-validator', { config: config });
|
|
2380
|
+
}
|
|
2381
|
+
/**
|
|
2382
|
+
* Measure text perplexity from logprobs
|
|
2383
|
+
|
|
2384
|
+
**Request Body:**
|
|
2385
|
+
- `input.logprobs` (string, required): JSON array of log probabilities from the model
|
|
2386
|
+
* Required task output fields: logprobs
|
|
2387
|
+
*/
|
|
2388
|
+
static perplexity() {
|
|
2389
|
+
return createEvaluator('perplexity');
|
|
2390
|
+
}
|
|
2391
|
+
/**
|
|
2392
|
+
* Detect personally identifiable information in text
|
|
2393
|
+
|
|
2394
|
+
**Request Body:**
|
|
2395
|
+
- `input.text` (string, required): The text to scan for personally identifiable information
|
|
2396
|
+
- `config.probability_threshold` (float, optional): Detection threshold (default: 0.8)
|
|
2397
|
+
* Required task output fields: text
|
|
2398
|
+
*/
|
|
2399
|
+
static piiDetector(config) {
|
|
2400
|
+
return createEvaluator('pii-detector', { config: config });
|
|
2401
|
+
}
|
|
2402
|
+
/**
|
|
2403
|
+
* Validate text against a placeholder regex pattern
|
|
2404
|
+
|
|
2405
|
+
**Request Body:**
|
|
2406
|
+
- `input.placeholder_value` (string, required): The regex pattern to match against
|
|
2407
|
+
- `input.text` (string, required): The text to validate against the regex pattern
|
|
2408
|
+
- `config.should_match` (bool, optional): Whether the text should match the regex
|
|
2409
|
+
- `config.case_sensitive` (bool, optional): Case-sensitive matching
|
|
2410
|
+
- `config.dot_include_nl` (bool, optional): Dot matches newlines
|
|
2411
|
+
- `config.multi_line` (bool, optional): Multi-line mode
|
|
2412
|
+
* Required task output fields: placeholder_value, text
|
|
2413
|
+
*/
|
|
2414
|
+
static placeholderRegex(config) {
|
|
2415
|
+
return createEvaluator('placeholder-regex', { config: config });
|
|
2416
|
+
}
|
|
2417
|
+
/**
|
|
2418
|
+
* Detect profanity in text
|
|
2419
|
+
|
|
2420
|
+
**Request Body:**
|
|
2421
|
+
- `input.text` (string, required): The text to scan for profanity
|
|
2422
|
+
* Required task output fields: text
|
|
2423
|
+
*/
|
|
2424
|
+
static profanityDetector() {
|
|
2425
|
+
return createEvaluator('profanity-detector');
|
|
2426
|
+
}
|
|
2427
|
+
/**
|
|
2428
|
+
* Detect prompt injection attempts
|
|
2429
|
+
|
|
2430
|
+
**Request Body:**
|
|
2431
|
+
- `input.prompt` (string, required): The prompt to check for injection attempts
|
|
2432
|
+
- `config.threshold` (float, optional): Detection threshold (default: 0.5)
|
|
2433
|
+
* Required task output fields: prompt
|
|
2434
|
+
*/
|
|
2435
|
+
static promptInjection(config) {
|
|
2436
|
+
return createEvaluator('prompt-injection', { config: config });
|
|
2437
|
+
}
|
|
2438
|
+
/**
|
|
2439
|
+
* Measure prompt perplexity to detect potential injection attempts
|
|
2440
|
+
|
|
2441
|
+
**Request Body:**
|
|
2442
|
+
- `input.prompt` (string, required): The prompt to calculate perplexity for
|
|
2443
|
+
* Required task output fields: prompt
|
|
2444
|
+
*/
|
|
2445
|
+
static promptPerplexity() {
|
|
2446
|
+
return createEvaluator('prompt-perplexity');
|
|
2447
|
+
}
|
|
2448
|
+
/**
|
|
2449
|
+
* Validate text against a regex pattern
|
|
2450
|
+
|
|
2451
|
+
**Request Body:**
|
|
2452
|
+
- `input.text` (string, required): The text to validate against a regex pattern
|
|
2453
|
+
- `config.regex` (string, optional): The regex pattern to match against
|
|
2454
|
+
- `config.should_match` (bool, optional): Whether the text should match the regex
|
|
2455
|
+
- `config.case_sensitive` (bool, optional): Case-sensitive matching
|
|
2456
|
+
- `config.dot_include_nl` (bool, optional): Dot matches newlines
|
|
2457
|
+
- `config.multi_line` (bool, optional): Multi-line mode
|
|
2458
|
+
* Required task output fields: text
|
|
2459
|
+
*/
|
|
2460
|
+
static regexValidator(config) {
|
|
2461
|
+
return createEvaluator('regex-validator', { config: config });
|
|
2462
|
+
}
|
|
2463
|
+
/**
|
|
2464
|
+
* Detect secrets and credentials in text
|
|
2465
|
+
|
|
2466
|
+
**Request Body:**
|
|
2467
|
+
- `input.text` (string, required): The text to scan for secrets (API keys, passwords, etc.)
|
|
2468
|
+
* Required task output fields: text
|
|
2469
|
+
*/
|
|
2470
|
+
static secretsDetector() {
|
|
2471
|
+
return createEvaluator('secrets-detector');
|
|
2472
|
+
}
|
|
2473
|
+
/**
|
|
2474
|
+
* Calculate semantic similarity between completion and reference
|
|
2475
|
+
|
|
2476
|
+
**Request Body:**
|
|
2477
|
+
- `input.completion` (string, required): The completion text to compare
|
|
2478
|
+
- `input.reference` (string, required): The reference text to compare against
|
|
2479
|
+
* Required task output fields: completion, reference
|
|
2480
|
+
*/
|
|
2481
|
+
static semanticSimilarity() {
|
|
2482
|
+
return createEvaluator('semantic-similarity');
|
|
2483
|
+
}
|
|
2484
|
+
/**
|
|
2485
|
+
* Detect sexist language and bias
|
|
2486
|
+
|
|
2487
|
+
**Request Body:**
|
|
2488
|
+
- `input.text` (string, required): The text to scan for sexist content
|
|
2489
|
+
- `config.threshold` (float, optional): Detection threshold (default: 0.5)
|
|
2490
|
+
* Required task output fields: text
|
|
2491
|
+
*/
|
|
2492
|
+
static sexismDetector(config) {
|
|
2493
|
+
return createEvaluator('sexism-detector', { config: config });
|
|
2494
|
+
}
|
|
2495
|
+
/**
|
|
2496
|
+
* Validate SQL query syntax
|
|
2497
|
+
|
|
2498
|
+
**Request Body:**
|
|
2499
|
+
- `input.text` (string, required): The text to validate as SQL
|
|
2500
|
+
* Required task output fields: text
|
|
2501
|
+
*/
|
|
2502
|
+
static sqlValidator() {
|
|
2503
|
+
return createEvaluator('sql-validator');
|
|
2504
|
+
}
|
|
2505
|
+
/**
|
|
2506
|
+
* Detect the tone of the text
|
|
2507
|
+
|
|
2508
|
+
**Request Body:**
|
|
2509
|
+
- `input.text` (string, required): The text to detect the tone of
|
|
2510
|
+
* Required task output fields: text
|
|
2511
|
+
*/
|
|
2512
|
+
static toneDetection() {
|
|
2513
|
+
return createEvaluator('tone-detection');
|
|
2514
|
+
}
|
|
2515
|
+
/**
|
|
2516
|
+
* Evaluate topic adherence
|
|
2517
|
+
|
|
2518
|
+
**Request Body:**
|
|
2519
|
+
- `input.question` (string, required): The original question
|
|
2520
|
+
- `input.completion` (string, required): The completion to evaluate
|
|
2521
|
+
- `input.reference_topics` (string, required): Comma-separated list of expected topics
|
|
2522
|
+
* Required task output fields: completion, question, reference_topics
|
|
2523
|
+
*/
|
|
2524
|
+
static topicAdherence() {
|
|
2525
|
+
return createEvaluator('topic-adherence');
|
|
2526
|
+
}
|
|
2527
|
+
/**
|
|
2528
|
+
* Detect toxic or harmful language
|
|
2529
|
+
|
|
2530
|
+
**Request Body:**
|
|
2531
|
+
- `input.text` (string, required): The text to scan for toxic content
|
|
2532
|
+
- `config.threshold` (float, optional): Detection threshold (default: 0.5)
|
|
2533
|
+
* Required task output fields: text
|
|
2534
|
+
*/
|
|
2535
|
+
static toxicityDetector(config) {
|
|
2536
|
+
return createEvaluator('toxicity-detector', { config: config });
|
|
2537
|
+
}
|
|
2538
|
+
/**
|
|
2539
|
+
* Detect uncertainty in the text
|
|
2540
|
+
|
|
2541
|
+
**Request Body:**
|
|
2542
|
+
- `input.prompt` (string, required): The text to detect uncertainty in
|
|
2543
|
+
* Required task output fields: prompt
|
|
2544
|
+
*/
|
|
2545
|
+
static uncertaintyDetector() {
|
|
2546
|
+
return createEvaluator('uncertainty-detector');
|
|
2547
|
+
}
|
|
2548
|
+
/**
|
|
2549
|
+
* Count the number of words in text
|
|
2550
|
+
|
|
2551
|
+
**Request Body:**
|
|
2552
|
+
- `input.text` (string, required): The text to count words in
|
|
2553
|
+
* Required task output fields: text
|
|
2554
|
+
*/
|
|
2555
|
+
static wordCount() {
|
|
2556
|
+
return createEvaluator('word-count');
|
|
2557
|
+
}
|
|
2558
|
+
/**
|
|
2559
|
+
* Calculate the ratio of words between two texts
|
|
2560
|
+
|
|
2561
|
+
**Request Body:**
|
|
2562
|
+
- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)
|
|
2563
|
+
- `input.denominator_text` (string, required): The denominator text (divides the numerator)
|
|
2564
|
+
* Required task output fields: denominator_text, numerator_text
|
|
2565
|
+
*/
|
|
2566
|
+
static wordCountRatio() {
|
|
2567
|
+
return createEvaluator('word-count-ratio');
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2570
|
+
|
|
2571
|
+
const validateConfiguration = (options) => {
|
|
2572
|
+
const { apiKey, traceloopSyncEnabled, traceloopSyncMaxRetries, traceloopSyncPollingInterval, traceloopSyncDevPollingInterval, } = options;
|
|
2573
|
+
if (apiKey && typeof apiKey !== "string") {
|
|
2574
|
+
throw new InitializationError('"apiKey" must be a string');
|
|
2575
|
+
}
|
|
2576
|
+
if (traceloopSyncEnabled) {
|
|
2577
|
+
if (typeof traceloopSyncMaxRetries !== "number" ||
|
|
2578
|
+
traceloopSyncMaxRetries <= 0) {
|
|
2579
|
+
throw new InitializationError('"traceloopSyncMaxRetries" must be an integer greater than 0.');
|
|
2580
|
+
}
|
|
2581
|
+
if (typeof traceloopSyncPollingInterval !== "number" ||
|
|
2582
|
+
traceloopSyncPollingInterval <= 0) {
|
|
2583
|
+
throw new InitializationError('"traceloopSyncPollingInterval" must be an integer greater than 0.');
|
|
2584
|
+
}
|
|
2585
|
+
if (typeof traceloopSyncDevPollingInterval !== "number" ||
|
|
2586
|
+
traceloopSyncDevPollingInterval <= 0) {
|
|
2587
|
+
throw new InitializationError('"traceloopSyncDevPollingInterval" must be an integer greater than 0.');
|
|
2588
|
+
}
|
|
2589
|
+
}
|
|
2590
|
+
};
|
|
2591
|
+
|
|
2592
|
+
const TRACER_NAME = "@anyway-sh/node-server-sdk";
|
|
2593
|
+
const TRACER_VERSION = version;
|
|
2594
|
+
const WORKFLOW_NAME_KEY = createContextKey("workflow_name");
|
|
2595
|
+
const ENTITY_NAME_KEY = createContextKey("entity_name");
|
|
2596
|
+
const AGENT_NAME_KEY = createContextKey("agent_name");
|
|
2597
|
+
const CONVERSATION_ID_KEY = createContextKey("conversation_id");
|
|
2598
|
+
const ASSOCATION_PROPERTIES_KEY = createContextKey("association_properties");
|
|
2599
|
+
const getTracer = () => {
|
|
2600
|
+
return trace.getTracer(TRACER_NAME, TRACER_VERSION);
|
|
2601
|
+
};
|
|
2602
|
+
const getTraceloopTracer = getTracer;
|
|
2603
|
+
const getEntityPath = (entityContext) => {
|
|
2604
|
+
const path = entityContext.getValue(ENTITY_NAME_KEY);
|
|
2605
|
+
return path ? `${path}` : undefined;
|
|
2606
|
+
};
|
|
2607
|
+
|
|
2608
|
+
const AI_GENERATE_TEXT = "ai.generateText";
|
|
2609
|
+
const AI_STREAM_TEXT = "ai.streamText";
|
|
2610
|
+
const AI_GENERATE_OBJECT = "ai.generateObject";
|
|
2611
|
+
const AI_STREAM_OBJECT = "ai.streamObject";
|
|
2612
|
+
const AI_GENERATE_TEXT_DO_GENERATE = "ai.generateText.doGenerate";
|
|
2613
|
+
const AI_GENERATE_OBJECT_DO_GENERATE = "ai.generateObject.doGenerate";
|
|
2614
|
+
const AI_STREAM_TEXT_DO_STREAM = "ai.streamText.doStream";
|
|
2615
|
+
const AI_STREAM_OBJECT_DO_STREAM = "ai.streamObject.doStream";
|
|
2616
|
+
const HANDLED_SPAN_NAMES = {
|
|
2617
|
+
[AI_GENERATE_TEXT]: "run.ai",
|
|
2618
|
+
[AI_STREAM_TEXT]: "stream.ai",
|
|
2619
|
+
[AI_GENERATE_OBJECT]: "object.ai",
|
|
2620
|
+
[AI_STREAM_OBJECT]: "stream-object.ai",
|
|
2621
|
+
[AI_GENERATE_TEXT_DO_GENERATE]: "text.generate",
|
|
2622
|
+
[AI_GENERATE_OBJECT_DO_GENERATE]: "object.generate",
|
|
2623
|
+
[AI_STREAM_TEXT_DO_STREAM]: "text.stream",
|
|
2624
|
+
[AI_STREAM_OBJECT_DO_STREAM]: "object.stream",
|
|
2625
|
+
};
|
|
2626
|
+
const TOOL_SPAN_NAME = "ai.toolCall";
|
|
2627
|
+
const AI_RESPONSE_TEXT = "ai.response.text";
|
|
2628
|
+
const AI_RESPONSE_OBJECT = "ai.response.object";
|
|
2629
|
+
const AI_RESPONSE_TOOL_CALLS = "ai.response.toolCalls";
|
|
2630
|
+
const AI_RESPONSE_PROVIDER_METADATA = "ai.response.providerMetadata";
|
|
2631
|
+
const AI_PROMPT_MESSAGES = "ai.prompt.messages";
|
|
2632
|
+
const AI_PROMPT = "ai.prompt";
|
|
2633
|
+
const AI_USAGE_PROMPT_TOKENS = "ai.usage.promptTokens";
|
|
2634
|
+
const AI_USAGE_COMPLETION_TOKENS = "ai.usage.completionTokens";
|
|
2635
|
+
const AI_MODEL_PROVIDER = "ai.model.provider";
|
|
2636
|
+
const AI_PROMPT_TOOLS = "ai.prompt.tools";
|
|
2637
|
+
const AI_TELEMETRY_METADATA_PREFIX = "ai.telemetry.metadata.";
|
|
2638
|
+
const TYPE_TEXT = "text";
|
|
2639
|
+
const TYPE_TOOL_CALL = "tool_call";
|
|
2640
|
+
const ROLE_ASSISTANT = "assistant";
|
|
2641
|
+
const ROLE_USER = "user";
|
|
2642
|
+
// Vendor mapping from AI SDK provider prefixes to standardized LLM_SYSTEM values
|
|
2643
|
+
// Uses prefixes to match AI SDK patterns like "openai.chat", "anthropic.messages", etc.
|
|
2644
|
+
const VENDOR_MAPPING = {
|
|
2645
|
+
openai: "OpenAI",
|
|
2646
|
+
azure: "Azure",
|
|
2647
|
+
"azure-openai": "Azure",
|
|
2648
|
+
anthropic: "Anthropic",
|
|
2649
|
+
cohere: "Cohere",
|
|
2650
|
+
mistral: "MistralAI",
|
|
2651
|
+
groq: "Groq",
|
|
2652
|
+
replicate: "Replicate",
|
|
2653
|
+
together: "TogetherAI",
|
|
2654
|
+
fireworks: "Fireworks",
|
|
2655
|
+
deepseek: "DeepSeek",
|
|
2656
|
+
perplexity: "Perplexity",
|
|
2657
|
+
"amazon-bedrock": "AWS",
|
|
2658
|
+
bedrock: "AWS",
|
|
2659
|
+
google: "Google",
|
|
2660
|
+
vertex: "Google",
|
|
2661
|
+
ollama: "Ollama",
|
|
2662
|
+
huggingface: "HuggingFace",
|
|
2663
|
+
openrouter: "OpenRouter",
|
|
2664
|
+
};
|
|
2665
|
+
const getAgentNameFromAttributes = (attributes) => {
|
|
2666
|
+
const agentAttr = attributes[`${AI_TELEMETRY_METADATA_PREFIX}agent`];
|
|
2667
|
+
return agentAttr && typeof agentAttr === "string" ? agentAttr : null;
|
|
2668
|
+
};
|
|
2669
|
+
const transformResponseText = (attributes) => {
|
|
2670
|
+
if (AI_RESPONSE_TEXT in attributes) {
|
|
2671
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.content`] =
|
|
2672
|
+
attributes[AI_RESPONSE_TEXT];
|
|
2673
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.role`] = ROLE_ASSISTANT;
|
|
2674
|
+
const outputMessage = {
|
|
2675
|
+
role: ROLE_ASSISTANT,
|
|
2676
|
+
parts: [
|
|
2677
|
+
{
|
|
2678
|
+
type: TYPE_TEXT,
|
|
2679
|
+
content: attributes[AI_RESPONSE_TEXT],
|
|
2680
|
+
},
|
|
2681
|
+
],
|
|
2682
|
+
};
|
|
2683
|
+
attributes[ATTR_GEN_AI_OUTPUT_MESSAGES] = JSON.stringify([outputMessage]);
|
|
2684
|
+
delete attributes[AI_RESPONSE_TEXT];
|
|
2685
|
+
}
|
|
2686
|
+
};
|
|
2687
|
+
const transformResponseObject = (attributes) => {
|
|
2688
|
+
if (AI_RESPONSE_OBJECT in attributes) {
|
|
2689
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.content`] =
|
|
2690
|
+
attributes[AI_RESPONSE_OBJECT];
|
|
2691
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.role`] = ROLE_ASSISTANT;
|
|
2692
|
+
const outputMessage = {
|
|
2693
|
+
role: ROLE_ASSISTANT,
|
|
2694
|
+
parts: [
|
|
2695
|
+
{
|
|
2696
|
+
type: TYPE_TEXT,
|
|
2697
|
+
content: attributes[AI_RESPONSE_OBJECT],
|
|
2698
|
+
},
|
|
2699
|
+
],
|
|
2700
|
+
};
|
|
2701
|
+
attributes[ATTR_GEN_AI_OUTPUT_MESSAGES] = JSON.stringify([outputMessage]);
|
|
2702
|
+
delete attributes[AI_RESPONSE_OBJECT];
|
|
2703
|
+
}
|
|
2704
|
+
};
|
|
2705
|
+
const transformResponseToolCalls = (attributes) => {
|
|
2706
|
+
if (AI_RESPONSE_TOOL_CALLS in attributes) {
|
|
2707
|
+
try {
|
|
2708
|
+
const toolCalls = JSON.parse(attributes[AI_RESPONSE_TOOL_CALLS]);
|
|
2709
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.role`] = ROLE_ASSISTANT;
|
|
2710
|
+
const toolCallParts = [];
|
|
2711
|
+
toolCalls.forEach((toolCall, index) => {
|
|
2712
|
+
var _a;
|
|
2713
|
+
// Support both v4 (args) and v5 (input) formats
|
|
2714
|
+
// Prefer v5 (input) if present
|
|
2715
|
+
const toolArgs = (_a = toolCall.input) !== null && _a !== void 0 ? _a : toolCall.args;
|
|
2716
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.tool_calls.${index}.name`] =
|
|
2717
|
+
toolCall.toolName;
|
|
2718
|
+
attributes[`${ATTR_GEN_AI_COMPLETION}.0.tool_calls.${index}.arguments`] = toolArgs;
|
|
2719
|
+
toolCallParts.push({
|
|
2720
|
+
type: TYPE_TOOL_CALL,
|
|
2721
|
+
tool_call: {
|
|
2722
|
+
name: toolCall.toolName,
|
|
2723
|
+
arguments: toolArgs,
|
|
2724
|
+
},
|
|
2725
|
+
});
|
|
2726
|
+
});
|
|
2727
|
+
if (toolCallParts.length > 0) {
|
|
2728
|
+
const outputMessage = {
|
|
2729
|
+
role: ROLE_ASSISTANT,
|
|
2730
|
+
parts: toolCallParts,
|
|
2731
|
+
};
|
|
2732
|
+
attributes[ATTR_GEN_AI_OUTPUT_MESSAGES] = JSON.stringify([
|
|
2733
|
+
outputMessage,
|
|
2734
|
+
]);
|
|
2735
|
+
}
|
|
2736
|
+
delete attributes[AI_RESPONSE_TOOL_CALLS];
|
|
2737
|
+
}
|
|
2738
|
+
catch (_a) {
|
|
2739
|
+
// Ignore parsing errors
|
|
2740
|
+
}
|
|
2741
|
+
}
|
|
2742
|
+
};
|
|
2743
|
+
const processMessageContent = (content) => {
|
|
2744
|
+
if (Array.isArray(content)) {
|
|
2745
|
+
const textItems = content.filter((item) => item &&
|
|
2746
|
+
typeof item === "object" &&
|
|
2747
|
+
item.type === TYPE_TEXT &&
|
|
2748
|
+
item.text);
|
|
2749
|
+
if (textItems.length > 0) {
|
|
2750
|
+
const joinedText = textItems.map((item) => item.text).join(" ");
|
|
2751
|
+
return joinedText;
|
|
2752
|
+
}
|
|
2753
|
+
else {
|
|
2754
|
+
return JSON.stringify(content);
|
|
2755
|
+
}
|
|
2756
|
+
}
|
|
2757
|
+
if (content && typeof content === "object") {
|
|
2758
|
+
if (content.type === TYPE_TEXT && content.text) {
|
|
2759
|
+
return content.text;
|
|
2760
|
+
}
|
|
2761
|
+
return JSON.stringify(content);
|
|
2762
|
+
}
|
|
2763
|
+
if (typeof content === "string") {
|
|
2764
|
+
try {
|
|
2765
|
+
const parsed = JSON.parse(content);
|
|
2766
|
+
if (Array.isArray(parsed)) {
|
|
2767
|
+
const allTextItems = parsed.every((item) => item &&
|
|
2768
|
+
typeof item === "object" &&
|
|
2769
|
+
item.type === TYPE_TEXT &&
|
|
2770
|
+
item.text);
|
|
2771
|
+
if (allTextItems && parsed.length > 0) {
|
|
2772
|
+
return parsed.map((item) => item.text).join(" ");
|
|
2773
|
+
}
|
|
2774
|
+
else {
|
|
2775
|
+
return content;
|
|
2776
|
+
}
|
|
2777
|
+
}
|
|
2778
|
+
}
|
|
2779
|
+
catch (_a) {
|
|
2780
|
+
// Ignore parsing errors
|
|
2781
|
+
}
|
|
2782
|
+
return content;
|
|
2783
|
+
}
|
|
2784
|
+
return String(content);
|
|
2785
|
+
};
|
|
2786
|
+
const transformTools = (attributes) => {
|
|
2787
|
+
if (AI_PROMPT_TOOLS in attributes) {
|
|
2788
|
+
try {
|
|
2789
|
+
const tools = attributes[AI_PROMPT_TOOLS];
|
|
2790
|
+
if (Array.isArray(tools)) {
|
|
2791
|
+
tools.forEach((toolItem, index) => {
|
|
2792
|
+
var _a;
|
|
2793
|
+
let tool = toolItem;
|
|
2794
|
+
if (typeof toolItem === "string") {
|
|
2795
|
+
try {
|
|
2796
|
+
tool = JSON.parse(toolItem);
|
|
2797
|
+
}
|
|
2798
|
+
catch (_b) {
|
|
2799
|
+
return;
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
if (tool && typeof tool === "object") {
|
|
2803
|
+
if (tool.name) {
|
|
2804
|
+
attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.${index}.name`] = tool.name;
|
|
2805
|
+
}
|
|
2806
|
+
if (tool.description) {
|
|
2807
|
+
attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.${index}.description`] = tool.description;
|
|
2808
|
+
}
|
|
2809
|
+
// Support both v4 (parameters) and v5 (inputSchema) formats
|
|
2810
|
+
// Prefer v5 (inputSchema) if present
|
|
2811
|
+
const schema = (_a = tool.inputSchema) !== null && _a !== void 0 ? _a : tool.parameters;
|
|
2812
|
+
if (schema) {
|
|
2813
|
+
attributes[`${SpanAttributes.LLM_REQUEST_FUNCTIONS}.${index}.parameters`] = typeof schema === "string" ? schema : JSON.stringify(schema);
|
|
2814
|
+
}
|
|
2815
|
+
}
|
|
2816
|
+
});
|
|
2817
|
+
}
|
|
2818
|
+
delete attributes[AI_PROMPT_TOOLS];
|
|
2819
|
+
}
|
|
2820
|
+
catch (_a) {
|
|
2821
|
+
// Ignore parsing errors
|
|
2822
|
+
}
|
|
2823
|
+
}
|
|
2824
|
+
};
|
|
2825
|
+
const transformPrompts = (attributes) => {
|
|
2826
|
+
if (AI_PROMPT_MESSAGES in attributes) {
|
|
2827
|
+
try {
|
|
2828
|
+
let jsonString = attributes[AI_PROMPT_MESSAGES];
|
|
2829
|
+
try {
|
|
2830
|
+
JSON.parse(jsonString);
|
|
2831
|
+
}
|
|
2832
|
+
catch (_a) {
|
|
2833
|
+
jsonString = jsonString.replace(/\\'/g, "'");
|
|
2834
|
+
jsonString = jsonString.replace(/\\\\\\\\/g, "\\\\");
|
|
2835
|
+
}
|
|
2836
|
+
const messages = JSON.parse(jsonString);
|
|
2837
|
+
const inputMessages = [];
|
|
2838
|
+
messages.forEach((msg, index) => {
|
|
2839
|
+
const processedContent = processMessageContent(msg.content);
|
|
2840
|
+
const contentKey = `${ATTR_GEN_AI_PROMPT}.${index}.content`;
|
|
2841
|
+
attributes[contentKey] = processedContent;
|
|
2842
|
+
attributes[`${ATTR_GEN_AI_PROMPT}.${index}.role`] = msg.role;
|
|
2843
|
+
// Add to OpenTelemetry standard gen_ai.input.messages format
|
|
2844
|
+
inputMessages.push({
|
|
2845
|
+
role: msg.role,
|
|
2846
|
+
parts: [
|
|
2847
|
+
{
|
|
2848
|
+
type: TYPE_TEXT,
|
|
2849
|
+
content: processedContent,
|
|
2850
|
+
},
|
|
2851
|
+
],
|
|
2852
|
+
});
|
|
2853
|
+
});
|
|
2854
|
+
// Set the OpenTelemetry standard input messages attribute
|
|
2855
|
+
if (inputMessages.length > 0) {
|
|
2856
|
+
attributes[ATTR_GEN_AI_INPUT_MESSAGES] = JSON.stringify(inputMessages);
|
|
2857
|
+
}
|
|
2858
|
+
delete attributes[AI_PROMPT_MESSAGES];
|
|
2859
|
+
}
|
|
2860
|
+
catch (_b) {
|
|
2861
|
+
// Ignore parsing errors
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
if (AI_PROMPT in attributes) {
|
|
2865
|
+
try {
|
|
2866
|
+
const promptData = JSON.parse(attributes[AI_PROMPT]);
|
|
2867
|
+
if (promptData.messages && Array.isArray(promptData.messages)) {
|
|
2868
|
+
const messages = promptData.messages;
|
|
2869
|
+
const inputMessages = [];
|
|
2870
|
+
messages.forEach((msg, index) => {
|
|
2871
|
+
const processedContent = processMessageContent(msg.content);
|
|
2872
|
+
const contentKey = `${ATTR_GEN_AI_PROMPT}.${index}.content`;
|
|
2873
|
+
attributes[contentKey] = processedContent;
|
|
2874
|
+
attributes[`${ATTR_GEN_AI_PROMPT}.${index}.role`] = msg.role;
|
|
2875
|
+
inputMessages.push({
|
|
2876
|
+
role: msg.role,
|
|
2877
|
+
parts: [
|
|
2878
|
+
{
|
|
2879
|
+
type: TYPE_TEXT,
|
|
2880
|
+
content: processedContent,
|
|
2881
|
+
},
|
|
2882
|
+
],
|
|
2883
|
+
});
|
|
2884
|
+
});
|
|
2885
|
+
if (inputMessages.length > 0) {
|
|
2886
|
+
attributes[ATTR_GEN_AI_INPUT_MESSAGES] =
|
|
2887
|
+
JSON.stringify(inputMessages);
|
|
2888
|
+
}
|
|
2889
|
+
delete attributes[AI_PROMPT];
|
|
2890
|
+
}
|
|
2891
|
+
else if (promptData.prompt && typeof promptData.prompt === "string") {
|
|
2892
|
+
attributes[`${ATTR_GEN_AI_PROMPT}.0.content`] = promptData.prompt;
|
|
2893
|
+
attributes[`${ATTR_GEN_AI_PROMPT}.0.role`] = ROLE_USER;
|
|
2894
|
+
const inputMessage = {
|
|
2895
|
+
role: ROLE_USER,
|
|
2896
|
+
parts: [
|
|
2897
|
+
{
|
|
2898
|
+
type: TYPE_TEXT,
|
|
2899
|
+
content: promptData.prompt,
|
|
2900
|
+
},
|
|
2901
|
+
],
|
|
2902
|
+
};
|
|
2903
|
+
attributes[ATTR_GEN_AI_INPUT_MESSAGES] = JSON.stringify([inputMessage]);
|
|
2904
|
+
delete attributes[AI_PROMPT];
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
catch (_c) {
|
|
2908
|
+
// Ignore parsing errors
|
|
2909
|
+
}
|
|
2910
|
+
}
|
|
2911
|
+
};
|
|
2912
|
+
const transformPromptTokens = (attributes) => {
|
|
2913
|
+
if (!(ATTR_GEN_AI_USAGE_INPUT_TOKENS in attributes) &&
|
|
2914
|
+
AI_USAGE_PROMPT_TOKENS in attributes) {
|
|
2915
|
+
attributes[ATTR_GEN_AI_USAGE_INPUT_TOKENS] =
|
|
2916
|
+
attributes[AI_USAGE_PROMPT_TOKENS];
|
|
2917
|
+
}
|
|
2918
|
+
delete attributes[AI_USAGE_PROMPT_TOKENS];
|
|
2919
|
+
};
|
|
2920
|
+
const transformCompletionTokens = (attributes) => {
|
|
2921
|
+
if (!(ATTR_GEN_AI_USAGE_OUTPUT_TOKENS in attributes) &&
|
|
2922
|
+
AI_USAGE_COMPLETION_TOKENS in attributes) {
|
|
2923
|
+
attributes[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS] =
|
|
2924
|
+
attributes[AI_USAGE_COMPLETION_TOKENS];
|
|
2925
|
+
}
|
|
2926
|
+
delete attributes[AI_USAGE_COMPLETION_TOKENS];
|
|
2927
|
+
};
|
|
2928
|
+
const transformProviderMetadata = (attributes) => {
|
|
2929
|
+
if (AI_RESPONSE_PROVIDER_METADATA in attributes) {
|
|
2930
|
+
try {
|
|
2931
|
+
const metadataStr = attributes[AI_RESPONSE_PROVIDER_METADATA];
|
|
2932
|
+
let metadata;
|
|
2933
|
+
if (typeof metadataStr === "string") {
|
|
2934
|
+
metadata = JSON.parse(metadataStr);
|
|
2935
|
+
}
|
|
2936
|
+
else if (typeof metadataStr === "object") {
|
|
2937
|
+
metadata = metadataStr;
|
|
2938
|
+
}
|
|
2939
|
+
else {
|
|
2940
|
+
return;
|
|
2941
|
+
}
|
|
2942
|
+
if (metadata.anthropic) {
|
|
2943
|
+
const anthropicMetadata = metadata.anthropic;
|
|
2944
|
+
if (anthropicMetadata.cacheCreationInputTokens !== undefined) {
|
|
2945
|
+
attributes[SpanAttributes.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] =
|
|
2946
|
+
anthropicMetadata.cacheCreationInputTokens;
|
|
2947
|
+
}
|
|
2948
|
+
if (anthropicMetadata.cacheReadInputTokens !== undefined) {
|
|
2949
|
+
attributes[SpanAttributes.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] =
|
|
2950
|
+
anthropicMetadata.cacheReadInputTokens;
|
|
2951
|
+
}
|
|
2952
|
+
}
|
|
2953
|
+
if (metadata.openai) {
|
|
2954
|
+
const openaiMetadata = metadata.openai;
|
|
2955
|
+
if (openaiMetadata.cachedPromptTokens !== undefined) {
|
|
2956
|
+
attributes[SpanAttributes.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] =
|
|
2957
|
+
openaiMetadata.cachedPromptTokens;
|
|
2958
|
+
}
|
|
2959
|
+
if (openaiMetadata.reasoningTokens !== undefined) {
|
|
2960
|
+
attributes[SpanAttributes.GEN_AI_USAGE_REASONING_TOKENS] =
|
|
2961
|
+
openaiMetadata.reasoningTokens;
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2964
|
+
delete attributes[AI_RESPONSE_PROVIDER_METADATA];
|
|
2965
|
+
}
|
|
2966
|
+
catch (_a) {
|
|
2967
|
+
// Ignore JSON parsing errors
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
};
|
|
2971
|
+
const calculateTotalTokens = (attributes) => {
|
|
2972
|
+
const inputTokens = attributes[ATTR_GEN_AI_USAGE_INPUT_TOKENS];
|
|
2973
|
+
const outputTokens = attributes[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS];
|
|
2974
|
+
if (inputTokens && outputTokens) {
|
|
2975
|
+
attributes[`${SpanAttributes.LLM_USAGE_TOTAL_TOKENS}`] =
|
|
2976
|
+
Number(inputTokens) + Number(outputTokens);
|
|
2977
|
+
}
|
|
2978
|
+
};
|
|
2979
|
+
const transformVendor = (attributes) => {
|
|
2980
|
+
if (AI_MODEL_PROVIDER in attributes) {
|
|
2981
|
+
const vendor = attributes[AI_MODEL_PROVIDER];
|
|
2982
|
+
// Find matching vendor prefix in mapping
|
|
2983
|
+
let mappedVendor = null;
|
|
2984
|
+
let providerName = vendor;
|
|
2985
|
+
if (typeof vendor === "string" && vendor.length > 0) {
|
|
2986
|
+
// Extract provider name (part before first dot, or entire string if no dot)
|
|
2987
|
+
const dotIndex = vendor.indexOf(".");
|
|
2988
|
+
providerName = dotIndex > 0 ? vendor.substring(0, dotIndex) : vendor;
|
|
2989
|
+
for (const prefix of Object.keys(VENDOR_MAPPING)) {
|
|
2990
|
+
if (vendor.startsWith(prefix)) {
|
|
2991
|
+
mappedVendor = VENDOR_MAPPING[prefix];
|
|
2992
|
+
break;
|
|
2993
|
+
}
|
|
2994
|
+
}
|
|
2995
|
+
attributes[ATTR_GEN_AI_SYSTEM] = mappedVendor || vendor;
|
|
2996
|
+
attributes[ATTR_GEN_AI_PROVIDER_NAME] = providerName;
|
|
2997
|
+
}
|
|
2998
|
+
delete attributes[AI_MODEL_PROVIDER];
|
|
2999
|
+
}
|
|
3000
|
+
};
|
|
3001
|
+
const transformOperationName = (attributes, spanName) => {
|
|
3002
|
+
if (!spanName)
|
|
3003
|
+
return;
|
|
3004
|
+
let operationName;
|
|
3005
|
+
if (spanName.includes("generateText") ||
|
|
3006
|
+
spanName.includes("streamText") ||
|
|
3007
|
+
spanName.includes("generateObject") ||
|
|
3008
|
+
spanName.includes("streamObject")) {
|
|
3009
|
+
operationName = "chat";
|
|
3010
|
+
}
|
|
3011
|
+
else if (spanName === "ai.toolCall" || spanName.endsWith(".tool")) {
|
|
3012
|
+
operationName = "execute_tool";
|
|
3013
|
+
}
|
|
3014
|
+
if (operationName) {
|
|
3015
|
+
attributes[ATTR_GEN_AI_OPERATION_NAME] = operationName;
|
|
3016
|
+
}
|
|
3017
|
+
};
|
|
3018
|
+
const transformModelId = (attributes) => {
|
|
3019
|
+
const AI_MODEL_ID = "ai.model.id";
|
|
3020
|
+
if (AI_MODEL_ID in attributes) {
|
|
3021
|
+
attributes[ATTR_GEN_AI_REQUEST_MODEL] = attributes[AI_MODEL_ID];
|
|
3022
|
+
delete attributes[AI_MODEL_ID];
|
|
3023
|
+
}
|
|
3024
|
+
};
|
|
3025
|
+
const transformFinishReason = (attributes) => {
|
|
3026
|
+
const AI_RESPONSE_FINISH_REASON = "ai.response.finishReason";
|
|
3027
|
+
if (AI_RESPONSE_FINISH_REASON in attributes) {
|
|
3028
|
+
const finishReason = attributes[AI_RESPONSE_FINISH_REASON];
|
|
3029
|
+
attributes[ATTR_GEN_AI_RESPONSE_FINISH_REASONS] = Array.isArray(finishReason)
|
|
3030
|
+
? finishReason
|
|
3031
|
+
: [finishReason];
|
|
3032
|
+
delete attributes[AI_RESPONSE_FINISH_REASON];
|
|
3033
|
+
}
|
|
3034
|
+
};
|
|
3035
|
+
const transformToolCallAttributes = (attributes) => {
|
|
3036
|
+
var _a, _b;
|
|
3037
|
+
if ("ai.toolCall.name" in attributes) {
|
|
3038
|
+
attributes[ATTR_GEN_AI_TOOL_NAME] = attributes["ai.toolCall.name"];
|
|
3039
|
+
// Keep ai.toolCall.name for now, will be deleted in transformToolCalls
|
|
3040
|
+
}
|
|
3041
|
+
if ("ai.toolCall.id" in attributes) {
|
|
3042
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_ID] = attributes["ai.toolCall.id"];
|
|
3043
|
+
delete attributes["ai.toolCall.id"];
|
|
3044
|
+
}
|
|
3045
|
+
// Support both v4 (args) and v5 (input) formats
|
|
3046
|
+
// Prefer v5 (input) if present
|
|
3047
|
+
const toolArgs = (_a = attributes["ai.toolCall.input"]) !== null && _a !== void 0 ? _a : attributes["ai.toolCall.args"];
|
|
3048
|
+
if (toolArgs !== undefined) {
|
|
3049
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_ARGUMENTS] = toolArgs;
|
|
3050
|
+
// Don't delete yet - transformToolCalls will handle entity input/output
|
|
3051
|
+
}
|
|
3052
|
+
// Support both v4 (result) and v5 (output) formats
|
|
3053
|
+
// Prefer v5 (output) if present
|
|
3054
|
+
const toolResult = (_b = attributes["ai.toolCall.output"]) !== null && _b !== void 0 ? _b : attributes["ai.toolCall.result"];
|
|
3055
|
+
if (toolResult !== undefined) {
|
|
3056
|
+
attributes[ATTR_GEN_AI_TOOL_CALL_RESULT] = toolResult;
|
|
3057
|
+
// Don't delete yet - transformToolCalls will handle entity input/output
|
|
3058
|
+
}
|
|
3059
|
+
};
|
|
3060
|
+
const transformConversationId = (attributes) => {
|
|
3061
|
+
const conversationId = attributes["ai.telemetry.metadata.conversationId"];
|
|
3062
|
+
const sessionId = attributes["ai.telemetry.metadata.sessionId"];
|
|
3063
|
+
if (conversationId) {
|
|
3064
|
+
attributes[ATTR_GEN_AI_CONVERSATION_ID] = conversationId;
|
|
3065
|
+
}
|
|
3066
|
+
else if (sessionId) {
|
|
3067
|
+
attributes[ATTR_GEN_AI_CONVERSATION_ID] = sessionId;
|
|
3068
|
+
}
|
|
3069
|
+
};
|
|
3070
|
+
const transformResponseMetadata = (attributes) => {
|
|
3071
|
+
const AI_RESPONSE_MODEL = "ai.response.model";
|
|
3072
|
+
const AI_RESPONSE_ID = "ai.response.id";
|
|
3073
|
+
if (AI_RESPONSE_MODEL in attributes) {
|
|
3074
|
+
attributes[ATTR_GEN_AI_RESPONSE_MODEL] = attributes[AI_RESPONSE_MODEL];
|
|
3075
|
+
delete attributes[AI_RESPONSE_MODEL];
|
|
3076
|
+
}
|
|
3077
|
+
if (AI_RESPONSE_ID in attributes) {
|
|
3078
|
+
attributes[ATTR_GEN_AI_RESPONSE_ID] = attributes[AI_RESPONSE_ID];
|
|
3079
|
+
delete attributes[AI_RESPONSE_ID];
|
|
3080
|
+
}
|
|
3081
|
+
};
|
|
3082
|
+
const transformTelemetryMetadata = (attributes, spanName) => {
|
|
3083
|
+
const keysToDelete = [];
|
|
3084
|
+
// Use the helper function to extract agent name
|
|
3085
|
+
const agentName = getAgentNameFromAttributes(attributes);
|
|
3086
|
+
// Find all ai.telemetry.metadata.* attributes
|
|
3087
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
3088
|
+
if (key.startsWith(AI_TELEMETRY_METADATA_PREFIX)) {
|
|
3089
|
+
const metadataKey = key.substring(AI_TELEMETRY_METADATA_PREFIX.length);
|
|
3090
|
+
// Always mark for deletion since it's a telemetry metadata attribute
|
|
3091
|
+
keysToDelete.push(key);
|
|
3092
|
+
if (metadataKey && value != null) {
|
|
3093
|
+
// Convert value to string for association properties
|
|
3094
|
+
const stringValue = typeof value === "string" ? value : String(value);
|
|
3095
|
+
// Also set as traceloop association property attribute
|
|
3096
|
+
attributes[`${SpanAttributes.TRACELOOP_ASSOCIATION_PROPERTIES}.${metadataKey}`] = stringValue;
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
}
|
|
3100
|
+
if (agentName) {
|
|
3101
|
+
attributes[ATTR_GEN_AI_AGENT_NAME] = agentName;
|
|
3102
|
+
const topLevelSpanNames = [
|
|
3103
|
+
AI_GENERATE_TEXT,
|
|
3104
|
+
AI_STREAM_TEXT,
|
|
3105
|
+
AI_GENERATE_OBJECT,
|
|
3106
|
+
AI_STREAM_OBJECT,
|
|
3107
|
+
];
|
|
3108
|
+
if (spanName &&
|
|
3109
|
+
(spanName === `${agentName}.agent` ||
|
|
3110
|
+
topLevelSpanNames.includes(spanName))) {
|
|
3111
|
+
attributes[SpanAttributes.TRACELOOP_SPAN_KIND] =
|
|
3112
|
+
TraceloopSpanKindValues.AGENT;
|
|
3113
|
+
attributes[SpanAttributes.TRACELOOP_ENTITY_NAME] = agentName;
|
|
3114
|
+
const inputMessages = attributes[ATTR_GEN_AI_INPUT_MESSAGES];
|
|
3115
|
+
const outputMessages = attributes[ATTR_GEN_AI_OUTPUT_MESSAGES];
|
|
3116
|
+
const toolArgs = attributes["ai.toolCall.args"];
|
|
3117
|
+
const toolResult = attributes["ai.toolCall.result"];
|
|
3118
|
+
if (inputMessages || outputMessages) {
|
|
3119
|
+
if (inputMessages) {
|
|
3120
|
+
attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT] = inputMessages;
|
|
3121
|
+
}
|
|
3122
|
+
if (outputMessages) {
|
|
3123
|
+
attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT] = outputMessages;
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
else if (toolArgs || toolResult) {
|
|
3127
|
+
if (toolArgs) {
|
|
3128
|
+
attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT] = toolArgs;
|
|
3129
|
+
}
|
|
3130
|
+
if (toolResult) {
|
|
3131
|
+
attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT] = toolResult;
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
keysToDelete.forEach((key) => {
|
|
3137
|
+
delete attributes[key];
|
|
3138
|
+
});
|
|
3139
|
+
};
|
|
3140
|
+
const transformLLMSpans = (attributes, spanName) => {
|
|
3141
|
+
transformOperationName(attributes, spanName);
|
|
3142
|
+
transformModelId(attributes);
|
|
3143
|
+
transformResponseText(attributes);
|
|
3144
|
+
transformResponseObject(attributes);
|
|
3145
|
+
transformResponseToolCalls(attributes);
|
|
3146
|
+
transformPrompts(attributes);
|
|
3147
|
+
transformTools(attributes);
|
|
3148
|
+
transformPromptTokens(attributes);
|
|
3149
|
+
transformCompletionTokens(attributes);
|
|
3150
|
+
transformProviderMetadata(attributes);
|
|
3151
|
+
transformFinishReason(attributes);
|
|
3152
|
+
transformResponseMetadata(attributes);
|
|
3153
|
+
calculateTotalTokens(attributes);
|
|
3154
|
+
transformVendor(attributes); // Also sets GEN_AI_PROVIDER_NAME
|
|
3155
|
+
transformConversationId(attributes);
|
|
3156
|
+
transformToolCallAttributes(attributes);
|
|
3157
|
+
transformTelemetryMetadata(attributes, spanName);
|
|
3158
|
+
};
|
|
3159
|
+
const transformToolCalls = (span) => {
|
|
3160
|
+
var _a, _b;
|
|
3161
|
+
// Support both v4 (args/result) and v5 (input/output) formats
|
|
3162
|
+
// Prefer v5 (input/output) if present
|
|
3163
|
+
const toolInput = (_a = span.attributes["ai.toolCall.input"]) !== null && _a !== void 0 ? _a : span.attributes["ai.toolCall.args"];
|
|
3164
|
+
const toolOutput = (_b = span.attributes["ai.toolCall.output"]) !== null && _b !== void 0 ? _b : span.attributes["ai.toolCall.result"];
|
|
3165
|
+
if (toolInput && toolOutput) {
|
|
3166
|
+
span.attributes[SpanAttributes.TRACELOOP_ENTITY_INPUT] = toolInput;
|
|
3167
|
+
delete span.attributes["ai.toolCall.args"];
|
|
3168
|
+
delete span.attributes["ai.toolCall.input"];
|
|
3169
|
+
span.attributes[SpanAttributes.TRACELOOP_ENTITY_OUTPUT] = toolOutput;
|
|
3170
|
+
delete span.attributes["ai.toolCall.result"];
|
|
3171
|
+
delete span.attributes["ai.toolCall.output"];
|
|
3172
|
+
span.attributes[SpanAttributes.TRACELOOP_SPAN_KIND] =
|
|
3173
|
+
TraceloopSpanKindValues.TOOL;
|
|
3174
|
+
// Set entity name from tool call name
|
|
3175
|
+
const toolName = span.attributes["ai.toolCall.name"];
|
|
3176
|
+
if (toolName) {
|
|
3177
|
+
span.attributes[SpanAttributes.TRACELOOP_ENTITY_NAME] = toolName;
|
|
3178
|
+
delete span.attributes["ai.toolCall.name"];
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
};
|
|
3182
|
+
const shouldHandleSpan = (span) => {
|
|
3183
|
+
var _a;
|
|
3184
|
+
return ((_a = span.instrumentationScope) === null || _a === void 0 ? void 0 : _a.name) === "ai";
|
|
3185
|
+
};
|
|
3186
|
+
const getAiSdkVersion = () => {
|
|
3187
|
+
try {
|
|
3188
|
+
return require("ai/package.json").version;
|
|
3189
|
+
}
|
|
3190
|
+
catch (_a) {
|
|
3191
|
+
return undefined;
|
|
3192
|
+
}
|
|
3193
|
+
};
|
|
3194
|
+
const TOP_LEVEL_AI_SPANS = [
|
|
3195
|
+
AI_GENERATE_TEXT,
|
|
3196
|
+
AI_STREAM_TEXT,
|
|
3197
|
+
AI_GENERATE_OBJECT,
|
|
3198
|
+
AI_STREAM_OBJECT,
|
|
3199
|
+
];
|
|
3200
|
+
const transformAiSdkSpanNames = (span) => {
|
|
3201
|
+
if (span.name === TOOL_SPAN_NAME) {
|
|
3202
|
+
span.updateName(`${span.attributes["ai.toolCall.name"]}.tool`);
|
|
3203
|
+
}
|
|
3204
|
+
if (span.name in HANDLED_SPAN_NAMES) {
|
|
3205
|
+
const agentName = getAgentNameFromAttributes(span.attributes);
|
|
3206
|
+
const isTopLevelSpan = TOP_LEVEL_AI_SPANS.includes(span.name);
|
|
3207
|
+
if (agentName && isTopLevelSpan) {
|
|
3208
|
+
span.updateName(`${agentName}.agent`);
|
|
3209
|
+
}
|
|
3210
|
+
else if (!isTopLevelSpan) {
|
|
3211
|
+
span.updateName(HANDLED_SPAN_NAMES[span.name]);
|
|
3212
|
+
}
|
|
3213
|
+
}
|
|
3214
|
+
};
|
|
3215
|
+
const transformAiSdkSpanAttributes = (span) => {
|
|
3216
|
+
if (!shouldHandleSpan(span)) {
|
|
3217
|
+
return;
|
|
3218
|
+
}
|
|
3219
|
+
const aiSdkVersion = getAiSdkVersion();
|
|
3220
|
+
if (aiSdkVersion) {
|
|
3221
|
+
span.attributes["ai.sdk.version"] = aiSdkVersion;
|
|
3222
|
+
}
|
|
3223
|
+
transformLLMSpans(span.attributes, span.name);
|
|
3224
|
+
transformToolCalls(span);
|
|
3225
|
+
};
|
|
3226
|
+
|
|
3227
|
+
function parseKeyPairsIntoRecord(keyPairs) {
|
|
3228
|
+
const result = {};
|
|
3229
|
+
if (!keyPairs)
|
|
3230
|
+
return result;
|
|
3231
|
+
keyPairs.split(",").forEach((pair) => {
|
|
3232
|
+
const [key, value] = pair.split("=");
|
|
3233
|
+
if (key && value) {
|
|
3234
|
+
result[key.trim()] = value.trim();
|
|
3235
|
+
}
|
|
3236
|
+
});
|
|
3237
|
+
return result;
|
|
3238
|
+
}
|
|
3239
|
+
|
|
3240
|
+
const ALL_INSTRUMENTATION_LIBRARIES = "all";
|
|
3241
|
+
const spanAgentNames = new Map();
|
|
3242
|
+
const SPAN_AGENT_NAME_TTL = 5 * 60 * 1000;
|
|
3243
|
+
const AI_TELEMETRY_METADATA_AGENT = "ai.telemetry.metadata.agent";
|
|
3244
|
+
const cleanupExpiredSpanAgentNames = () => {
|
|
3245
|
+
const now = Date.now();
|
|
3246
|
+
for (const [spanId, entry] of spanAgentNames.entries()) {
|
|
3247
|
+
if (now - entry.timestamp > SPAN_AGENT_NAME_TTL) {
|
|
3248
|
+
spanAgentNames.delete(spanId);
|
|
3249
|
+
}
|
|
3250
|
+
}
|
|
3251
|
+
};
|
|
3252
|
+
/**
|
|
3253
|
+
* Creates a span processor with Traceloop's custom span handling logic.
|
|
3254
|
+
* This can be used independently of the full SDK initialization.
|
|
3255
|
+
*
|
|
3256
|
+
* @param options - Configuration options for the span processor
|
|
3257
|
+
* @returns A configured SpanProcessor instance
|
|
3258
|
+
*/
|
|
3259
|
+
const createSpanProcessor = (options) => {
|
|
3260
|
+
var _a;
|
|
3261
|
+
const url = `${options.baseUrl || process.env.ANYWAY_BASE_URL || "https://api.traceloop.com"}/v1/traces`;
|
|
3262
|
+
const headers = options.headers ||
|
|
3263
|
+
(process.env.ANYWAY_HEADERS
|
|
3264
|
+
? parseKeyPairsIntoRecord(process.env.ANYWAY_HEADERS)
|
|
3265
|
+
: { Authorization: `Bearer ${options.apiKey}` });
|
|
3266
|
+
const traceExporter = (_a = options.exporter) !== null && _a !== void 0 ? _a : new OTLPTraceExporter({
|
|
3267
|
+
url,
|
|
3268
|
+
headers,
|
|
3269
|
+
});
|
|
3270
|
+
const spanProcessor = options.disableBatch
|
|
3271
|
+
? new SimpleSpanProcessor(traceExporter)
|
|
3272
|
+
: new BatchSpanProcessor(traceExporter);
|
|
3273
|
+
// Store the original onEnd method
|
|
3274
|
+
const originalOnEnd = spanProcessor.onEnd.bind(spanProcessor);
|
|
3275
|
+
spanProcessor.onStart = onSpanStart;
|
|
3276
|
+
if (options.allowedInstrumentationLibraries === ALL_INSTRUMENTATION_LIBRARIES) {
|
|
3277
|
+
spanProcessor.onEnd = onSpanEnd(originalOnEnd);
|
|
3278
|
+
}
|
|
3279
|
+
else {
|
|
3280
|
+
const instrumentationLibraries = [...traceloopInstrumentationLibraries];
|
|
3281
|
+
if (options.allowedInstrumentationLibraries) {
|
|
3282
|
+
instrumentationLibraries.push(...options.allowedInstrumentationLibraries);
|
|
3283
|
+
}
|
|
3284
|
+
spanProcessor.onEnd = onSpanEnd(originalOnEnd, instrumentationLibraries);
|
|
3285
|
+
}
|
|
3286
|
+
return spanProcessor;
|
|
3287
|
+
};
|
|
3288
|
+
const traceloopInstrumentationLibraries = [
|
|
3289
|
+
"ai",
|
|
3290
|
+
"@anyway-sh/node-server-sdk",
|
|
3291
|
+
"@traceloop/instrumentation-openai",
|
|
3292
|
+
"@traceloop/instrumentation-langchain",
|
|
3293
|
+
"@traceloop/instrumentation-chroma",
|
|
3294
|
+
"@traceloop/instrumentation-anthropic",
|
|
3295
|
+
"@traceloop/instrumentation-llamaindex",
|
|
3296
|
+
"@traceloop/instrumentation-vertexai",
|
|
3297
|
+
"@traceloop/instrumentation-bedrock",
|
|
3298
|
+
"@traceloop/instrumentation-cohere",
|
|
3299
|
+
"@traceloop/instrumentation-pinecone",
|
|
3300
|
+
"@traceloop/instrumentation-qdrant",
|
|
3301
|
+
"@traceloop/instrumentation-together",
|
|
3302
|
+
"@traceloop/instrumentation-mcp",
|
|
3303
|
+
];
|
|
3304
|
+
const onSpanStart = (span) => {
|
|
3305
|
+
const workflowName = context.active().getValue(WORKFLOW_NAME_KEY);
|
|
3306
|
+
if (workflowName) {
|
|
3307
|
+
span.setAttribute(SpanAttributes.TRACELOOP_WORKFLOW_NAME, workflowName);
|
|
3308
|
+
}
|
|
3309
|
+
const entityName = context.active().getValue(ENTITY_NAME_KEY);
|
|
3310
|
+
if (entityName) {
|
|
3311
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_PATH, entityName);
|
|
3312
|
+
}
|
|
3313
|
+
let agentName = context.active().getValue(AGENT_NAME_KEY);
|
|
3314
|
+
if (!agentName) {
|
|
3315
|
+
const aiSdkAgent = span.attributes[AI_TELEMETRY_METADATA_AGENT];
|
|
3316
|
+
if (aiSdkAgent && typeof aiSdkAgent === "string") {
|
|
3317
|
+
agentName = aiSdkAgent;
|
|
3318
|
+
}
|
|
3319
|
+
}
|
|
3320
|
+
if (!agentName) {
|
|
3321
|
+
const parentSpanContext = span.parentSpanContext;
|
|
3322
|
+
const parentSpanId = parentSpanContext === null || parentSpanContext === void 0 ? void 0 : parentSpanContext.spanId;
|
|
3323
|
+
if (parentSpanId &&
|
|
3324
|
+
parentSpanId !== "0000000000000000" &&
|
|
3325
|
+
spanAgentNames.has(parentSpanId)) {
|
|
3326
|
+
agentName = spanAgentNames.get(parentSpanId).agentName;
|
|
3327
|
+
}
|
|
3328
|
+
}
|
|
3329
|
+
if (agentName) {
|
|
3330
|
+
span.setAttribute(ATTR_GEN_AI_AGENT_NAME, agentName);
|
|
3331
|
+
const spanId = span.spanContext().spanId;
|
|
3332
|
+
spanAgentNames.set(spanId, { agentName, timestamp: Date.now() });
|
|
3333
|
+
}
|
|
3334
|
+
// Check for conversation ID in context
|
|
3335
|
+
const conversationId = context.active().getValue(CONVERSATION_ID_KEY);
|
|
3336
|
+
if (conversationId) {
|
|
3337
|
+
span.setAttribute(ATTR_GEN_AI_CONVERSATION_ID, conversationId);
|
|
3338
|
+
}
|
|
3339
|
+
// Check for association properties in context (set by decorators or withAssociationProperties)
|
|
3340
|
+
const associationProperties = context
|
|
3341
|
+
.active()
|
|
3342
|
+
.getValue(ASSOCATION_PROPERTIES_KEY);
|
|
3343
|
+
if (associationProperties && Object.keys(associationProperties).length > 0) {
|
|
3344
|
+
for (const [key, value] of Object.entries(associationProperties)) {
|
|
3345
|
+
span.setAttribute(`${SpanAttributes.TRACELOOP_ASSOCIATION_PROPERTIES}.${key}`, value);
|
|
3346
|
+
}
|
|
3347
|
+
}
|
|
3348
|
+
transformAiSdkSpanNames(span);
|
|
3349
|
+
};
|
|
3350
|
+
/**
|
|
3351
|
+
* Ensures span compatibility between OTel v1.x and v2.x for OTLP transformer
|
|
3352
|
+
*/
|
|
3353
|
+
const ensureSpanCompatibility = (span) => {
|
|
3354
|
+
const spanAny = span;
|
|
3355
|
+
// If the span already has instrumentationLibrary, it's compatible (OTel v2.x)
|
|
3356
|
+
if (spanAny.instrumentationLibrary) {
|
|
3357
|
+
return span;
|
|
3358
|
+
}
|
|
3359
|
+
// If it has instrumentationScope but no instrumentationLibrary (OTel v1.x),
|
|
3360
|
+
// add instrumentationLibrary as an alias to prevent OTLP transformer errors
|
|
3361
|
+
if (spanAny.instrumentationScope) {
|
|
3362
|
+
// Create a proxy that provides both properties
|
|
3363
|
+
return new Proxy(span, {
|
|
3364
|
+
get(target, prop) {
|
|
3365
|
+
if (prop === "instrumentationLibrary") {
|
|
3366
|
+
return target.instrumentationScope;
|
|
3367
|
+
}
|
|
3368
|
+
return target[prop];
|
|
3369
|
+
},
|
|
3370
|
+
});
|
|
3371
|
+
}
|
|
3372
|
+
// Fallback: add both properties with defaults
|
|
3373
|
+
return new Proxy(span, {
|
|
3374
|
+
get(target, prop) {
|
|
3375
|
+
if (prop === "instrumentationLibrary" ||
|
|
3376
|
+
prop === "instrumentationScope") {
|
|
3377
|
+
return {
|
|
3378
|
+
name: "unknown",
|
|
3379
|
+
version: undefined,
|
|
3380
|
+
schemaUrl: undefined,
|
|
3381
|
+
};
|
|
3382
|
+
}
|
|
3383
|
+
return target[prop];
|
|
3384
|
+
},
|
|
3385
|
+
});
|
|
3386
|
+
};
|
|
3387
|
+
const onSpanEnd = (originalOnEnd, instrumentationLibraries) => {
|
|
3388
|
+
return (span) => {
|
|
3389
|
+
var _a, _b, _c;
|
|
3390
|
+
if (instrumentationLibraries &&
|
|
3391
|
+
!instrumentationLibraries.includes(((_a = span.instrumentationScope) === null || _a === void 0 ? void 0 : _a.name) ||
|
|
3392
|
+
((_b = span.instrumentationLibrary) === null || _b === void 0 ? void 0 : _b.name))) {
|
|
3393
|
+
return;
|
|
3394
|
+
}
|
|
3395
|
+
transformAiSdkSpanAttributes(span);
|
|
3396
|
+
const spanId = span.spanContext().spanId;
|
|
3397
|
+
const parentSpanId = (_c = span.parentSpanContext) === null || _c === void 0 ? void 0 : _c.spanId;
|
|
3398
|
+
let agentName = span.attributes[ATTR_GEN_AI_AGENT_NAME];
|
|
3399
|
+
if (agentName && typeof agentName === "string") {
|
|
3400
|
+
spanAgentNames.set(spanId, {
|
|
3401
|
+
agentName,
|
|
3402
|
+
timestamp: Date.now(),
|
|
3403
|
+
});
|
|
3404
|
+
}
|
|
3405
|
+
else if (parentSpanId &&
|
|
3406
|
+
parentSpanId !== "0000000000000000" &&
|
|
3407
|
+
spanAgentNames.has(parentSpanId)) {
|
|
3408
|
+
agentName = spanAgentNames.get(parentSpanId).agentName;
|
|
3409
|
+
span.attributes[ATTR_GEN_AI_AGENT_NAME] = agentName;
|
|
3410
|
+
spanAgentNames.set(spanId, {
|
|
3411
|
+
agentName,
|
|
3412
|
+
timestamp: Date.now(),
|
|
3413
|
+
});
|
|
3414
|
+
}
|
|
3415
|
+
if (Math.random() < 0.01) {
|
|
3416
|
+
cleanupExpiredSpanAgentNames();
|
|
3417
|
+
}
|
|
3418
|
+
const compatibleSpan = ensureSpanCompatibility(span);
|
|
3419
|
+
originalOnEnd(compatibleSpan);
|
|
3420
|
+
};
|
|
3421
|
+
};
|
|
3422
|
+
|
|
3423
|
+
class ImageUploader {
|
|
3424
|
+
constructor(baseUrl, apiKey) {
|
|
3425
|
+
this.baseUrl = baseUrl;
|
|
3426
|
+
this.apiKey = apiKey;
|
|
3427
|
+
}
|
|
3428
|
+
async uploadBase64Image(traceId, spanId, imageName, base64ImageData) {
|
|
3429
|
+
try {
|
|
3430
|
+
const imageUrl = await this.getImageUrl(traceId, spanId, imageName);
|
|
3431
|
+
await this.uploadImageData(imageUrl, base64ImageData);
|
|
3432
|
+
return imageUrl;
|
|
3433
|
+
}
|
|
3434
|
+
catch (error) {
|
|
3435
|
+
console.error("Failed to upload image:", error);
|
|
3436
|
+
throw error;
|
|
3437
|
+
}
|
|
3438
|
+
}
|
|
3439
|
+
async getImageUrl(traceId, spanId, imageName) {
|
|
3440
|
+
const response = await fetch(`${this.baseUrl}/v2/traces/${traceId}/spans/${spanId}/images`, {
|
|
3441
|
+
method: "POST",
|
|
3442
|
+
headers: {
|
|
3443
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
3444
|
+
"Content-Type": "application/json",
|
|
3445
|
+
},
|
|
3446
|
+
body: JSON.stringify({
|
|
3447
|
+
image_name: imageName,
|
|
3448
|
+
}),
|
|
3449
|
+
});
|
|
3450
|
+
if (!response.ok) {
|
|
3451
|
+
throw new Error(`Failed to get image URL: ${response.status} ${response.statusText}`);
|
|
3452
|
+
}
|
|
3453
|
+
const result = await response.json();
|
|
3454
|
+
return result.url;
|
|
3455
|
+
}
|
|
3456
|
+
async uploadImageData(url, base64ImageData) {
|
|
3457
|
+
const payload = {
|
|
3458
|
+
image_data: base64ImageData,
|
|
3459
|
+
};
|
|
3460
|
+
const response = await fetch(url, {
|
|
3461
|
+
method: "POST",
|
|
3462
|
+
headers: {
|
|
3463
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
3464
|
+
"Content-Type": "application/json",
|
|
3465
|
+
},
|
|
3466
|
+
body: JSON.stringify(payload),
|
|
3467
|
+
});
|
|
3468
|
+
if (!response.ok) {
|
|
3469
|
+
const errorText = await response.text();
|
|
3470
|
+
throw new Error(`Failed to upload image data: ${response.status} ${response.statusText}. ${errorText}`);
|
|
3471
|
+
}
|
|
3472
|
+
}
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3475
|
+
let _sdk;
|
|
3476
|
+
let _spanProcessor;
|
|
3477
|
+
let openAIInstrumentation;
|
|
3478
|
+
let anthropicInstrumentation;
|
|
3479
|
+
let cohereInstrumentation;
|
|
3480
|
+
let vertexaiInstrumentation;
|
|
3481
|
+
let aiplatformInstrumentation;
|
|
3482
|
+
let bedrockInstrumentation;
|
|
3483
|
+
let langchainInstrumentation;
|
|
3484
|
+
let llamaIndexInstrumentation;
|
|
3485
|
+
let pineconeInstrumentation;
|
|
3486
|
+
let chromadbInstrumentation;
|
|
3487
|
+
let qdrantInstrumentation;
|
|
3488
|
+
let togetherInstrumentation;
|
|
3489
|
+
let mcpInstrumentation;
|
|
3490
|
+
const instrumentations = [];
|
|
3491
|
+
const initInstrumentations = (apiKey, baseUrl) => {
|
|
3492
|
+
const exceptionLogger = (e) => {
|
|
3493
|
+
console.debug("[Traceloop] Instrumentation exception:", e.message);
|
|
3494
|
+
};
|
|
3495
|
+
const enrichTokens = (process.env.ANYWAY_ENRICH_TOKENS || "true").toLowerCase() === "true";
|
|
3496
|
+
// Create image upload callback if we have credentials
|
|
3497
|
+
let uploadBase64ImageCallback;
|
|
3498
|
+
if (apiKey && baseUrl) {
|
|
3499
|
+
const imageUploader = new ImageUploader(baseUrl, apiKey);
|
|
3500
|
+
uploadBase64ImageCallback =
|
|
3501
|
+
imageUploader.uploadBase64Image.bind(imageUploader);
|
|
3502
|
+
}
|
|
3503
|
+
// Create or update OpenAI instrumentation
|
|
3504
|
+
if (openAIInstrumentation) {
|
|
3505
|
+
// Update existing instrumentation with new callback
|
|
3506
|
+
openAIInstrumentation.setConfig({
|
|
3507
|
+
enrichTokens,
|
|
3508
|
+
exceptionLogger,
|
|
3509
|
+
uploadBase64Image: uploadBase64ImageCallback,
|
|
3510
|
+
});
|
|
3511
|
+
}
|
|
3512
|
+
else {
|
|
3513
|
+
// Create new instrumentation
|
|
3514
|
+
openAIInstrumentation = new OpenAIInstrumentation({
|
|
3515
|
+
enrichTokens,
|
|
3516
|
+
exceptionLogger,
|
|
3517
|
+
uploadBase64Image: uploadBase64ImageCallback,
|
|
3518
|
+
});
|
|
3519
|
+
instrumentations.push(openAIInstrumentation);
|
|
3520
|
+
}
|
|
3521
|
+
if (!anthropicInstrumentation) {
|
|
3522
|
+
anthropicInstrumentation = new AnthropicInstrumentation({
|
|
3523
|
+
exceptionLogger,
|
|
3524
|
+
});
|
|
3525
|
+
instrumentations.push(anthropicInstrumentation);
|
|
3526
|
+
}
|
|
3527
|
+
cohereInstrumentation = new CohereInstrumentation({ exceptionLogger });
|
|
3528
|
+
instrumentations.push(cohereInstrumentation);
|
|
3529
|
+
vertexaiInstrumentation = new VertexAIInstrumentation({
|
|
3530
|
+
exceptionLogger,
|
|
3531
|
+
});
|
|
3532
|
+
instrumentations.push(vertexaiInstrumentation);
|
|
3533
|
+
aiplatformInstrumentation = new AIPlatformInstrumentation({
|
|
3534
|
+
exceptionLogger,
|
|
3535
|
+
});
|
|
3536
|
+
instrumentations.push(aiplatformInstrumentation);
|
|
3537
|
+
bedrockInstrumentation = new BedrockInstrumentation({ exceptionLogger });
|
|
3538
|
+
instrumentations.push(bedrockInstrumentation);
|
|
3539
|
+
pineconeInstrumentation = new PineconeInstrumentation({ exceptionLogger });
|
|
3540
|
+
instrumentations.push(pineconeInstrumentation);
|
|
3541
|
+
langchainInstrumentation = new LangChainInstrumentation({ exceptionLogger });
|
|
3542
|
+
instrumentations.push(langchainInstrumentation);
|
|
3543
|
+
llamaIndexInstrumentation = new LlamaIndexInstrumentation({
|
|
3544
|
+
exceptionLogger,
|
|
3545
|
+
});
|
|
3546
|
+
instrumentations.push(llamaIndexInstrumentation);
|
|
3547
|
+
chromadbInstrumentation = new ChromaDBInstrumentation({ exceptionLogger });
|
|
3548
|
+
instrumentations.push(chromadbInstrumentation);
|
|
3549
|
+
qdrantInstrumentation = new QdrantInstrumentation({ exceptionLogger });
|
|
3550
|
+
instrumentations.push(qdrantInstrumentation);
|
|
3551
|
+
togetherInstrumentation = new TogetherInstrumentation({ exceptionLogger });
|
|
3552
|
+
instrumentations.push(togetherInstrumentation);
|
|
3553
|
+
mcpInstrumentation = new McpInstrumentation({ exceptionLogger });
|
|
3554
|
+
instrumentations.push(mcpInstrumentation);
|
|
3555
|
+
};
|
|
3556
|
+
const manuallyInitInstrumentations = (instrumentModules, apiKey, baseUrl) => {
|
|
3557
|
+
const exceptionLogger = (e) => {
|
|
3558
|
+
console.debug("[Traceloop] Instrumentation exception:", e.message);
|
|
3559
|
+
};
|
|
3560
|
+
const enrichTokens = (process.env.ANYWAY_ENRICH_TOKENS || "true").toLowerCase() === "true";
|
|
3561
|
+
// Create image upload callback if we have credentials
|
|
3562
|
+
let uploadBase64ImageCallback;
|
|
3563
|
+
if (apiKey && baseUrl) {
|
|
3564
|
+
const imageUploader = new ImageUploader(baseUrl, apiKey);
|
|
3565
|
+
uploadBase64ImageCallback =
|
|
3566
|
+
imageUploader.uploadBase64Image.bind(imageUploader);
|
|
3567
|
+
}
|
|
3568
|
+
// Clear the instrumentations array that was initialized by default
|
|
3569
|
+
instrumentations.length = 0;
|
|
3570
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.openAI) {
|
|
3571
|
+
openAIInstrumentation = new OpenAIInstrumentation({
|
|
3572
|
+
enrichTokens,
|
|
3573
|
+
exceptionLogger,
|
|
3574
|
+
uploadBase64Image: uploadBase64ImageCallback,
|
|
3575
|
+
});
|
|
3576
|
+
instrumentations.push(openAIInstrumentation);
|
|
3577
|
+
openAIInstrumentation.manuallyInstrument(instrumentModules.openAI);
|
|
3578
|
+
}
|
|
3579
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.anthropic) {
|
|
3580
|
+
anthropicInstrumentation = new AnthropicInstrumentation({
|
|
3581
|
+
exceptionLogger,
|
|
3582
|
+
});
|
|
3583
|
+
instrumentations.push(anthropicInstrumentation);
|
|
3584
|
+
anthropicInstrumentation.manuallyInstrument(instrumentModules.anthropic);
|
|
3585
|
+
}
|
|
3586
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.cohere) {
|
|
3587
|
+
cohereInstrumentation = new CohereInstrumentation({ exceptionLogger });
|
|
3588
|
+
instrumentations.push(cohereInstrumentation);
|
|
3589
|
+
cohereInstrumentation.manuallyInstrument(instrumentModules.cohere);
|
|
3590
|
+
}
|
|
3591
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.google_vertexai) {
|
|
3592
|
+
vertexaiInstrumentation = new VertexAIInstrumentation({
|
|
3593
|
+
exceptionLogger,
|
|
3594
|
+
});
|
|
3595
|
+
instrumentations.push(vertexaiInstrumentation);
|
|
3596
|
+
vertexaiInstrumentation.manuallyInstrument(instrumentModules.google_vertexai);
|
|
3597
|
+
}
|
|
3598
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.google_aiplatform) {
|
|
3599
|
+
aiplatformInstrumentation = new AIPlatformInstrumentation({
|
|
3600
|
+
exceptionLogger,
|
|
3601
|
+
});
|
|
3602
|
+
instrumentations.push(aiplatformInstrumentation);
|
|
3603
|
+
aiplatformInstrumentation.manuallyInstrument(instrumentModules.google_aiplatform);
|
|
3604
|
+
}
|
|
3605
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.bedrock) {
|
|
3606
|
+
bedrockInstrumentation = new BedrockInstrumentation({ exceptionLogger });
|
|
3607
|
+
instrumentations.push(bedrockInstrumentation);
|
|
3608
|
+
bedrockInstrumentation.manuallyInstrument(instrumentModules.bedrock);
|
|
3609
|
+
}
|
|
3610
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.pinecone) {
|
|
3611
|
+
const instrumentation = new PineconeInstrumentation({ exceptionLogger });
|
|
3612
|
+
instrumentations.push(instrumentation);
|
|
3613
|
+
instrumentation.manuallyInstrument(instrumentModules.pinecone);
|
|
3614
|
+
}
|
|
3615
|
+
// Always enable LangChain instrumentation
|
|
3616
|
+
langchainInstrumentation = new LangChainInstrumentation({
|
|
3617
|
+
exceptionLogger,
|
|
3618
|
+
});
|
|
3619
|
+
instrumentations.push(langchainInstrumentation);
|
|
3620
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.llamaIndex) {
|
|
3621
|
+
llamaIndexInstrumentation = new LlamaIndexInstrumentation({
|
|
3622
|
+
exceptionLogger,
|
|
3623
|
+
});
|
|
3624
|
+
instrumentations.push(llamaIndexInstrumentation);
|
|
3625
|
+
llamaIndexInstrumentation.manuallyInstrument(instrumentModules.llamaIndex);
|
|
3626
|
+
}
|
|
3627
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.chromadb) {
|
|
3628
|
+
chromadbInstrumentation = new ChromaDBInstrumentation({ exceptionLogger });
|
|
3629
|
+
instrumentations.push(chromadbInstrumentation);
|
|
3630
|
+
chromadbInstrumentation.manuallyInstrument(instrumentModules.chromadb);
|
|
3631
|
+
}
|
|
3632
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.qdrant) {
|
|
3633
|
+
qdrantInstrumentation = new QdrantInstrumentation({ exceptionLogger });
|
|
3634
|
+
instrumentations.push(qdrantInstrumentation);
|
|
3635
|
+
qdrantInstrumentation.manuallyInstrument(instrumentModules.qdrant);
|
|
3636
|
+
}
|
|
3637
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.together) {
|
|
3638
|
+
togetherInstrumentation = new TogetherInstrumentation({ exceptionLogger });
|
|
3639
|
+
instrumentations.push(togetherInstrumentation);
|
|
3640
|
+
togetherInstrumentation.manuallyInstrument(instrumentModules.together);
|
|
3641
|
+
}
|
|
3642
|
+
if (instrumentModules === null || instrumentModules === void 0 ? void 0 : instrumentModules.mcp) {
|
|
3643
|
+
mcpInstrumentation = new McpInstrumentation({ exceptionLogger });
|
|
3644
|
+
instrumentations.push(mcpInstrumentation);
|
|
3645
|
+
mcpInstrumentation.manuallyInstrument(instrumentModules.mcp);
|
|
3646
|
+
}
|
|
3647
|
+
};
|
|
3648
|
+
/**
|
|
3649
|
+
* Initializes the Traceloop SDK.
|
|
3650
|
+
* Must be called once before any other SDK methods.
|
|
3651
|
+
*
|
|
3652
|
+
* @param options - The options to initialize the SDK. See the {@link InitializeOptions} for details.
|
|
3653
|
+
* @throws {InitializationError} if the configuration is invalid or if failed to fetch feature data.
|
|
3654
|
+
*/
|
|
3655
|
+
const startTracing = (options) => {
|
|
3656
|
+
var _a;
|
|
3657
|
+
const apiKey = options.apiKey || process.env.ANYWAY_API_KEY;
|
|
3658
|
+
const baseUrl = options.baseUrl ||
|
|
3659
|
+
process.env.ANYWAY_BASE_URL ||
|
|
3660
|
+
"https://api.traceloop.com";
|
|
3661
|
+
if (Object.keys(options.instrumentModules || {}).length > 0) {
|
|
3662
|
+
manuallyInitInstrumentations(options.instrumentModules, apiKey, baseUrl);
|
|
3663
|
+
}
|
|
3664
|
+
else {
|
|
3665
|
+
// Initialize default instrumentations if no manual modules specified
|
|
3666
|
+
initInstrumentations(apiKey, baseUrl);
|
|
3667
|
+
}
|
|
3668
|
+
if (!shouldSendTraces()) {
|
|
3669
|
+
openAIInstrumentation === null || openAIInstrumentation === void 0 ? void 0 : openAIInstrumentation.setConfig({
|
|
3670
|
+
traceContent: false,
|
|
3671
|
+
});
|
|
3672
|
+
llamaIndexInstrumentation === null || llamaIndexInstrumentation === void 0 ? void 0 : llamaIndexInstrumentation.setConfig({
|
|
3673
|
+
traceContent: false,
|
|
3674
|
+
});
|
|
3675
|
+
vertexaiInstrumentation === null || vertexaiInstrumentation === void 0 ? void 0 : vertexaiInstrumentation.setConfig({
|
|
3676
|
+
traceContent: false,
|
|
3677
|
+
});
|
|
3678
|
+
aiplatformInstrumentation === null || aiplatformInstrumentation === void 0 ? void 0 : aiplatformInstrumentation.setConfig({
|
|
3679
|
+
traceContent: false,
|
|
3680
|
+
});
|
|
3681
|
+
bedrockInstrumentation === null || bedrockInstrumentation === void 0 ? void 0 : bedrockInstrumentation.setConfig({
|
|
3682
|
+
traceContent: false,
|
|
3683
|
+
});
|
|
3684
|
+
cohereInstrumentation === null || cohereInstrumentation === void 0 ? void 0 : cohereInstrumentation.setConfig({
|
|
3685
|
+
traceContent: false,
|
|
3686
|
+
});
|
|
3687
|
+
chromadbInstrumentation === null || chromadbInstrumentation === void 0 ? void 0 : chromadbInstrumentation.setConfig({
|
|
3688
|
+
traceContent: false,
|
|
3689
|
+
});
|
|
3690
|
+
togetherInstrumentation === null || togetherInstrumentation === void 0 ? void 0 : togetherInstrumentation.setConfig({
|
|
3691
|
+
traceContent: false,
|
|
3692
|
+
});
|
|
3693
|
+
}
|
|
3694
|
+
const headers = options.headers ||
|
|
3695
|
+
(process.env.ANYWAY_HEADERS
|
|
3696
|
+
? parseKeyPairsIntoRecord(process.env.ANYWAY_HEADERS)
|
|
3697
|
+
: { Authorization: `Bearer ${options.apiKey}` });
|
|
3698
|
+
const traceExporter = (_a = options.exporter) !== null && _a !== void 0 ? _a : (options.gcpProjectId
|
|
3699
|
+
? new TraceExporter({ projectId: options.gcpProjectId })
|
|
3700
|
+
: new OTLPTraceExporter({
|
|
3701
|
+
url: `${baseUrl}/v1/traces`,
|
|
3702
|
+
headers,
|
|
3703
|
+
}));
|
|
3704
|
+
_spanProcessor = createSpanProcessor({
|
|
3705
|
+
apiKey: options.apiKey,
|
|
3706
|
+
baseUrl: options.baseUrl,
|
|
3707
|
+
disableBatch: options.disableBatch,
|
|
3708
|
+
exporter: traceExporter,
|
|
3709
|
+
headers,
|
|
3710
|
+
allowedInstrumentationLibraries: ALL_INSTRUMENTATION_LIBRARIES,
|
|
3711
|
+
});
|
|
3712
|
+
const spanProcessors = [_spanProcessor];
|
|
3713
|
+
if (options.processor) {
|
|
3714
|
+
spanProcessors.push(options.processor);
|
|
3715
|
+
}
|
|
3716
|
+
const resource = createResource({
|
|
3717
|
+
[ATTR_SERVICE_NAME]: options.appName || process.env.npm_package_name || "unknown_service",
|
|
3718
|
+
});
|
|
3719
|
+
_sdk = new NodeSDK({
|
|
3720
|
+
resource,
|
|
3721
|
+
spanProcessors,
|
|
3722
|
+
contextManager: options.contextManager,
|
|
3723
|
+
textMapPropagator: options.propagator,
|
|
3724
|
+
traceExporter,
|
|
3725
|
+
instrumentations,
|
|
3726
|
+
// We should re-consider removing irrelevant spans here in the future
|
|
3727
|
+
// sampler: new TraceloopSampler(),
|
|
3728
|
+
});
|
|
3729
|
+
_sdk.start();
|
|
3730
|
+
};
|
|
3731
|
+
const shouldSendTraces = () => {
|
|
3732
|
+
if (!_configuration) {
|
|
3733
|
+
diag.warn("Traceloop not initialized");
|
|
3734
|
+
return false;
|
|
3735
|
+
}
|
|
3736
|
+
const contextShouldSendPrompts = context
|
|
3737
|
+
.active()
|
|
3738
|
+
.getValue(CONTEXT_KEY_ALLOW_TRACE_CONTENT);
|
|
3739
|
+
if (contextShouldSendPrompts !== undefined) {
|
|
3740
|
+
return contextShouldSendPrompts;
|
|
3741
|
+
}
|
|
3742
|
+
if (_configuration.traceContent === false ||
|
|
3743
|
+
(process.env.ANYWAY_TRACE_CONTENT || "true").toLowerCase() === "false") {
|
|
3744
|
+
return false;
|
|
3745
|
+
}
|
|
3746
|
+
return true;
|
|
3747
|
+
};
|
|
3748
|
+
const forceFlush = async () => {
|
|
3749
|
+
await _spanProcessor.forceFlush();
|
|
3750
|
+
};
|
|
3751
|
+
// Compatibility function for creating resources that works with both OTel v1.x and v2.x
|
|
3752
|
+
function createResource(attributes) {
|
|
3753
|
+
// Import the resource module at runtime to handle both v1.x and v2.x
|
|
3754
|
+
const resourcesModule = require("@opentelemetry/resources");
|
|
3755
|
+
// Try to use resourceFromAttributes if it exists (OTel v2.x)
|
|
3756
|
+
if (resourcesModule.resourceFromAttributes) {
|
|
3757
|
+
return resourcesModule.resourceFromAttributes(attributes);
|
|
3758
|
+
}
|
|
3759
|
+
// Fallback to constructor for OTel v1.x
|
|
3760
|
+
return new resourcesModule.Resource(attributes);
|
|
3761
|
+
}
|
|
3762
|
+
|
|
3763
|
+
const fetchRetry = fetchBuilder(fetch$1);
|
|
3764
|
+
const fetchPrompts = async (options) => {
|
|
3765
|
+
const { apiKey, baseUrl, traceloopSyncMaxRetries } = options;
|
|
3766
|
+
const response = await fetchRetry(`${baseUrl}/v1/traceloop/prompts`, {
|
|
3767
|
+
method: "GET",
|
|
3768
|
+
headers: {
|
|
3769
|
+
"Content-Type": "application/json",
|
|
3770
|
+
Authorization: `Bearer ${apiKey}`,
|
|
3771
|
+
"X-Traceloop-SDK-Version": "0.0.30",
|
|
3772
|
+
},
|
|
3773
|
+
retries: traceloopSyncMaxRetries,
|
|
3774
|
+
retryOn: function (attempt, error, response) {
|
|
3775
|
+
if (attempt >= traceloopSyncMaxRetries)
|
|
3776
|
+
return false;
|
|
3777
|
+
if ((response === null || response === void 0 ? void 0 : response.status) && response.status >= 500) {
|
|
3778
|
+
return true;
|
|
3779
|
+
}
|
|
3780
|
+
return false;
|
|
3781
|
+
},
|
|
3782
|
+
retryDelay: function (attempt) {
|
|
3783
|
+
return Math.pow(2, attempt) * 1000; // 1000, 2000, 4000
|
|
3784
|
+
},
|
|
3785
|
+
});
|
|
3786
|
+
return await response.json();
|
|
3787
|
+
};
|
|
3788
|
+
|
|
3789
|
+
const _prompts = {};
|
|
3790
|
+
let _initialized = false;
|
|
3791
|
+
let _initializedPromise;
|
|
3792
|
+
/**
|
|
3793
|
+
* Returns true once SDK prompt registry has been initialized, else rejects with an error.
|
|
3794
|
+
* @returns Promise<boolean>
|
|
3795
|
+
*/
|
|
3796
|
+
const waitForInitialization = async () => {
|
|
3797
|
+
if (_initialized) {
|
|
3798
|
+
return true;
|
|
3799
|
+
}
|
|
3800
|
+
return await _initializedPromise;
|
|
3801
|
+
};
|
|
3802
|
+
const getPromptByKey = (key) => {
|
|
3803
|
+
if (!_prompts[key]) {
|
|
3804
|
+
throw new PromptNotFoundError(key);
|
|
3805
|
+
}
|
|
3806
|
+
return _prompts[key];
|
|
3807
|
+
};
|
|
3808
|
+
const populateRegistry = (prompts) => {
|
|
3809
|
+
prompts === null || prompts === void 0 ? void 0 : prompts.forEach((prompt) => {
|
|
3810
|
+
_prompts[prompt.key] = prompt;
|
|
3811
|
+
});
|
|
3812
|
+
};
|
|
3813
|
+
const initializeRegistry = (options) => {
|
|
3814
|
+
const { baseUrl, traceloopSyncEnabled, traceloopSyncPollingInterval, traceloopSyncDevPollingInterval, } = options;
|
|
3815
|
+
if (!traceloopSyncEnabled || !(baseUrl === null || baseUrl === void 0 ? void 0 : baseUrl.includes("traceloop")))
|
|
3816
|
+
return;
|
|
3817
|
+
let pollingInterval = traceloopSyncPollingInterval;
|
|
3818
|
+
_initializedPromise = fetchPrompts(options)
|
|
3819
|
+
.then(({ prompts, environment }) => {
|
|
3820
|
+
if (environment === "dev") {
|
|
3821
|
+
pollingInterval = traceloopSyncDevPollingInterval;
|
|
3822
|
+
}
|
|
3823
|
+
populateRegistry(prompts);
|
|
3824
|
+
_initialized = true;
|
|
3825
|
+
setInterval(async () => {
|
|
3826
|
+
try {
|
|
3827
|
+
const { prompts } = await fetchPrompts(options);
|
|
3828
|
+
populateRegistry(prompts);
|
|
3829
|
+
}
|
|
3830
|
+
catch (err) {
|
|
3831
|
+
diag.error("Failed to fetch prompt data", err);
|
|
3832
|
+
}
|
|
3833
|
+
}, pollingInterval * 1000).unref();
|
|
3834
|
+
return true;
|
|
3835
|
+
})
|
|
3836
|
+
.catch((e) => {
|
|
3837
|
+
throw new InitializationError("Failed to fetch prompt data to initialize Traceloop SDK", e);
|
|
3838
|
+
});
|
|
3839
|
+
};
|
|
3840
|
+
|
|
3841
|
+
let _configuration;
|
|
3842
|
+
let _client;
|
|
3843
|
+
/**
|
|
3844
|
+
* Initializes the Traceloop SDK and creates a singleton client instance if API key is provided.
|
|
3845
|
+
* Must be called once before any other SDK methods.
|
|
3846
|
+
*
|
|
3847
|
+
* @param options - The options to initialize the SDK. See the {@link InitializeOptions} for details.
|
|
3848
|
+
* @returns TraceloopClient - The singleton client instance if API key is provided, otherwise undefined.
|
|
3849
|
+
* @throws {InitializationError} if the configuration is invalid or if failed to fetch feature data.
|
|
3850
|
+
*
|
|
3851
|
+
* @example
|
|
3852
|
+
* ```typescript
|
|
3853
|
+
* initialize({
|
|
3854
|
+
* apiKey: 'your-api-key',
|
|
3855
|
+
* appName: 'your-app',
|
|
3856
|
+
* });
|
|
3857
|
+
* ```
|
|
3858
|
+
*/
|
|
3859
|
+
const initialize = (options = {}) => {
|
|
3860
|
+
if (_configuration) {
|
|
3861
|
+
return;
|
|
3862
|
+
}
|
|
3863
|
+
if (!options.baseUrl) {
|
|
3864
|
+
options.baseUrl =
|
|
3865
|
+
process.env.ANYWAY_BASE_URL || "https://api.traceloop.com";
|
|
3866
|
+
}
|
|
3867
|
+
if (!options.apiKey) {
|
|
3868
|
+
options.apiKey = process.env.ANYWAY_API_KEY;
|
|
3869
|
+
}
|
|
3870
|
+
if (!options.appName) {
|
|
3871
|
+
options.appName = process.env.npm_package_name;
|
|
3872
|
+
}
|
|
3873
|
+
if (!options.experimentSlug) {
|
|
3874
|
+
options.experimentSlug = process.env.ANYWAY_EXP_SLUG;
|
|
3875
|
+
}
|
|
3876
|
+
if (options.traceloopSyncEnabled === undefined) {
|
|
3877
|
+
if (process.env.ANYWAY_SYNC_ENABLED !== undefined) {
|
|
3878
|
+
options.traceloopSyncEnabled = ["1", "true"].includes(process.env.ANYWAY_SYNC_ENABLED.toLowerCase());
|
|
3879
|
+
}
|
|
3880
|
+
else {
|
|
3881
|
+
options.traceloopSyncEnabled = true;
|
|
3882
|
+
}
|
|
3883
|
+
}
|
|
3884
|
+
if (options.traceloopSyncEnabled) {
|
|
3885
|
+
if (!options.traceloopSyncMaxRetries) {
|
|
3886
|
+
options.traceloopSyncMaxRetries =
|
|
3887
|
+
Number(process.env.ANYWAY_SYNC_MAX_RETRIES) || 3;
|
|
3888
|
+
}
|
|
3889
|
+
if (!options.traceloopSyncPollingInterval) {
|
|
3890
|
+
options.traceloopSyncPollingInterval =
|
|
3891
|
+
Number(process.env.ANYWAY_SYNC_POLLING_INTERVAL) || 60;
|
|
3892
|
+
}
|
|
3893
|
+
if (!options.traceloopSyncDevPollingInterval) {
|
|
3894
|
+
options.traceloopSyncDevPollingInterval =
|
|
3895
|
+
Number(process.env.ANYWAY_SYNC_DEV_POLLING_INTERVAL) || 5;
|
|
3896
|
+
}
|
|
3897
|
+
}
|
|
3898
|
+
validateConfiguration(options);
|
|
3899
|
+
_configuration = Object.freeze(options);
|
|
3900
|
+
if (!options.silenceInitializationMessage) {
|
|
3901
|
+
console.log(`Traceloop exporting traces to ${_configuration.exporter ? "a custom exporter" : _configuration.baseUrl}`);
|
|
3902
|
+
}
|
|
3903
|
+
if (options.tracingEnabled === undefined || options.tracingEnabled) {
|
|
3904
|
+
if (options.logLevel) {
|
|
3905
|
+
diag.setLogger(new DiagConsoleLogger(), logLevelToOtelLogLevel(options.logLevel));
|
|
3906
|
+
}
|
|
3907
|
+
startTracing(_configuration);
|
|
3908
|
+
}
|
|
3909
|
+
initializeRegistry(_configuration);
|
|
3910
|
+
if (options.apiKey) {
|
|
3911
|
+
_client = new TraceloopClient({
|
|
3912
|
+
apiKey: options.apiKey,
|
|
3913
|
+
baseUrl: options.baseUrl,
|
|
3914
|
+
appName: options.appName,
|
|
3915
|
+
experimentSlug: options.experimentSlug,
|
|
3916
|
+
});
|
|
3917
|
+
return _client;
|
|
3918
|
+
}
|
|
3919
|
+
return;
|
|
3920
|
+
};
|
|
3921
|
+
const logLevelToOtelLogLevel = (logLevel) => {
|
|
3922
|
+
switch (logLevel) {
|
|
3923
|
+
case "debug":
|
|
3924
|
+
return DiagLogLevel.DEBUG;
|
|
3925
|
+
case "info":
|
|
3926
|
+
return DiagLogLevel.INFO;
|
|
3927
|
+
case "warn":
|
|
3928
|
+
return DiagLogLevel.WARN;
|
|
3929
|
+
case "error":
|
|
3930
|
+
return DiagLogLevel.ERROR;
|
|
3931
|
+
}
|
|
3932
|
+
};
|
|
3933
|
+
/**
|
|
3934
|
+
* Gets the singleton instance of the TraceloopClient.
|
|
3935
|
+
* The SDK must be initialized with an API key before calling this function.
|
|
3936
|
+
*
|
|
3937
|
+
* @returns The TraceloopClient singleton instance
|
|
3938
|
+
* @throws {Error} if the SDK hasn't been initialized or was initialized without an API key
|
|
3939
|
+
*
|
|
3940
|
+
* @example
|
|
3941
|
+
* ```typescript
|
|
3942
|
+
* const client = getClient();
|
|
3943
|
+
* await client.annotation.create({ annotationTask: 'taskId', entityInstanceId: 'entityId', tags: { score: 0.9 } });
|
|
3944
|
+
* ```
|
|
3945
|
+
*/
|
|
3946
|
+
const getClient = () => {
|
|
3947
|
+
if (!_client) {
|
|
3948
|
+
throw new Error("Traceloop must be initialized before getting client, Call initialize() first." +
|
|
3949
|
+
"If you already called initialize(), make sure you have an api key.");
|
|
3950
|
+
}
|
|
3951
|
+
return _client;
|
|
3952
|
+
};
|
|
3953
|
+
|
|
3954
|
+
function withEntity(type, { name, version, associationProperties, conversationId, traceContent: overrideTraceContent, inputParameters, suppressTracing: shouldSuppressTracing, }, fn, thisArg, ...args) {
|
|
3955
|
+
let entityContext = context.active();
|
|
3956
|
+
if (type === TraceloopSpanKindValues.WORKFLOW ||
|
|
3957
|
+
type === TraceloopSpanKindValues.AGENT) {
|
|
3958
|
+
entityContext = entityContext.setValue(WORKFLOW_NAME_KEY, name);
|
|
3959
|
+
}
|
|
3960
|
+
if (type === TraceloopSpanKindValues.AGENT) {
|
|
3961
|
+
entityContext = entityContext.setValue(AGENT_NAME_KEY, name);
|
|
3962
|
+
}
|
|
3963
|
+
const entityPath = getEntityPath(entityContext);
|
|
3964
|
+
if (type === TraceloopSpanKindValues.TOOL ||
|
|
3965
|
+
type === TraceloopSpanKindValues.TASK) {
|
|
3966
|
+
const fullEntityName = entityPath ? `${entityPath}.${name}` : name;
|
|
3967
|
+
entityContext = entityContext.setValue(ENTITY_NAME_KEY, fullEntityName);
|
|
3968
|
+
}
|
|
3969
|
+
if (overrideTraceContent != undefined) {
|
|
3970
|
+
entityContext = entityContext.setValue(CONTEXT_KEY_ALLOW_TRACE_CONTENT, overrideTraceContent);
|
|
3971
|
+
}
|
|
3972
|
+
if (associationProperties) {
|
|
3973
|
+
entityContext = entityContext.setValue(ASSOCATION_PROPERTIES_KEY, associationProperties);
|
|
3974
|
+
}
|
|
3975
|
+
if (conversationId) {
|
|
3976
|
+
entityContext = entityContext.setValue(CONVERSATION_ID_KEY, conversationId);
|
|
3977
|
+
}
|
|
3978
|
+
if (shouldSuppressTracing) {
|
|
3979
|
+
entityContext = suppressTracing(entityContext);
|
|
3980
|
+
}
|
|
3981
|
+
return context.with(entityContext, () => getTracer().startActiveSpan(`${name}.${type}`, {}, entityContext, async (span) => {
|
|
3982
|
+
if (type === TraceloopSpanKindValues.WORKFLOW ||
|
|
3983
|
+
type === TraceloopSpanKindValues.AGENT) {
|
|
3984
|
+
span.setAttribute(SpanAttributes.TRACELOOP_WORKFLOW_NAME, name);
|
|
3985
|
+
}
|
|
3986
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_NAME, name);
|
|
3987
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_PATH, entityPath || "");
|
|
3988
|
+
span.setAttribute(SpanAttributes.TRACELOOP_SPAN_KIND, type);
|
|
3989
|
+
// Set agent name on all spans when there's an active agent context
|
|
3990
|
+
const agentName = entityContext.getValue(AGENT_NAME_KEY);
|
|
3991
|
+
if (agentName) {
|
|
3992
|
+
span.setAttribute(ATTR_GEN_AI_AGENT_NAME, agentName);
|
|
3993
|
+
}
|
|
3994
|
+
// Set conversation ID on all spans when there's an active conversation context
|
|
3995
|
+
const conversationId = entityContext.getValue(CONVERSATION_ID_KEY);
|
|
3996
|
+
if (conversationId) {
|
|
3997
|
+
span.setAttribute(ATTR_GEN_AI_CONVERSATION_ID, conversationId);
|
|
3998
|
+
}
|
|
3999
|
+
if (version) {
|
|
4000
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_VERSION, version);
|
|
4001
|
+
}
|
|
4002
|
+
if (shouldSendTraces()) {
|
|
4003
|
+
try {
|
|
4004
|
+
const input = inputParameters !== null && inputParameters !== void 0 ? inputParameters : args;
|
|
4005
|
+
if (input.length === 1 &&
|
|
4006
|
+
typeof input[0] === "object" &&
|
|
4007
|
+
!(input[0] instanceof Map)) {
|
|
4008
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_INPUT, serialize({ args: [], kwargs: input[0] }));
|
|
4009
|
+
}
|
|
4010
|
+
else {
|
|
4011
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_INPUT, serialize({
|
|
4012
|
+
args: input,
|
|
4013
|
+
kwargs: {},
|
|
4014
|
+
}));
|
|
4015
|
+
}
|
|
4016
|
+
}
|
|
4017
|
+
catch (error) {
|
|
4018
|
+
console.debug("Error setting input attributes", error);
|
|
4019
|
+
}
|
|
4020
|
+
}
|
|
4021
|
+
const res = fn.apply(thisArg, args);
|
|
4022
|
+
if (res instanceof Promise) {
|
|
4023
|
+
return res.then((resolvedRes) => {
|
|
4024
|
+
try {
|
|
4025
|
+
if (shouldSendTraces()) {
|
|
4026
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_OUTPUT, serialize(resolvedRes));
|
|
4027
|
+
}
|
|
4028
|
+
}
|
|
4029
|
+
catch (error) {
|
|
4030
|
+
console.debug("Error setting output attributes", error);
|
|
4031
|
+
}
|
|
4032
|
+
finally {
|
|
4033
|
+
span.end();
|
|
4034
|
+
}
|
|
4035
|
+
return resolvedRes;
|
|
4036
|
+
});
|
|
4037
|
+
}
|
|
4038
|
+
try {
|
|
4039
|
+
if (shouldSendTraces()) {
|
|
4040
|
+
span.setAttribute(SpanAttributes.TRACELOOP_ENTITY_OUTPUT, serialize(res));
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
catch (error) {
|
|
4044
|
+
console.debug("Error setting output attributes", error);
|
|
4045
|
+
}
|
|
4046
|
+
finally {
|
|
4047
|
+
span.end();
|
|
4048
|
+
}
|
|
4049
|
+
return res;
|
|
4050
|
+
}));
|
|
4051
|
+
}
|
|
4052
|
+
function withWorkflow(config, fn, ...args) {
|
|
4053
|
+
return withEntity(TraceloopSpanKindValues.WORKFLOW, config, fn, undefined, ...args);
|
|
4054
|
+
}
|
|
4055
|
+
function withTask(config, fn, ...args) {
|
|
4056
|
+
return withEntity(TraceloopSpanKindValues.TASK, config, fn, undefined, ...args);
|
|
4057
|
+
}
|
|
4058
|
+
function withAgent(config, fn, ...args) {
|
|
4059
|
+
return withEntity(TraceloopSpanKindValues.AGENT, config, fn, undefined, ...args);
|
|
4060
|
+
}
|
|
4061
|
+
function withTool(config, fn, ...args) {
|
|
4062
|
+
return withEntity(TraceloopSpanKindValues.TOOL, config, fn, undefined, ...args);
|
|
4063
|
+
}
|
|
4064
|
+
function entity(type, config) {
|
|
4065
|
+
return function (target, propertyKey, descriptor) {
|
|
4066
|
+
const originalMethod = descriptor.value;
|
|
4067
|
+
descriptor.value = function (...args) {
|
|
4068
|
+
var _a;
|
|
4069
|
+
let actualConfig;
|
|
4070
|
+
if (typeof config === "function") {
|
|
4071
|
+
actualConfig = config(this, ...args);
|
|
4072
|
+
}
|
|
4073
|
+
else {
|
|
4074
|
+
actualConfig = config;
|
|
4075
|
+
}
|
|
4076
|
+
const entityName = (_a = actualConfig.name) !== null && _a !== void 0 ? _a : originalMethod.name;
|
|
4077
|
+
return withEntity(type, Object.assign(Object.assign({}, actualConfig), { name: entityName }), originalMethod, this, ...args);
|
|
4078
|
+
};
|
|
4079
|
+
};
|
|
4080
|
+
}
|
|
4081
|
+
function workflow(config) {
|
|
4082
|
+
return entity(TraceloopSpanKindValues.WORKFLOW, config !== null && config !== void 0 ? config : {});
|
|
4083
|
+
}
|
|
4084
|
+
function task(config) {
|
|
4085
|
+
return entity(TraceloopSpanKindValues.TASK, config !== null && config !== void 0 ? config : {});
|
|
4086
|
+
}
|
|
4087
|
+
function agent(config) {
|
|
4088
|
+
return entity(TraceloopSpanKindValues.AGENT, config !== null && config !== void 0 ? config : {});
|
|
4089
|
+
}
|
|
4090
|
+
function tool(config) {
|
|
4091
|
+
return entity(TraceloopSpanKindValues.TOOL, config !== null && config !== void 0 ? config : {});
|
|
4092
|
+
}
|
|
4093
|
+
function withConversation(conversationId, fn, thisArg, ...args) {
|
|
4094
|
+
const conversationContext = context
|
|
4095
|
+
.active()
|
|
4096
|
+
.setValue(CONVERSATION_ID_KEY, conversationId);
|
|
4097
|
+
return context.with(conversationContext, () => withEntity(TraceloopSpanKindValues.WORKFLOW, { name: `conversation.${conversationId}` }, fn, thisArg, ...args));
|
|
4098
|
+
}
|
|
4099
|
+
function conversation(conversationId) {
|
|
4100
|
+
return function (target, propertyKey, descriptor) {
|
|
4101
|
+
const originalMethod = descriptor.value;
|
|
4102
|
+
descriptor.value = function (...args) {
|
|
4103
|
+
let actualConversationId;
|
|
4104
|
+
if (typeof conversationId === "function") {
|
|
4105
|
+
actualConversationId = conversationId(this, ...args);
|
|
4106
|
+
}
|
|
4107
|
+
else {
|
|
4108
|
+
actualConversationId = conversationId;
|
|
4109
|
+
}
|
|
4110
|
+
return withConversation(actualConversationId, originalMethod, this, ...args);
|
|
4111
|
+
};
|
|
4112
|
+
};
|
|
4113
|
+
}
|
|
4114
|
+
function cleanInput(input) {
|
|
4115
|
+
if (input instanceof Map) {
|
|
4116
|
+
return Array.from(input.entries());
|
|
4117
|
+
}
|
|
4118
|
+
else if (Array.isArray(input)) {
|
|
4119
|
+
return input.map((value) => cleanInput(value));
|
|
4120
|
+
}
|
|
4121
|
+
else if (!input) {
|
|
4122
|
+
return input;
|
|
4123
|
+
}
|
|
4124
|
+
else if (typeof input === "object") {
|
|
4125
|
+
// serialize object one by one
|
|
4126
|
+
const output = {};
|
|
4127
|
+
Object.entries(input).forEach(([key, value]) => {
|
|
4128
|
+
output[key] = cleanInput(value);
|
|
4129
|
+
});
|
|
4130
|
+
return output;
|
|
4131
|
+
}
|
|
4132
|
+
return input;
|
|
4133
|
+
}
|
|
4134
|
+
function serialize(input) {
|
|
4135
|
+
return JSON.stringify(cleanInput(input));
|
|
4136
|
+
}
|
|
4137
|
+
|
|
4138
|
+
class VectorSpan {
|
|
4139
|
+
constructor(span) {
|
|
4140
|
+
this.span = span;
|
|
4141
|
+
}
|
|
4142
|
+
reportQuery({ queryVector }) {
|
|
4143
|
+
if (!shouldSendTraces()) {
|
|
4144
|
+
this.span.addEvent(Events.DB_QUERY_EMBEDDINGS);
|
|
4145
|
+
}
|
|
4146
|
+
this.span.addEvent(Events.DB_QUERY_EMBEDDINGS, {
|
|
4147
|
+
[EventAttributes.DB_QUERY_EMBEDDINGS_VECTOR]: JSON.stringify(queryVector),
|
|
4148
|
+
});
|
|
4149
|
+
}
|
|
4150
|
+
reportResults({ results, }) {
|
|
4151
|
+
for (let i = 0; i < results.length; i++) {
|
|
4152
|
+
this.span.addEvent(Events.DB_QUERY_RESULT, {
|
|
4153
|
+
[EventAttributes.DB_QUERY_RESULT_ID]: results[i].ids,
|
|
4154
|
+
[EventAttributes.DB_QUERY_RESULT_SCORE]: results[i].scores,
|
|
4155
|
+
[EventAttributes.DB_QUERY_RESULT_DISTANCE]: results[i].distances,
|
|
4156
|
+
[EventAttributes.DB_QUERY_RESULT_METADATA]: JSON.stringify(results[i].metadata),
|
|
4157
|
+
[EventAttributes.DB_QUERY_RESULT_VECTOR]: results[i].vectors,
|
|
4158
|
+
[EventAttributes.DB_QUERY_RESULT_DOCUMENT]: results[i].documents,
|
|
4159
|
+
});
|
|
4160
|
+
}
|
|
4161
|
+
}
|
|
4162
|
+
}
|
|
4163
|
+
class LLMSpan {
|
|
4164
|
+
constructor(span) {
|
|
4165
|
+
this.span = span;
|
|
4166
|
+
}
|
|
4167
|
+
reportRequest({ model, messages, }) {
|
|
4168
|
+
this.span.setAttributes({
|
|
4169
|
+
[ATTR_GEN_AI_REQUEST_MODEL]: model,
|
|
4170
|
+
});
|
|
4171
|
+
messages.forEach((message, index) => {
|
|
4172
|
+
this.span.setAttributes({
|
|
4173
|
+
[`${ATTR_GEN_AI_PROMPT}.${index}.role`]: message.role,
|
|
4174
|
+
[`${ATTR_GEN_AI_PROMPT}.${index}.content`]: typeof message.content === "string"
|
|
4175
|
+
? message.content
|
|
4176
|
+
: JSON.stringify(message.content),
|
|
4177
|
+
});
|
|
4178
|
+
});
|
|
4179
|
+
}
|
|
4180
|
+
reportResponse({ model, usage, completions, }) {
|
|
4181
|
+
this.span.setAttribute(ATTR_GEN_AI_RESPONSE_MODEL, model);
|
|
4182
|
+
if (usage) {
|
|
4183
|
+
this.span.setAttributes({
|
|
4184
|
+
[ATTR_GEN_AI_USAGE_INPUT_TOKENS]: usage.prompt_tokens,
|
|
4185
|
+
[ATTR_GEN_AI_USAGE_OUTPUT_TOKENS]: usage.completion_tokens,
|
|
4186
|
+
[SpanAttributes.LLM_USAGE_TOTAL_TOKENS]: usage.total_tokens,
|
|
4187
|
+
});
|
|
4188
|
+
}
|
|
4189
|
+
completions === null || completions === void 0 ? void 0 : completions.forEach((completion, index) => {
|
|
4190
|
+
this.span.setAttributes({
|
|
4191
|
+
[`${ATTR_GEN_AI_COMPLETION}.${index}.finish_reason`]: completion.finish_reason,
|
|
4192
|
+
[`${ATTR_GEN_AI_COMPLETION}.${index}.role`]: completion.message.role,
|
|
4193
|
+
[`${ATTR_GEN_AI_COMPLETION}.${index}.content`]: completion.message.content || "",
|
|
4194
|
+
});
|
|
4195
|
+
});
|
|
4196
|
+
}
|
|
4197
|
+
}
|
|
4198
|
+
function withVectorDBCall({ vendor, type }, fn, thisArg) {
|
|
4199
|
+
const entityContext = context.active();
|
|
4200
|
+
return getTracer().startActiveSpan(`${vendor}.${type}`, { [SpanAttributes.LLM_REQUEST_TYPE]: type }, entityContext, (span) => {
|
|
4201
|
+
// Set agent name if there's an active agent context
|
|
4202
|
+
const agentName = entityContext.getValue(AGENT_NAME_KEY);
|
|
4203
|
+
if (agentName) {
|
|
4204
|
+
span.setAttribute(ATTR_GEN_AI_AGENT_NAME, agentName);
|
|
4205
|
+
}
|
|
4206
|
+
const res = fn.apply(thisArg, [{ span: new VectorSpan(span) }]);
|
|
4207
|
+
if (res instanceof Promise) {
|
|
4208
|
+
return res.then((resolvedRes) => {
|
|
4209
|
+
span.end();
|
|
4210
|
+
return resolvedRes;
|
|
4211
|
+
});
|
|
4212
|
+
}
|
|
4213
|
+
span.end();
|
|
4214
|
+
return res;
|
|
4215
|
+
});
|
|
4216
|
+
}
|
|
4217
|
+
function withLLMCall({ vendor, type }, fn, thisArg) {
|
|
4218
|
+
const currentContext = context.active();
|
|
4219
|
+
const span = getTracer().startSpan(`${vendor}.${type}`, {}, currentContext);
|
|
4220
|
+
span.setAttribute(SpanAttributes.LLM_REQUEST_TYPE, type);
|
|
4221
|
+
// Set agent name if there's an active agent context
|
|
4222
|
+
const agentName = currentContext.getValue(AGENT_NAME_KEY);
|
|
4223
|
+
if (agentName) {
|
|
4224
|
+
span.setAttribute(ATTR_GEN_AI_AGENT_NAME, agentName);
|
|
4225
|
+
}
|
|
4226
|
+
trace.setSpan(currentContext, span);
|
|
4227
|
+
const res = fn.apply(thisArg, [{ span: new LLMSpan(span) }]);
|
|
4228
|
+
if (res instanceof Promise) {
|
|
4229
|
+
return res.then((resolvedRes) => {
|
|
4230
|
+
span.end();
|
|
4231
|
+
return resolvedRes;
|
|
4232
|
+
});
|
|
4233
|
+
}
|
|
4234
|
+
span.end();
|
|
4235
|
+
return res;
|
|
4236
|
+
}
|
|
4237
|
+
|
|
4238
|
+
function withAssociationProperties(properties, fn, thisArg, ...args) {
|
|
4239
|
+
if (Object.keys(properties).length === 0) {
|
|
4240
|
+
return fn.apply(thisArg, args);
|
|
4241
|
+
}
|
|
4242
|
+
const newContext = context
|
|
4243
|
+
.active()
|
|
4244
|
+
.setValue(ASSOCATION_PROPERTIES_KEY, properties);
|
|
4245
|
+
return context.with(newContext, fn, thisArg, ...args);
|
|
4246
|
+
}
|
|
4247
|
+
|
|
4248
|
+
/**
|
|
4249
|
+
* Reports a custom metric to the current active span.
|
|
4250
|
+
*
|
|
4251
|
+
* This function allows you to add a custom metric to the current span in the trace.
|
|
4252
|
+
* If there is no active span, a warning will be logged.
|
|
4253
|
+
*
|
|
4254
|
+
* @param {string} metricName - The name of the custom metric.
|
|
4255
|
+
* @param {number} metricValue - The numeric value of the custom metric.
|
|
4256
|
+
*
|
|
4257
|
+
* @example
|
|
4258
|
+
* reportCustomMetric('processing_time', 150);
|
|
4259
|
+
*/
|
|
4260
|
+
const reportCustomMetric = (metricName, metricValue) => {
|
|
4261
|
+
const currentContext = context.active();
|
|
4262
|
+
const currentSpan = trace.getSpan(currentContext);
|
|
4263
|
+
if (currentSpan) {
|
|
4264
|
+
currentSpan.setAttribute(`traceloop.custom_metric.${metricName}`, metricValue);
|
|
4265
|
+
}
|
|
4266
|
+
else {
|
|
4267
|
+
diag.warn(`No active span found to report custom metric: ${metricName}`);
|
|
4268
|
+
}
|
|
4269
|
+
};
|
|
4270
|
+
|
|
4271
|
+
const TEMPLATING_ENGINE = {
|
|
4272
|
+
JINJA2: "jinja2",
|
|
4273
|
+
};
|
|
4274
|
+
|
|
4275
|
+
const env = new Environment(null, {
|
|
4276
|
+
throwOnUndefined: true, // throw error if param not found
|
|
4277
|
+
});
|
|
4278
|
+
const renderMessages = (promptVersion, variables) => {
|
|
4279
|
+
if (promptVersion.templating_engine === TEMPLATING_ENGINE.JINJA2) {
|
|
4280
|
+
return promptVersion.messages.map((message) => {
|
|
4281
|
+
try {
|
|
4282
|
+
if (typeof message.template === "string") {
|
|
4283
|
+
return {
|
|
4284
|
+
content: env.renderString(message.template, variables),
|
|
4285
|
+
role: message.role,
|
|
4286
|
+
};
|
|
4287
|
+
}
|
|
4288
|
+
else {
|
|
4289
|
+
return {
|
|
4290
|
+
content: message.template.map((content) => {
|
|
4291
|
+
if (content.type === "text") {
|
|
4292
|
+
return {
|
|
4293
|
+
type: "text",
|
|
4294
|
+
text: env.renderString(content.text, variables),
|
|
4295
|
+
};
|
|
4296
|
+
}
|
|
4297
|
+
else {
|
|
4298
|
+
return content;
|
|
4299
|
+
}
|
|
4300
|
+
}),
|
|
4301
|
+
role: message.role,
|
|
4302
|
+
};
|
|
4303
|
+
}
|
|
4304
|
+
}
|
|
4305
|
+
catch (err) {
|
|
4306
|
+
throw new TraceloopError(`Failed to render message template. Missing variables?`);
|
|
4307
|
+
}
|
|
4308
|
+
});
|
|
4309
|
+
}
|
|
4310
|
+
else {
|
|
4311
|
+
throw new TraceloopError(`Templating engine ${promptVersion.templating_engine} is not supported`);
|
|
4312
|
+
}
|
|
4313
|
+
};
|
|
4314
|
+
|
|
4315
|
+
const getEffectiveVersion = (prompt) => {
|
|
4316
|
+
const version = prompt.versions.find((v) => v.id === prompt.target.version);
|
|
4317
|
+
if (!version) {
|
|
4318
|
+
throw new TraceloopError(`Prompt version ${prompt.target.version} not found`);
|
|
4319
|
+
}
|
|
4320
|
+
return version;
|
|
4321
|
+
};
|
|
4322
|
+
const managedPromptTracingAttributes = (prompt, promptVersion, variables) => {
|
|
4323
|
+
const variableAttributes = Object.keys(variables).reduce((acc, key) => {
|
|
4324
|
+
acc[`traceloop.prompt.template_variables.${key}`] = variables[key];
|
|
4325
|
+
return acc;
|
|
4326
|
+
}, {});
|
|
4327
|
+
return Object.assign({ "traceloop.prompt.key": prompt.key, "traceloop.prompt.version": promptVersion.version, "traceloop.prompt.version_hash": promptVersion.hash, "traceloop.prompt.version_name": promptVersion.name }, variableAttributes);
|
|
4328
|
+
};
|
|
4329
|
+
const getPrompt = (key, variables) => {
|
|
4330
|
+
var _a;
|
|
4331
|
+
const prompt = getPromptByKey(key);
|
|
4332
|
+
const promptVersion = getEffectiveVersion(prompt);
|
|
4333
|
+
let result = {}; //TODO - SDK needs to do work specific to each vendor/model? maybe we do this in the backend?
|
|
4334
|
+
if (promptVersion.llm_config.mode === "completion") {
|
|
4335
|
+
const message = renderMessages(promptVersion, variables);
|
|
4336
|
+
result = Object.assign(Object.assign({}, promptVersion.llm_config), { prompt: (_a = message === null || message === void 0 ? void 0 : message[0]) === null || _a === void 0 ? void 0 : _a.content });
|
|
4337
|
+
}
|
|
4338
|
+
else {
|
|
4339
|
+
result = Object.assign({ messages: renderMessages(promptVersion, variables) }, promptVersion.llm_config);
|
|
4340
|
+
}
|
|
4341
|
+
if ((result === null || result === void 0 ? void 0 : result["stop"].length) === 0)
|
|
4342
|
+
delete result["stop"];
|
|
4343
|
+
delete result["mode"];
|
|
4344
|
+
result.extraAttributes = managedPromptTracingAttributes(prompt, promptVersion, variables);
|
|
4345
|
+
return result;
|
|
4346
|
+
};
|
|
4347
|
+
|
|
4348
|
+
/**
|
|
4349
|
+
* Standard association properties for tracing.
|
|
4350
|
+
* Use these with withAssociationProperties() or decorator associationProperties config.
|
|
4351
|
+
*
|
|
4352
|
+
* @example
|
|
4353
|
+
* ```typescript
|
|
4354
|
+
* // With withAssociationProperties
|
|
4355
|
+
* await traceloop.withAssociationProperties(
|
|
4356
|
+
* {
|
|
4357
|
+
* [traceloop.AssociationProperty.USER_ID]: "12345",
|
|
4358
|
+
* [traceloop.AssociationProperty.SESSION_ID]: "session-abc"
|
|
4359
|
+
* },
|
|
4360
|
+
* async () => {
|
|
4361
|
+
* await chat();
|
|
4362
|
+
* }
|
|
4363
|
+
* );
|
|
4364
|
+
*
|
|
4365
|
+
* // With decorator
|
|
4366
|
+
* @traceloop.workflow((thisArg) => ({
|
|
4367
|
+
* name: "my_workflow",
|
|
4368
|
+
* associationProperties: {
|
|
4369
|
+
* [traceloop.AssociationProperty.USER_ID]: (thisArg as MyClass).userId,
|
|
4370
|
+
* },
|
|
4371
|
+
* }))
|
|
4372
|
+
* ```
|
|
4373
|
+
*/
|
|
4374
|
+
var AssociationProperty;
|
|
4375
|
+
(function (AssociationProperty) {
|
|
4376
|
+
AssociationProperty["CUSTOMER_ID"] = "customer_id";
|
|
4377
|
+
AssociationProperty["USER_ID"] = "user_id";
|
|
4378
|
+
AssociationProperty["SESSION_ID"] = "session_id";
|
|
4379
|
+
})(AssociationProperty || (AssociationProperty = {}));
|
|
4380
|
+
|
|
4381
|
+
export { ALL_INSTRUMENTATION_LIBRARIES, ArgumentNotProvidedError, AssociationProperty, Attachment, AttachmentReference, AttachmentUploader, Column, Dataset, Datasets, Evaluator, EvaluatorMadeByTraceloop, Experiment, ExternalAttachment, ImageUploader, InitializationError, LLMSpan, NotInitializedError, PromptNotFoundError, Row, SEVERITY, TraceloopClient, TraceloopError, VectorSpan, agent, attachment, conversation, createEvaluator, createSpanProcessor, forceFlush, getAvailableEvaluatorSlugs, getClient, getEvaluatorSchemaInfo, getPrompt, getTraceloopTracer, initialize, isAnyAttachment, isAttachment, isAttachmentReference, isExternalAttachment, reportCustomMetric, task, tool, traceloopInstrumentationLibraries, validateEvaluatorInput, waitForInitialization, withAgent, withAssociationProperties, withConversation, withLLMCall, withTask, withTool, withVectorDBCall, withWorkflow, workflow };
|
|
4382
|
+
//# sourceMappingURL=index.mjs.map
|