@pauly4010/evalai-sdk 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +289 -0
- package/LICENSE +21 -0
- package/README.md +565 -0
- package/dist/assertions.d.ts +189 -0
- package/dist/assertions.js +596 -0
- package/dist/batch.d.ts +68 -0
- package/dist/batch.js +178 -0
- package/dist/cache.d.ts +65 -0
- package/dist/cache.js +135 -0
- package/dist/cli/index.d.ts +6 -0
- package/dist/cli/index.js +181 -0
- package/dist/client.d.ts +358 -0
- package/dist/client.js +802 -0
- package/dist/context.d.ts +134 -0
- package/dist/context.js +215 -0
- package/dist/errors.d.ts +80 -0
- package/dist/errors.js +285 -0
- package/dist/export.d.ts +195 -0
- package/dist/export.js +334 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +111 -0
- package/dist/integrations/anthropic.d.ts +72 -0
- package/dist/integrations/anthropic.js +159 -0
- package/dist/integrations/openai.d.ts +69 -0
- package/dist/integrations/openai.js +156 -0
- package/dist/local.d.ts +39 -0
- package/dist/local.js +146 -0
- package/dist/logger.d.ts +128 -0
- package/dist/logger.js +227 -0
- package/dist/pagination.d.ts +74 -0
- package/dist/pagination.js +135 -0
- package/dist/snapshot.d.ts +176 -0
- package/dist/snapshot.js +322 -0
- package/dist/streaming.d.ts +173 -0
- package/dist/streaming.js +268 -0
- package/dist/testing.d.ts +204 -0
- package/dist/testing.js +252 -0
- package/dist/types.d.ts +715 -0
- package/dist/types.js +54 -0
- package/dist/workflows.d.ts +378 -0
- package/dist/workflows.js +628 -0
- package/package.json +102 -0
package/dist/client.js
ADDED
|
@@ -0,0 +1,802 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AIEvalClient = void 0;
|
|
4
|
+
const errors_1 = require("./errors");
|
|
5
|
+
const logger_1 = require("./logger");
|
|
6
|
+
const context_1 = require("./context");
|
|
7
|
+
const cache_1 = require("./cache");
|
|
8
|
+
const batch_1 = require("./batch");
|
|
9
|
+
/**
|
|
10
|
+
* Safe environment variable access (works in both Node.js and browsers)
|
|
11
|
+
*/
|
|
12
|
+
function getEnvVar(name) {
|
|
13
|
+
if (typeof process !== 'undefined' && process.env) {
|
|
14
|
+
return process.env[name];
|
|
15
|
+
}
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* AI Evaluation Platform SDK Client
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```typescript
|
|
23
|
+
* import { AIEvalClient } from '@ai-eval-platform/sdk';
|
|
24
|
+
*
|
|
25
|
+
* // Zero-config initialization (uses env variables)
|
|
26
|
+
* const client = AIEvalClient.init();
|
|
27
|
+
*
|
|
28
|
+
* // Or with explicit config
|
|
29
|
+
* const client = new AIEvalClient({
|
|
30
|
+
* apiKey: 'your-api-key',
|
|
31
|
+
* organizationId: 123,
|
|
32
|
+
* debug: true
|
|
33
|
+
* });
|
|
34
|
+
*
|
|
35
|
+
* // Create a trace with automatic context propagation
|
|
36
|
+
* const trace = await client.traces.create({
|
|
37
|
+
* name: 'User Query',
|
|
38
|
+
* traceId: 'trace-123'
|
|
39
|
+
* });
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
class AIEvalClient {
|
|
43
|
+
constructor(config = {}) {
|
|
44
|
+
// Tier 1.1: Zero-config with env variable detection (works in Node.js and browsers)
|
|
45
|
+
this.apiKey = config.apiKey || getEnvVar('EVALAI_API_KEY') || getEnvVar('AI_EVAL_API_KEY') || '';
|
|
46
|
+
if (!this.apiKey) {
|
|
47
|
+
throw new errors_1.EvalAIError('API key is required. Provide via config.apiKey or EVALAI_API_KEY environment variable.', 'MISSING_API_KEY', 0);
|
|
48
|
+
}
|
|
49
|
+
// Auto-detect organization ID from env
|
|
50
|
+
const orgIdFromEnv = getEnvVar('EVALAI_ORGANIZATION_ID') || getEnvVar('AI_EVAL_ORGANIZATION_ID');
|
|
51
|
+
this.organizationId = config.organizationId || (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
|
|
52
|
+
// Default to relative URLs for browser, or allow custom baseUrl
|
|
53
|
+
const isBrowser = typeof globalThis.window !== 'undefined';
|
|
54
|
+
this.baseUrl = config.baseUrl || (isBrowser ? '' : 'http://localhost:3000');
|
|
55
|
+
this.timeout = config.timeout || 30000;
|
|
56
|
+
// Tier 4.17: Debug mode with request logging
|
|
57
|
+
const logLevel = config.logLevel || (config.debug ? 'debug' : 'info');
|
|
58
|
+
this.logger = (0, logger_1.createLogger)({
|
|
59
|
+
level: logLevel,
|
|
60
|
+
pretty: config.debug,
|
|
61
|
+
prefix: 'EvalAI'
|
|
62
|
+
});
|
|
63
|
+
this.requestLogger = new logger_1.RequestLogger(this.logger);
|
|
64
|
+
// Retry configuration
|
|
65
|
+
this.retryConfig = {
|
|
66
|
+
maxAttempts: config.retry?.maxAttempts || 3,
|
|
67
|
+
backoff: config.retry?.backoff || 'exponential',
|
|
68
|
+
retryableErrors: config.retry?.retryableErrors || [
|
|
69
|
+
'RATE_LIMIT_EXCEEDED',
|
|
70
|
+
'TIMEOUT',
|
|
71
|
+
'NETWORK_ERROR',
|
|
72
|
+
'INTERNAL_SERVER_ERROR'
|
|
73
|
+
]
|
|
74
|
+
};
|
|
75
|
+
// Initialize cache for GET requests
|
|
76
|
+
this.cache = new cache_1.RequestCache(config.cacheSize || 1000);
|
|
77
|
+
// Initialize request batcher if enabled (default: enabled)
|
|
78
|
+
if (config.enableBatching !== false) {
|
|
79
|
+
this.batcher = new batch_1.RequestBatcher(async (requests) => {
|
|
80
|
+
// Batch execution placeholder - will be implemented per API
|
|
81
|
+
return requests.map(req => ({
|
|
82
|
+
id: req.id,
|
|
83
|
+
status: 200,
|
|
84
|
+
data: null,
|
|
85
|
+
}));
|
|
86
|
+
}, {
|
|
87
|
+
maxBatchSize: config.batchSize || 10,
|
|
88
|
+
batchDelay: config.batchDelay || 50,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
this.batcher = null;
|
|
93
|
+
}
|
|
94
|
+
// Initialize API modules
|
|
95
|
+
this.traces = new TraceAPI(this);
|
|
96
|
+
this.evaluations = new EvaluationAPI(this);
|
|
97
|
+
this.llmJudge = new LLMJudgeAPI(this);
|
|
98
|
+
this.annotations = new AnnotationsAPI(this);
|
|
99
|
+
this.developer = new DeveloperAPI(this);
|
|
100
|
+
this.organizations = new OrganizationsAPI(this);
|
|
101
|
+
this.logger.info('SDK initialized', {
|
|
102
|
+
hasOrganizationId: !!this.organizationId,
|
|
103
|
+
baseUrl: this.baseUrl
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Zero-config initialization using environment variables
|
|
108
|
+
*
|
|
109
|
+
* Works in both Node.js and browsers. In Node.js, reads from environment variables.
|
|
110
|
+
* In browsers, you must provide config explicitly.
|
|
111
|
+
*
|
|
112
|
+
* Environment variables (Node.js only):
|
|
113
|
+
* - EVALAI_API_KEY or AI_EVAL_API_KEY: Your API key
|
|
114
|
+
* - EVALAI_ORGANIZATION_ID or AI_EVAL_ORGANIZATION_ID: Your organization ID
|
|
115
|
+
* - EVALAI_BASE_URL: Custom API base URL (optional)
|
|
116
|
+
*
|
|
117
|
+
* @example
|
|
118
|
+
* ```typescript
|
|
119
|
+
* // Node.js - reads from env vars:
|
|
120
|
+
* // EVALAI_API_KEY=your-key
|
|
121
|
+
* // EVALAI_ORGANIZATION_ID=123
|
|
122
|
+
* const client = AIEvalClient.init();
|
|
123
|
+
*
|
|
124
|
+
* // Browser - must provide config:
|
|
125
|
+
* const client = AIEvalClient.init({
|
|
126
|
+
* apiKey: 'your-key',
|
|
127
|
+
* organizationId: 123
|
|
128
|
+
* });
|
|
129
|
+
* ```
|
|
130
|
+
*/
|
|
131
|
+
static init(config = {}) {
|
|
132
|
+
return new AIEvalClient({
|
|
133
|
+
baseUrl: getEnvVar('EVALAI_BASE_URL'),
|
|
134
|
+
...config
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Internal method to make HTTP requests with retry logic and error handling
|
|
139
|
+
*/
|
|
140
|
+
async request(endpoint, options = {}, attempt = 1) {
|
|
141
|
+
const method = (options.method || 'GET').toUpperCase();
|
|
142
|
+
const url = `${this.baseUrl}${endpoint}`;
|
|
143
|
+
// Check cache for GET requests
|
|
144
|
+
if (method === 'GET' && (0, cache_1.shouldCache)(method, endpoint)) {
|
|
145
|
+
const cached = this.cache.get(method, endpoint, options.body);
|
|
146
|
+
if (cached !== null) {
|
|
147
|
+
this.logger.debug('Cache hit', { endpoint });
|
|
148
|
+
return cached;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const controller = new AbortController();
|
|
152
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
153
|
+
const startTime = Date.now();
|
|
154
|
+
// Log request
|
|
155
|
+
this.requestLogger.logRequest({
|
|
156
|
+
method: options.method || 'GET',
|
|
157
|
+
url,
|
|
158
|
+
headers: options.headers,
|
|
159
|
+
body: options.body
|
|
160
|
+
});
|
|
161
|
+
try {
|
|
162
|
+
const response = await fetch(url, {
|
|
163
|
+
...options,
|
|
164
|
+
headers: {
|
|
165
|
+
'Content-Type': 'application/json',
|
|
166
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
167
|
+
...options.headers,
|
|
168
|
+
},
|
|
169
|
+
signal: controller.signal,
|
|
170
|
+
});
|
|
171
|
+
clearTimeout(timeoutId);
|
|
172
|
+
const duration = Date.now() - startTime;
|
|
173
|
+
let data;
|
|
174
|
+
try {
|
|
175
|
+
data = await response.json();
|
|
176
|
+
}
|
|
177
|
+
catch (e) {
|
|
178
|
+
data = {};
|
|
179
|
+
}
|
|
180
|
+
// Log response
|
|
181
|
+
this.requestLogger.logResponse({
|
|
182
|
+
method: options.method || 'GET',
|
|
183
|
+
url,
|
|
184
|
+
status: response.status,
|
|
185
|
+
duration,
|
|
186
|
+
body: data
|
|
187
|
+
});
|
|
188
|
+
if (!response.ok) {
|
|
189
|
+
const error = (0, errors_1.createErrorFromResponse)(response, data);
|
|
190
|
+
// Retry logic
|
|
191
|
+
if (attempt < this.retryConfig.maxAttempts &&
|
|
192
|
+
this.retryConfig.retryableErrors.includes(error.code)) {
|
|
193
|
+
const delay = this.calculateBackoff(attempt);
|
|
194
|
+
this.logger.warn(`Retrying request (attempt ${attempt + 1}/${this.retryConfig.maxAttempts}) after ${delay}ms`, {
|
|
195
|
+
error: error.code,
|
|
196
|
+
url
|
|
197
|
+
});
|
|
198
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
199
|
+
return this.request(endpoint, options, attempt + 1);
|
|
200
|
+
}
|
|
201
|
+
throw error;
|
|
202
|
+
}
|
|
203
|
+
// Cache successful GET responses
|
|
204
|
+
if (method === 'GET' && (0, cache_1.shouldCache)(method, endpoint)) {
|
|
205
|
+
const ttl = (0, cache_1.getTTL)(endpoint);
|
|
206
|
+
this.cache.set(method, endpoint, data, ttl, options.body);
|
|
207
|
+
this.logger.debug('Cached response', { endpoint, ttl });
|
|
208
|
+
}
|
|
209
|
+
// Invalidate cache for mutation operations
|
|
210
|
+
if (['POST', 'PUT', 'DELETE', 'PATCH'].includes(method)) {
|
|
211
|
+
// Invalidate related cached entries
|
|
212
|
+
const resourceMatch = endpoint.match(/\/api\/(\w+)/);
|
|
213
|
+
if (resourceMatch) {
|
|
214
|
+
this.cache.invalidatePattern(resourceMatch[1]);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return data;
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
clearTimeout(timeoutId);
|
|
221
|
+
if (error instanceof errors_1.EvalAIError) {
|
|
222
|
+
throw error;
|
|
223
|
+
}
|
|
224
|
+
if (error instanceof Error) {
|
|
225
|
+
if (error.name === 'AbortError') {
|
|
226
|
+
throw new errors_1.EvalAIError('Request timeout', 'TIMEOUT', 408);
|
|
227
|
+
}
|
|
228
|
+
throw new errors_1.EvalAIError(error.message, 'NETWORK_ERROR', 0);
|
|
229
|
+
}
|
|
230
|
+
throw error;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Calculate backoff delay for retries
|
|
235
|
+
*/
|
|
236
|
+
calculateBackoff(attempt) {
|
|
237
|
+
const baseDelay = 1000; // 1 second
|
|
238
|
+
switch (this.retryConfig.backoff) {
|
|
239
|
+
case 'exponential':
|
|
240
|
+
return baseDelay * Math.pow(2, attempt - 1);
|
|
241
|
+
case 'linear':
|
|
242
|
+
return baseDelay * attempt;
|
|
243
|
+
case 'fixed':
|
|
244
|
+
default:
|
|
245
|
+
return baseDelay;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
getOrganizationId() {
|
|
249
|
+
return this.organizationId;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Get the logger instance for custom logging
|
|
253
|
+
*/
|
|
254
|
+
getLogger() {
|
|
255
|
+
return this.logger;
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Get organization resource limits and usage
|
|
259
|
+
* Returns feature usage data for per-organization quotas
|
|
260
|
+
*
|
|
261
|
+
* @example
|
|
262
|
+
* ```typescript
|
|
263
|
+
* const limits = await client.getOrganizationLimits();
|
|
264
|
+
* console.log('Traces:', limits.traces_per_organization);
|
|
265
|
+
* console.log('Evaluations:', limits.evals_per_organization);
|
|
266
|
+
* ```
|
|
267
|
+
*/
|
|
268
|
+
async getOrganizationLimits() {
|
|
269
|
+
const orgId = this.getOrganizationId();
|
|
270
|
+
if (!orgId) {
|
|
271
|
+
throw new errors_1.EvalAIError('Organization ID is required', 'MISSING_ORGANIZATION_ID', 0);
|
|
272
|
+
}
|
|
273
|
+
return this.request(`/api/organizations/${orgId}/limits`);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
exports.AIEvalClient = AIEvalClient;
|
|
277
|
+
/**
|
|
278
|
+
* Trace API methods
|
|
279
|
+
*/
|
|
280
|
+
class TraceAPI {
|
|
281
|
+
constructor(client) {
|
|
282
|
+
this.client = client;
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Create a new trace with automatic context propagation
|
|
286
|
+
*
|
|
287
|
+
* @example
|
|
288
|
+
* ```typescript
|
|
289
|
+
* const trace = await client.traces.create({
|
|
290
|
+
* name: 'User Query',
|
|
291
|
+
* traceId: 'trace-123',
|
|
292
|
+
* metadata: { userId: '456' }
|
|
293
|
+
* });
|
|
294
|
+
* ```
|
|
295
|
+
*/
|
|
296
|
+
async create(params) {
|
|
297
|
+
const orgId = params.organizationId || this.client.getOrganizationId();
|
|
298
|
+
if (!orgId) {
|
|
299
|
+
throw new errors_1.EvalAIError('Organization ID is required', 'MISSING_ORGANIZATION_ID', 0);
|
|
300
|
+
}
|
|
301
|
+
// Merge with context
|
|
302
|
+
const metadata = (0, context_1.mergeWithContext)(params.metadata || {});
|
|
303
|
+
return this.client.request('/api/traces', {
|
|
304
|
+
method: 'POST',
|
|
305
|
+
body: JSON.stringify({ ...params, organizationId: orgId, metadata }),
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* List traces with optional filtering
|
|
310
|
+
*/
|
|
311
|
+
async list(params = {}) {
|
|
312
|
+
const searchParams = new URLSearchParams();
|
|
313
|
+
if (params.limit)
|
|
314
|
+
searchParams.set('limit', params.limit.toString());
|
|
315
|
+
if (params.offset)
|
|
316
|
+
searchParams.set('offset', params.offset.toString());
|
|
317
|
+
if (params.organizationId)
|
|
318
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
319
|
+
if (params.status)
|
|
320
|
+
searchParams.set('status', params.status);
|
|
321
|
+
if (params.search)
|
|
322
|
+
searchParams.set('search', params.search);
|
|
323
|
+
const query = searchParams.toString();
|
|
324
|
+
const endpoint = query ? `/api/traces?${query}` : '/api/traces';
|
|
325
|
+
return this.client.request(endpoint);
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Delete a trace by ID
|
|
329
|
+
*/
|
|
330
|
+
async delete(id) {
|
|
331
|
+
return this.client.request(`/api/traces?id=${id}`, {
|
|
332
|
+
method: 'DELETE',
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Get a single trace by ID
|
|
337
|
+
*/
|
|
338
|
+
async get(id) {
|
|
339
|
+
return this.client.request(`/api/traces/${id}`);
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Create a span for a trace
|
|
343
|
+
*/
|
|
344
|
+
async createSpan(traceId, params) {
|
|
345
|
+
return this.client.request(`/api/traces/${traceId}/spans`, {
|
|
346
|
+
method: 'POST',
|
|
347
|
+
body: JSON.stringify(params),
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* List spans for a trace
|
|
352
|
+
*/
|
|
353
|
+
async listSpans(traceId) {
|
|
354
|
+
return this.client.request(`/api/traces/${traceId}/spans`);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Evaluation API methods
|
|
359
|
+
*/
|
|
360
|
+
class EvaluationAPI {
|
|
361
|
+
constructor(client) {
|
|
362
|
+
this.client = client;
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Create a new evaluation
|
|
366
|
+
*/
|
|
367
|
+
async create(params) {
|
|
368
|
+
const orgId = params.organizationId || this.client.getOrganizationId();
|
|
369
|
+
if (!orgId) {
|
|
370
|
+
throw new errors_1.EvalAIError('Organization ID is required', 'MISSING_ORGANIZATION_ID', 0);
|
|
371
|
+
}
|
|
372
|
+
return this.client.request('/api/evaluations', {
|
|
373
|
+
method: 'POST',
|
|
374
|
+
body: JSON.stringify({ ...params, organizationId: orgId }),
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Get a single evaluation by ID
|
|
379
|
+
*/
|
|
380
|
+
async get(id) {
|
|
381
|
+
return this.client.request(`/api/evaluations?id=${id}`);
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* List evaluations with optional filtering
|
|
385
|
+
*/
|
|
386
|
+
async list(params = {}) {
|
|
387
|
+
const searchParams = new URLSearchParams();
|
|
388
|
+
if (params.limit)
|
|
389
|
+
searchParams.set('limit', params.limit.toString());
|
|
390
|
+
if (params.offset)
|
|
391
|
+
searchParams.set('offset', params.offset.toString());
|
|
392
|
+
if (params.organizationId)
|
|
393
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
394
|
+
if (params.type)
|
|
395
|
+
searchParams.set('type', params.type);
|
|
396
|
+
if (params.status)
|
|
397
|
+
searchParams.set('status', params.status);
|
|
398
|
+
if (params.search)
|
|
399
|
+
searchParams.set('search', params.search);
|
|
400
|
+
const query = searchParams.toString();
|
|
401
|
+
const endpoint = query ? `/api/evaluations?${query}` : '/api/evaluations';
|
|
402
|
+
return this.client.request(endpoint);
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Update an evaluation
|
|
406
|
+
*/
|
|
407
|
+
async update(id, params) {
|
|
408
|
+
return this.client.request(`/api/evaluations?id=${id}`, {
|
|
409
|
+
method: 'PUT',
|
|
410
|
+
body: JSON.stringify(params),
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Delete an evaluation
|
|
415
|
+
*/
|
|
416
|
+
async delete(id) {
|
|
417
|
+
return this.client.request(`/api/evaluations?id=${id}`, {
|
|
418
|
+
method: 'DELETE',
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Create a test case for an evaluation
|
|
423
|
+
*/
|
|
424
|
+
async createTestCase(evaluationId, params) {
|
|
425
|
+
return this.client.request(`/api/evaluations/${evaluationId}/test-cases`, {
|
|
426
|
+
method: 'POST',
|
|
427
|
+
body: JSON.stringify(params),
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* List test cases for an evaluation
|
|
432
|
+
*/
|
|
433
|
+
async listTestCases(evaluationId) {
|
|
434
|
+
return this.client.request(`/api/evaluations/${evaluationId}/test-cases`);
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Create a run for an evaluation
|
|
438
|
+
*/
|
|
439
|
+
async createRun(evaluationId, params) {
|
|
440
|
+
return this.client.request(`/api/evaluations/${evaluationId}/runs`, {
|
|
441
|
+
method: 'POST',
|
|
442
|
+
body: JSON.stringify(params),
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* List runs for an evaluation
|
|
447
|
+
*/
|
|
448
|
+
async listRuns(evaluationId) {
|
|
449
|
+
return this.client.request(`/api/evaluations/${evaluationId}/runs`);
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* Get a specific run
|
|
453
|
+
*/
|
|
454
|
+
async getRun(evaluationId, runId) {
|
|
455
|
+
return this.client.request(`/api/evaluations/${evaluationId}/runs/${runId}`);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
/**
|
|
459
|
+
* LLM Judge API methods
|
|
460
|
+
*/
|
|
461
|
+
class LLMJudgeAPI {
|
|
462
|
+
constructor(client) {
|
|
463
|
+
this.client = client;
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Run an LLM judge evaluation
|
|
467
|
+
*/
|
|
468
|
+
async evaluate(params) {
|
|
469
|
+
return this.client.request('/api/llm-judge/evaluate', {
|
|
470
|
+
method: 'POST',
|
|
471
|
+
body: JSON.stringify(params),
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
/**
|
|
475
|
+
* Create an LLM judge configuration
|
|
476
|
+
*/
|
|
477
|
+
async createConfig(params) {
|
|
478
|
+
return this.client.request('/api/llm-judge/configs', {
|
|
479
|
+
method: 'POST',
|
|
480
|
+
body: JSON.stringify(params),
|
|
481
|
+
});
|
|
482
|
+
}
|
|
483
|
+
/**
|
|
484
|
+
* List LLM judge configurations
|
|
485
|
+
*/
|
|
486
|
+
async listConfigs(params = {}) {
|
|
487
|
+
const searchParams = new URLSearchParams();
|
|
488
|
+
if (params.organizationId)
|
|
489
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
490
|
+
if (params.limit)
|
|
491
|
+
searchParams.set('limit', params.limit.toString());
|
|
492
|
+
if (params.offset)
|
|
493
|
+
searchParams.set('offset', params.offset.toString());
|
|
494
|
+
const query = searchParams.toString();
|
|
495
|
+
const endpoint = query ? `/api/llm-judge/configs?${query}` : '/api/llm-judge/configs';
|
|
496
|
+
return this.client.request(endpoint);
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* List LLM judge results
|
|
500
|
+
*/
|
|
501
|
+
async listResults(params = {}) {
|
|
502
|
+
const searchParams = new URLSearchParams();
|
|
503
|
+
if (params.configId)
|
|
504
|
+
searchParams.set('configId', params.configId.toString());
|
|
505
|
+
if (params.evaluationId)
|
|
506
|
+
searchParams.set('evaluationId', params.evaluationId.toString());
|
|
507
|
+
if (params.limit)
|
|
508
|
+
searchParams.set('limit', params.limit.toString());
|
|
509
|
+
if (params.offset)
|
|
510
|
+
searchParams.set('offset', params.offset.toString());
|
|
511
|
+
const query = searchParams.toString();
|
|
512
|
+
const endpoint = query ? `/api/llm-judge/results?${query}` : '/api/llm-judge/results';
|
|
513
|
+
return this.client.request(endpoint);
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* Get alignment analysis
|
|
517
|
+
*/
|
|
518
|
+
async getAlignment(params) {
|
|
519
|
+
const searchParams = new URLSearchParams();
|
|
520
|
+
searchParams.set('configId', params.configId.toString());
|
|
521
|
+
if (params.startDate)
|
|
522
|
+
searchParams.set('startDate', params.startDate);
|
|
523
|
+
if (params.endDate)
|
|
524
|
+
searchParams.set('endDate', params.endDate);
|
|
525
|
+
const query = searchParams.toString();
|
|
526
|
+
return this.client.request(`/api/llm-judge/alignment?${query}`);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Annotations API methods
|
|
531
|
+
*/
|
|
532
|
+
class AnnotationsAPI {
|
|
533
|
+
constructor(client) {
|
|
534
|
+
this.client = client;
|
|
535
|
+
this.tasks = new AnnotationTasksAPI(client);
|
|
536
|
+
}
|
|
537
|
+
/**
|
|
538
|
+
* Create an annotation
|
|
539
|
+
*/
|
|
540
|
+
async create(params) {
|
|
541
|
+
return this.client.request('/api/annotations', {
|
|
542
|
+
method: 'POST',
|
|
543
|
+
body: JSON.stringify(params),
|
|
544
|
+
}).then(res => res.annotation);
|
|
545
|
+
}
|
|
546
|
+
/**
|
|
547
|
+
* List annotations
|
|
548
|
+
*/
|
|
549
|
+
async list(params = {}) {
|
|
550
|
+
const searchParams = new URLSearchParams();
|
|
551
|
+
if (params.evaluationRunId)
|
|
552
|
+
searchParams.set('evaluationRunId', params.evaluationRunId.toString());
|
|
553
|
+
if (params.testCaseId)
|
|
554
|
+
searchParams.set('testCaseId', params.testCaseId.toString());
|
|
555
|
+
if (params.limit)
|
|
556
|
+
searchParams.set('limit', params.limit.toString());
|
|
557
|
+
if (params.offset)
|
|
558
|
+
searchParams.set('offset', params.offset.toString());
|
|
559
|
+
const query = searchParams.toString();
|
|
560
|
+
const endpoint = query ? `/api/annotations?${query}` : '/api/annotations';
|
|
561
|
+
return this.client.request(endpoint).then(res => res.annotations);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
/**
|
|
565
|
+
* Annotation Tasks API methods
|
|
566
|
+
*/
|
|
567
|
+
class AnnotationTasksAPI {
|
|
568
|
+
constructor(client) {
|
|
569
|
+
this.client = client;
|
|
570
|
+
this.items = new AnnotationTaskItemsAPI(client);
|
|
571
|
+
}
|
|
572
|
+
/**
|
|
573
|
+
* Create an annotation task
|
|
574
|
+
*/
|
|
575
|
+
async create(params) {
|
|
576
|
+
return this.client.request('/api/annotations/tasks', {
|
|
577
|
+
method: 'POST',
|
|
578
|
+
body: JSON.stringify(params),
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* List annotation tasks
|
|
583
|
+
*/
|
|
584
|
+
async list(params = {}) {
|
|
585
|
+
const searchParams = new URLSearchParams();
|
|
586
|
+
if (params.organizationId)
|
|
587
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
588
|
+
if (params.status)
|
|
589
|
+
searchParams.set('status', params.status);
|
|
590
|
+
if (params.limit)
|
|
591
|
+
searchParams.set('limit', params.limit.toString());
|
|
592
|
+
if (params.offset)
|
|
593
|
+
searchParams.set('offset', params.offset.toString());
|
|
594
|
+
const query = searchParams.toString();
|
|
595
|
+
const endpoint = query ? `/api/annotations/tasks?${query}` : '/api/annotations/tasks';
|
|
596
|
+
return this.client.request(endpoint);
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Get an annotation task
|
|
600
|
+
*/
|
|
601
|
+
async get(taskId) {
|
|
602
|
+
return this.client.request(`/api/annotations/tasks/${taskId}`);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Annotation Task Items API methods
|
|
607
|
+
*/
|
|
608
|
+
class AnnotationTaskItemsAPI {
|
|
609
|
+
constructor(client) {
|
|
610
|
+
this.client = client;
|
|
611
|
+
}
|
|
612
|
+
/**
|
|
613
|
+
* Create an annotation item
|
|
614
|
+
*/
|
|
615
|
+
async create(taskId, params) {
|
|
616
|
+
return this.client.request(`/api/annotations/tasks/${taskId}/items`, {
|
|
617
|
+
method: 'POST',
|
|
618
|
+
body: JSON.stringify(params),
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* List annotation items
|
|
623
|
+
*/
|
|
624
|
+
async list(taskId, params = {}) {
|
|
625
|
+
const searchParams = new URLSearchParams();
|
|
626
|
+
if (params.limit)
|
|
627
|
+
searchParams.set('limit', params.limit.toString());
|
|
628
|
+
if (params.offset)
|
|
629
|
+
searchParams.set('offset', params.offset.toString());
|
|
630
|
+
const query = searchParams.toString();
|
|
631
|
+
const endpoint = query ? `/api/annotations/tasks/${taskId}/items?${query}` : `/api/annotations/tasks/${taskId}/items`;
|
|
632
|
+
return this.client.request(endpoint);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* Developer API methods
|
|
637
|
+
*/
|
|
638
|
+
class DeveloperAPI {
|
|
639
|
+
constructor(client) {
|
|
640
|
+
this.client = client;
|
|
641
|
+
this.apiKeys = new APIKeysAPI(client);
|
|
642
|
+
this.webhooks = new WebhooksAPI(client);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Get usage statistics
|
|
646
|
+
*/
|
|
647
|
+
async getUsage(params) {
|
|
648
|
+
const searchParams = new URLSearchParams();
|
|
649
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
650
|
+
if (params.startDate)
|
|
651
|
+
searchParams.set('startDate', params.startDate);
|
|
652
|
+
if (params.endDate)
|
|
653
|
+
searchParams.set('endDate', params.endDate);
|
|
654
|
+
const query = searchParams.toString();
|
|
655
|
+
return this.client.request(`/api/developer/usage?${query}`);
|
|
656
|
+
}
|
|
657
|
+
/**
|
|
658
|
+
* Get usage summary
|
|
659
|
+
*/
|
|
660
|
+
async getUsageSummary(organizationId) {
|
|
661
|
+
return this.client.request(`/api/developer/usage/summary?organizationId=${organizationId}`);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* API Keys API methods
|
|
666
|
+
*/
|
|
667
|
+
class APIKeysAPI {
|
|
668
|
+
constructor(client) {
|
|
669
|
+
this.client = client;
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* Create an API key
|
|
673
|
+
*/
|
|
674
|
+
async create(params) {
|
|
675
|
+
return this.client.request('/api/developer/api-keys', {
|
|
676
|
+
method: 'POST',
|
|
677
|
+
body: JSON.stringify(params),
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
/**
|
|
681
|
+
* List API keys
|
|
682
|
+
*/
|
|
683
|
+
async list(params = {}) {
|
|
684
|
+
const searchParams = new URLSearchParams();
|
|
685
|
+
if (params.organizationId)
|
|
686
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
687
|
+
if (params.limit)
|
|
688
|
+
searchParams.set('limit', params.limit.toString());
|
|
689
|
+
if (params.offset)
|
|
690
|
+
searchParams.set('offset', params.offset.toString());
|
|
691
|
+
const query = searchParams.toString();
|
|
692
|
+
const endpoint = query ? `/api/developer/api-keys?${query}` : '/api/developer/api-keys';
|
|
693
|
+
return this.client.request(endpoint);
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Update an API key
|
|
697
|
+
*/
|
|
698
|
+
async update(keyId, params) {
|
|
699
|
+
return this.client.request(`/api/developer/api-keys/${keyId}`, {
|
|
700
|
+
method: 'PATCH',
|
|
701
|
+
body: JSON.stringify(params),
|
|
702
|
+
});
|
|
703
|
+
}
|
|
704
|
+
/**
|
|
705
|
+
* Revoke an API key
|
|
706
|
+
*/
|
|
707
|
+
async revoke(keyId) {
|
|
708
|
+
return this.client.request(`/api/developer/api-keys/${keyId}`, {
|
|
709
|
+
method: 'DELETE',
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Get API key usage
|
|
714
|
+
*/
|
|
715
|
+
async getUsage(keyId) {
|
|
716
|
+
return this.client.request(`/api/developer/api-keys/${keyId}/usage`);
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* Webhooks API methods
|
|
721
|
+
*/
|
|
722
|
+
class WebhooksAPI {
|
|
723
|
+
constructor(client) {
|
|
724
|
+
this.client = client;
|
|
725
|
+
}
|
|
726
|
+
/**
|
|
727
|
+
* Create a webhook
|
|
728
|
+
*/
|
|
729
|
+
async create(params) {
|
|
730
|
+
return this.client.request('/api/developer/webhooks', {
|
|
731
|
+
method: 'POST',
|
|
732
|
+
body: JSON.stringify(params),
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
/**
|
|
736
|
+
* List webhooks
|
|
737
|
+
*/
|
|
738
|
+
async list(params) {
|
|
739
|
+
const searchParams = new URLSearchParams();
|
|
740
|
+
searchParams.set('organizationId', params.organizationId.toString());
|
|
741
|
+
if (params.status)
|
|
742
|
+
searchParams.set('status', params.status);
|
|
743
|
+
if (params.limit)
|
|
744
|
+
searchParams.set('limit', params.limit.toString());
|
|
745
|
+
if (params.offset)
|
|
746
|
+
searchParams.set('offset', params.offset.toString());
|
|
747
|
+
const query = searchParams.toString();
|
|
748
|
+
return this.client.request(`/api/developer/webhooks?${query}`);
|
|
749
|
+
}
|
|
750
|
+
/**
|
|
751
|
+
* Get a webhook
|
|
752
|
+
*/
|
|
753
|
+
async get(webhookId) {
|
|
754
|
+
return this.client.request(`/api/developer/webhooks/${webhookId}`);
|
|
755
|
+
}
|
|
756
|
+
/**
|
|
757
|
+
* Update a webhook
|
|
758
|
+
*/
|
|
759
|
+
async update(webhookId, params) {
|
|
760
|
+
return this.client.request(`/api/developer/webhooks/${webhookId}`, {
|
|
761
|
+
method: 'PATCH',
|
|
762
|
+
body: JSON.stringify(params),
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
/**
|
|
766
|
+
* Delete a webhook
|
|
767
|
+
*/
|
|
768
|
+
async delete(webhookId) {
|
|
769
|
+
return this.client.request(`/api/developer/webhooks/${webhookId}`, {
|
|
770
|
+
method: 'DELETE',
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
/**
|
|
774
|
+
* Get webhook deliveries
|
|
775
|
+
*/
|
|
776
|
+
async getDeliveries(webhookId, params = {}) {
|
|
777
|
+
const searchParams = new URLSearchParams();
|
|
778
|
+
if (params.limit)
|
|
779
|
+
searchParams.set('limit', params.limit.toString());
|
|
780
|
+
if (params.offset)
|
|
781
|
+
searchParams.set('offset', params.offset.toString());
|
|
782
|
+
if (params.success !== undefined)
|
|
783
|
+
searchParams.set('success', params.success.toString());
|
|
784
|
+
const query = searchParams.toString();
|
|
785
|
+
const endpoint = query ? `/api/developer/webhooks/${webhookId}/deliveries?${query}` : `/api/developer/webhooks/${webhookId}/deliveries`;
|
|
786
|
+
return this.client.request(endpoint);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
/**
|
|
790
|
+
* Organizations API methods
|
|
791
|
+
*/
|
|
792
|
+
class OrganizationsAPI {
|
|
793
|
+
constructor(client) {
|
|
794
|
+
this.client = client;
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Get current organization
|
|
798
|
+
*/
|
|
799
|
+
async getCurrent() {
|
|
800
|
+
return this.client.request('/api/organizations/current');
|
|
801
|
+
}
|
|
802
|
+
}
|