@pauly4010/evalai-sdk 1.4.1 → 1.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +205 -543
- package/dist/assertions.d.ts +2 -2
- package/dist/assertions.js +104 -71
- package/dist/batch.js +12 -17
- package/dist/cache.js +7 -11
- package/dist/cli/api.d.ts +108 -0
- package/dist/cli/api.js +130 -0
- package/dist/cli/check.d.ts +28 -13
- package/dist/cli/check.js +249 -142
- package/dist/cli/ci-context.d.ts +6 -0
- package/dist/cli/ci-context.js +110 -0
- package/dist/cli/config.d.ts +30 -0
- package/dist/cli/config.js +207 -0
- package/dist/cli/constants.d.ts +15 -0
- package/dist/cli/constants.js +18 -0
- package/dist/cli/doctor.d.ts +11 -0
- package/dist/cli/doctor.js +82 -0
- package/dist/cli/formatters/github.d.ts +8 -0
- package/dist/cli/formatters/github.js +130 -0
- package/dist/cli/formatters/human.d.ts +6 -0
- package/dist/cli/formatters/human.js +107 -0
- package/dist/cli/formatters/json.d.ts +6 -0
- package/dist/cli/formatters/json.js +10 -0
- package/dist/cli/formatters/pr-comment.d.ts +12 -0
- package/dist/cli/formatters/pr-comment.js +101 -0
- package/dist/cli/formatters/types.d.ts +100 -0
- package/dist/cli/formatters/types.js +5 -0
- package/dist/cli/gate.d.ts +21 -0
- package/dist/cli/gate.js +175 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +67 -23
- package/dist/cli/init.d.ts +7 -0
- package/dist/cli/init.js +69 -0
- package/dist/cli/policy-packs.d.ts +23 -0
- package/dist/cli/policy-packs.js +83 -0
- package/dist/cli/profiles.d.ts +28 -0
- package/dist/cli/profiles.js +30 -0
- package/dist/cli/reason-codes.d.ts +17 -0
- package/dist/cli/reason-codes.js +19 -0
- package/dist/cli/render/snippet.d.ts +5 -0
- package/dist/cli/render/snippet.js +15 -0
- package/dist/cli/render/sort.d.ts +10 -0
- package/dist/cli/render/sort.js +24 -0
- package/dist/cli/report/build-check-report.d.ts +19 -0
- package/dist/cli/report/build-check-report.js +124 -0
- package/dist/cli/share.d.ts +17 -0
- package/dist/cli/share.js +83 -0
- package/dist/client.d.ts +2 -2
- package/dist/client.js +144 -132
- package/dist/context.d.ts +1 -1
- package/dist/context.js +4 -6
- package/dist/errors.d.ts +2 -0
- package/dist/errors.js +116 -107
- package/dist/export.d.ts +6 -6
- package/dist/export.js +39 -33
- package/dist/index.d.ts +25 -24
- package/dist/index.js +62 -56
- package/dist/integrations/anthropic.d.ts +1 -1
- package/dist/integrations/anthropic.js +23 -19
- package/dist/integrations/openai-eval.d.ts +57 -0
- package/dist/integrations/openai-eval.js +230 -0
- package/dist/integrations/openai.d.ts +1 -1
- package/dist/integrations/openai.js +23 -19
- package/dist/local.d.ts +2 -2
- package/dist/local.js +25 -25
- package/dist/logger.d.ts +1 -1
- package/dist/logger.js +24 -28
- package/dist/matchers/index.d.ts +1 -0
- package/dist/matchers/index.js +6 -0
- package/dist/matchers/to-pass-gate.d.ts +29 -0
- package/dist/matchers/to-pass-gate.js +35 -0
- package/dist/pagination.d.ts +1 -1
- package/dist/pagination.js +6 -6
- package/dist/snapshot.js +24 -24
- package/dist/streaming.js +11 -11
- package/dist/testing.d.ts +6 -2
- package/dist/testing.js +30 -12
- package/dist/types.d.ts +22 -22
- package/dist/types.js +13 -13
- package/dist/utils/input-hash.d.ts +8 -0
- package/dist/utils/input-hash.js +38 -0
- package/dist/version.d.ts +7 -0
- package/dist/version.js +10 -0
- package/dist/workflows.d.ts +7 -7
- package/dist/workflows.js +44 -44
- package/package.json +102 -90
- package/dist/__tests__/assertions.test.d.ts +0 -1
- package/dist/__tests__/assertions.test.js +0 -288
- package/dist/__tests__/client.test.d.ts +0 -1
- package/dist/__tests__/client.test.js +0 -185
- package/dist/__tests__/testing.test.d.ts +0 -1
- package/dist/__tests__/testing.test.js +0 -230
- package/dist/__tests__/workflows.test.d.ts +0 -1
- package/dist/__tests__/workflows.test.js +0 -222
package/dist/client.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.AIEvalClient = void 0;
|
|
4
|
+
const batch_1 = require("./batch");
|
|
5
|
+
const cache_1 = require("./cache");
|
|
6
|
+
const context_1 = require("./context");
|
|
4
7
|
const errors_1 = require("./errors");
|
|
5
8
|
const logger_1 = require("./logger");
|
|
6
|
-
const
|
|
7
|
-
const cache_1 = require("./cache");
|
|
8
|
-
const batch_1 = require("./batch");
|
|
9
|
+
const version_1 = require("./version");
|
|
9
10
|
/**
|
|
10
11
|
* Safe environment variable access (works in both Node.js and browsers)
|
|
11
12
|
*/
|
|
12
13
|
function getEnvVar(name) {
|
|
13
|
-
if (typeof process !==
|
|
14
|
+
if (typeof process !== "undefined" && process.env) {
|
|
14
15
|
return process.env[name];
|
|
15
16
|
}
|
|
16
17
|
return undefined;
|
|
@@ -42,35 +43,37 @@ function getEnvVar(name) {
|
|
|
42
43
|
class AIEvalClient {
|
|
43
44
|
constructor(config = {}) {
|
|
44
45
|
// Tier 1.1: Zero-config with env variable detection (works in Node.js and browsers)
|
|
45
|
-
this.apiKey =
|
|
46
|
+
this.apiKey =
|
|
47
|
+
config.apiKey || getEnvVar("EVALAI_API_KEY") || getEnvVar("AI_EVAL_API_KEY") || "";
|
|
46
48
|
if (!this.apiKey) {
|
|
47
|
-
throw new errors_1.EvalAIError(
|
|
49
|
+
throw new errors_1.EvalAIError("API key is required. Provide via config.apiKey or EVALAI_API_KEY environment variable.", "MISSING_API_KEY", 0);
|
|
48
50
|
}
|
|
49
51
|
// Auto-detect organization ID from env
|
|
50
|
-
const orgIdFromEnv = getEnvVar(
|
|
51
|
-
this.organizationId =
|
|
52
|
+
const orgIdFromEnv = getEnvVar("EVALAI_ORGANIZATION_ID") || getEnvVar("AI_EVAL_ORGANIZATION_ID");
|
|
53
|
+
this.organizationId =
|
|
54
|
+
config.organizationId || (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
|
|
52
55
|
// Default to relative URLs for browser, or allow custom baseUrl
|
|
53
|
-
const isBrowser = typeof globalThis.window !==
|
|
54
|
-
this.baseUrl = config.baseUrl || (isBrowser ?
|
|
56
|
+
const isBrowser = typeof globalThis.window !== "undefined";
|
|
57
|
+
this.baseUrl = config.baseUrl || (isBrowser ? "" : "http://localhost:3000");
|
|
55
58
|
this.timeout = config.timeout || 30000;
|
|
56
59
|
// Tier 4.17: Debug mode with request logging
|
|
57
|
-
const logLevel = config.logLevel || (config.debug ?
|
|
60
|
+
const logLevel = config.logLevel || (config.debug ? "debug" : "info");
|
|
58
61
|
this.logger = (0, logger_1.createLogger)({
|
|
59
62
|
level: logLevel,
|
|
60
63
|
pretty: config.debug,
|
|
61
|
-
prefix:
|
|
64
|
+
prefix: "EvalAI",
|
|
62
65
|
});
|
|
63
66
|
this.requestLogger = new logger_1.RequestLogger(this.logger);
|
|
64
67
|
// Retry configuration
|
|
65
68
|
this.retryConfig = {
|
|
66
69
|
maxAttempts: config.retry?.maxAttempts || 3,
|
|
67
|
-
backoff: config.retry?.backoff ||
|
|
70
|
+
backoff: config.retry?.backoff || "exponential",
|
|
68
71
|
retryableErrors: config.retry?.retryableErrors || [
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
]
|
|
72
|
+
"RATE_LIMIT_EXCEEDED",
|
|
73
|
+
"TIMEOUT",
|
|
74
|
+
"NETWORK_ERROR",
|
|
75
|
+
"INTERNAL_SERVER_ERROR",
|
|
76
|
+
],
|
|
74
77
|
};
|
|
75
78
|
// Initialize cache for GET requests
|
|
76
79
|
this.cache = new cache_1.RequestCache(config.cacheSize || 1000);
|
|
@@ -95,7 +98,7 @@ class AIEvalClient {
|
|
|
95
98
|
id: req.id,
|
|
96
99
|
status: err?.statusCode || 500,
|
|
97
100
|
data: null,
|
|
98
|
-
error: err?.message ||
|
|
101
|
+
error: err?.message || "Unknown error",
|
|
99
102
|
});
|
|
100
103
|
}
|
|
101
104
|
})();
|
|
@@ -130,9 +133,9 @@ class AIEvalClient {
|
|
|
130
133
|
this.annotations = new AnnotationsAPI(this);
|
|
131
134
|
this.developer = new DeveloperAPI(this);
|
|
132
135
|
this.organizations = new OrganizationsAPI(this);
|
|
133
|
-
this.logger.info(
|
|
136
|
+
this.logger.info("SDK initialized", {
|
|
134
137
|
hasOrganizationId: !!this.organizationId,
|
|
135
|
-
baseUrl: this.baseUrl
|
|
138
|
+
baseUrl: this.baseUrl,
|
|
136
139
|
});
|
|
137
140
|
}
|
|
138
141
|
/**
|
|
@@ -162,21 +165,21 @@ class AIEvalClient {
|
|
|
162
165
|
*/
|
|
163
166
|
static init(config = {}) {
|
|
164
167
|
return new AIEvalClient({
|
|
165
|
-
baseUrl: getEnvVar(
|
|
166
|
-
...config
|
|
168
|
+
baseUrl: getEnvVar("EVALAI_BASE_URL"),
|
|
169
|
+
...config,
|
|
167
170
|
});
|
|
168
171
|
}
|
|
169
172
|
/**
|
|
170
173
|
* Internal method to make HTTP requests with retry logic and error handling
|
|
171
174
|
*/
|
|
172
175
|
async request(endpoint, options = {}, attempt = 1) {
|
|
173
|
-
const method = (options.method ||
|
|
176
|
+
const method = (options.method || "GET").toUpperCase();
|
|
174
177
|
const url = `${this.baseUrl}${endpoint}`;
|
|
175
178
|
// Check cache for GET requests
|
|
176
|
-
if (method ===
|
|
179
|
+
if (method === "GET" && (0, cache_1.shouldCache)(method, endpoint)) {
|
|
177
180
|
const cached = this.cache.get(method, endpoint, options.body);
|
|
178
181
|
if (cached !== null) {
|
|
179
|
-
this.logger.debug(
|
|
182
|
+
this.logger.debug("Cache hit", { endpoint });
|
|
180
183
|
return cached;
|
|
181
184
|
}
|
|
182
185
|
}
|
|
@@ -185,17 +188,19 @@ class AIEvalClient {
|
|
|
185
188
|
const startTime = Date.now();
|
|
186
189
|
// Log request
|
|
187
190
|
this.requestLogger.logRequest({
|
|
188
|
-
method: options.method ||
|
|
191
|
+
method: options.method || "GET",
|
|
189
192
|
url,
|
|
190
193
|
headers: options.headers,
|
|
191
|
-
body: options.body
|
|
194
|
+
body: options.body,
|
|
192
195
|
});
|
|
193
196
|
try {
|
|
194
197
|
const response = await fetch(url, {
|
|
195
198
|
...options,
|
|
196
199
|
headers: {
|
|
197
|
-
|
|
198
|
-
|
|
200
|
+
"Content-Type": "application/json",
|
|
201
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
202
|
+
"X-EvalAI-SDK-Version": version_1.SDK_VERSION,
|
|
203
|
+
"X-EvalAI-Spec-Version": version_1.SPEC_VERSION,
|
|
199
204
|
...options.headers,
|
|
200
205
|
},
|
|
201
206
|
signal: controller.signal,
|
|
@@ -206,16 +211,16 @@ class AIEvalClient {
|
|
|
206
211
|
try {
|
|
207
212
|
data = await response.json();
|
|
208
213
|
}
|
|
209
|
-
catch (
|
|
214
|
+
catch (_e) {
|
|
210
215
|
data = {};
|
|
211
216
|
}
|
|
212
217
|
// Log response
|
|
213
218
|
this.requestLogger.logResponse({
|
|
214
|
-
method: options.method ||
|
|
219
|
+
method: options.method || "GET",
|
|
215
220
|
url,
|
|
216
221
|
status: response.status,
|
|
217
222
|
duration,
|
|
218
|
-
body: data
|
|
223
|
+
body: data,
|
|
219
224
|
});
|
|
220
225
|
if (!response.ok) {
|
|
221
226
|
const error = (0, errors_1.createErrorFromResponse)(response, data);
|
|
@@ -225,21 +230,21 @@ class AIEvalClient {
|
|
|
225
230
|
const delay = this.calculateBackoff(attempt);
|
|
226
231
|
this.logger.warn(`Retrying request (attempt ${attempt + 1}/${this.retryConfig.maxAttempts}) after ${delay}ms`, {
|
|
227
232
|
error: error.code,
|
|
228
|
-
url
|
|
233
|
+
url,
|
|
229
234
|
});
|
|
230
|
-
await new Promise(resolve => setTimeout(resolve, delay));
|
|
235
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
231
236
|
return this.request(endpoint, options, attempt + 1);
|
|
232
237
|
}
|
|
233
238
|
throw error;
|
|
234
239
|
}
|
|
235
240
|
// Cache successful GET responses
|
|
236
|
-
if (method ===
|
|
241
|
+
if (method === "GET" && (0, cache_1.shouldCache)(method, endpoint)) {
|
|
237
242
|
const ttl = (0, cache_1.getTTL)(endpoint);
|
|
238
243
|
this.cache.set(method, endpoint, data, ttl, options.body);
|
|
239
|
-
this.logger.debug(
|
|
244
|
+
this.logger.debug("Cached response", { endpoint, ttl });
|
|
240
245
|
}
|
|
241
246
|
// Invalidate cache for mutation operations
|
|
242
|
-
if ([
|
|
247
|
+
if (["POST", "PUT", "DELETE", "PATCH"].includes(method)) {
|
|
243
248
|
// Invalidate related cached entries
|
|
244
249
|
const resourceMatch = endpoint.match(/\/api\/(\w+)/);
|
|
245
250
|
if (resourceMatch) {
|
|
@@ -254,10 +259,10 @@ class AIEvalClient {
|
|
|
254
259
|
throw error;
|
|
255
260
|
}
|
|
256
261
|
if (error instanceof Error) {
|
|
257
|
-
if (error.name ===
|
|
258
|
-
throw new errors_1.EvalAIError(
|
|
262
|
+
if (error.name === "AbortError") {
|
|
263
|
+
throw new errors_1.EvalAIError("Request timeout", "TIMEOUT", 408);
|
|
259
264
|
}
|
|
260
|
-
throw new errors_1.EvalAIError(error.message,
|
|
265
|
+
throw new errors_1.EvalAIError(error.message, "NETWORK_ERROR", 0);
|
|
261
266
|
}
|
|
262
267
|
throw error;
|
|
263
268
|
}
|
|
@@ -268,11 +273,10 @@ class AIEvalClient {
|
|
|
268
273
|
calculateBackoff(attempt) {
|
|
269
274
|
const baseDelay = 1000; // 1 second
|
|
270
275
|
switch (this.retryConfig.backoff) {
|
|
271
|
-
case
|
|
272
|
-
return baseDelay *
|
|
273
|
-
case
|
|
276
|
+
case "exponential":
|
|
277
|
+
return baseDelay * 2 ** (attempt - 1);
|
|
278
|
+
case "linear":
|
|
274
279
|
return baseDelay * attempt;
|
|
275
|
-
case 'fixed':
|
|
276
280
|
default:
|
|
277
281
|
return baseDelay;
|
|
278
282
|
}
|
|
@@ -300,7 +304,7 @@ class AIEvalClient {
|
|
|
300
304
|
async getOrganizationLimits() {
|
|
301
305
|
const orgId = this.getOrganizationId();
|
|
302
306
|
if (!orgId) {
|
|
303
|
-
throw new errors_1.EvalAIError(
|
|
307
|
+
throw new errors_1.EvalAIError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
|
|
304
308
|
}
|
|
305
309
|
return this.request(`/api/organizations/${orgId}/limits`);
|
|
306
310
|
}
|
|
@@ -328,12 +332,12 @@ class TraceAPI {
|
|
|
328
332
|
async create(params) {
|
|
329
333
|
const orgId = params.organizationId || this.client.getOrganizationId();
|
|
330
334
|
if (!orgId) {
|
|
331
|
-
throw new errors_1.EvalAIError(
|
|
335
|
+
throw new errors_1.EvalAIError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
|
|
332
336
|
}
|
|
333
337
|
// Merge with context
|
|
334
338
|
const metadata = (0, context_1.mergeWithContext)(params.metadata || {});
|
|
335
|
-
return this.client.request(
|
|
336
|
-
method:
|
|
339
|
+
return this.client.request("/api/traces", {
|
|
340
|
+
method: "POST",
|
|
337
341
|
body: JSON.stringify({ ...params, organizationId: orgId, metadata }),
|
|
338
342
|
});
|
|
339
343
|
}
|
|
@@ -343,17 +347,17 @@ class TraceAPI {
|
|
|
343
347
|
async list(params = {}) {
|
|
344
348
|
const searchParams = new URLSearchParams();
|
|
345
349
|
if (params.limit)
|
|
346
|
-
searchParams.set(
|
|
350
|
+
searchParams.set("limit", params.limit.toString());
|
|
347
351
|
if (params.offset)
|
|
348
|
-
searchParams.set(
|
|
352
|
+
searchParams.set("offset", params.offset.toString());
|
|
349
353
|
if (params.organizationId)
|
|
350
|
-
searchParams.set(
|
|
354
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
351
355
|
if (params.status)
|
|
352
|
-
searchParams.set(
|
|
356
|
+
searchParams.set("status", params.status);
|
|
353
357
|
if (params.search)
|
|
354
|
-
searchParams.set(
|
|
358
|
+
searchParams.set("search", params.search);
|
|
355
359
|
const query = searchParams.toString();
|
|
356
|
-
const endpoint = query ? `/api/traces?${query}` :
|
|
360
|
+
const endpoint = query ? `/api/traces?${query}` : "/api/traces";
|
|
357
361
|
return this.client.request(endpoint);
|
|
358
362
|
}
|
|
359
363
|
/**
|
|
@@ -361,7 +365,7 @@ class TraceAPI {
|
|
|
361
365
|
*/
|
|
362
366
|
async delete(id) {
|
|
363
367
|
return this.client.request(`/api/traces?id=${id}`, {
|
|
364
|
-
method:
|
|
368
|
+
method: "DELETE",
|
|
365
369
|
});
|
|
366
370
|
}
|
|
367
371
|
/**
|
|
@@ -384,7 +388,7 @@ class TraceAPI {
|
|
|
384
388
|
*/
|
|
385
389
|
async update(id, params) {
|
|
386
390
|
return this.client.request(`/api/traces/${id}`, {
|
|
387
|
-
method:
|
|
391
|
+
method: "PATCH",
|
|
388
392
|
body: JSON.stringify(params),
|
|
389
393
|
});
|
|
390
394
|
}
|
|
@@ -393,7 +397,7 @@ class TraceAPI {
|
|
|
393
397
|
*/
|
|
394
398
|
async createSpan(traceId, params) {
|
|
395
399
|
return this.client.request(`/api/traces/${traceId}/spans`, {
|
|
396
|
-
method:
|
|
400
|
+
method: "POST",
|
|
397
401
|
body: JSON.stringify(params),
|
|
398
402
|
});
|
|
399
403
|
}
|
|
@@ -417,10 +421,10 @@ class EvaluationAPI {
|
|
|
417
421
|
async create(params) {
|
|
418
422
|
const orgId = params.organizationId || this.client.getOrganizationId();
|
|
419
423
|
if (!orgId) {
|
|
420
|
-
throw new errors_1.EvalAIError(
|
|
424
|
+
throw new errors_1.EvalAIError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
|
|
421
425
|
}
|
|
422
|
-
return this.client.request(
|
|
423
|
-
method:
|
|
426
|
+
return this.client.request("/api/evaluations", {
|
|
427
|
+
method: "POST",
|
|
424
428
|
body: JSON.stringify({ ...params, organizationId: orgId }),
|
|
425
429
|
});
|
|
426
430
|
}
|
|
@@ -436,19 +440,19 @@ class EvaluationAPI {
|
|
|
436
440
|
async list(params = {}) {
|
|
437
441
|
const searchParams = new URLSearchParams();
|
|
438
442
|
if (params.limit)
|
|
439
|
-
searchParams.set(
|
|
443
|
+
searchParams.set("limit", params.limit.toString());
|
|
440
444
|
if (params.offset)
|
|
441
|
-
searchParams.set(
|
|
445
|
+
searchParams.set("offset", params.offset.toString());
|
|
442
446
|
if (params.organizationId)
|
|
443
|
-
searchParams.set(
|
|
447
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
444
448
|
if (params.type)
|
|
445
|
-
searchParams.set(
|
|
449
|
+
searchParams.set("type", params.type);
|
|
446
450
|
if (params.status)
|
|
447
|
-
searchParams.set(
|
|
451
|
+
searchParams.set("status", params.status);
|
|
448
452
|
if (params.search)
|
|
449
|
-
searchParams.set(
|
|
453
|
+
searchParams.set("search", params.search);
|
|
450
454
|
const query = searchParams.toString();
|
|
451
|
-
const endpoint = query ? `/api/evaluations?${query}` :
|
|
455
|
+
const endpoint = query ? `/api/evaluations?${query}` : "/api/evaluations";
|
|
452
456
|
return this.client.request(endpoint);
|
|
453
457
|
}
|
|
454
458
|
/**
|
|
@@ -456,7 +460,7 @@ class EvaluationAPI {
|
|
|
456
460
|
*/
|
|
457
461
|
async update(id, params) {
|
|
458
462
|
return this.client.request(`/api/evaluations?id=${id}`, {
|
|
459
|
-
method:
|
|
463
|
+
method: "PUT",
|
|
460
464
|
body: JSON.stringify(params),
|
|
461
465
|
});
|
|
462
466
|
}
|
|
@@ -465,7 +469,7 @@ class EvaluationAPI {
|
|
|
465
469
|
*/
|
|
466
470
|
async delete(id) {
|
|
467
471
|
return this.client.request(`/api/evaluations?id=${id}`, {
|
|
468
|
-
method:
|
|
472
|
+
method: "DELETE",
|
|
469
473
|
});
|
|
470
474
|
}
|
|
471
475
|
/**
|
|
@@ -473,7 +477,7 @@ class EvaluationAPI {
|
|
|
473
477
|
*/
|
|
474
478
|
async createTestCase(evaluationId, params) {
|
|
475
479
|
return this.client.request(`/api/evaluations/${evaluationId}/test-cases`, {
|
|
476
|
-
method:
|
|
480
|
+
method: "POST",
|
|
477
481
|
body: JSON.stringify(params),
|
|
478
482
|
});
|
|
479
483
|
}
|
|
@@ -488,7 +492,7 @@ class EvaluationAPI {
|
|
|
488
492
|
*/
|
|
489
493
|
async createRun(evaluationId, params) {
|
|
490
494
|
return this.client.request(`/api/evaluations/${evaluationId}/runs`, {
|
|
491
|
-
method:
|
|
495
|
+
method: "POST",
|
|
492
496
|
body: JSON.stringify(params),
|
|
493
497
|
});
|
|
494
498
|
}
|
|
@@ -516,8 +520,8 @@ class LLMJudgeAPI {
|
|
|
516
520
|
* Run an LLM judge evaluation
|
|
517
521
|
*/
|
|
518
522
|
async evaluate(params) {
|
|
519
|
-
return this.client.request(
|
|
520
|
-
method:
|
|
523
|
+
return this.client.request("/api/llm-judge/evaluate", {
|
|
524
|
+
method: "POST",
|
|
521
525
|
body: JSON.stringify(params),
|
|
522
526
|
});
|
|
523
527
|
}
|
|
@@ -525,8 +529,8 @@ class LLMJudgeAPI {
|
|
|
525
529
|
* Create an LLM judge configuration
|
|
526
530
|
*/
|
|
527
531
|
async createConfig(params) {
|
|
528
|
-
return this.client.request(
|
|
529
|
-
method:
|
|
532
|
+
return this.client.request("/api/llm-judge/configs", {
|
|
533
|
+
method: "POST",
|
|
530
534
|
body: JSON.stringify(params),
|
|
531
535
|
});
|
|
532
536
|
}
|
|
@@ -536,13 +540,13 @@ class LLMJudgeAPI {
|
|
|
536
540
|
async listConfigs(params = {}) {
|
|
537
541
|
const searchParams = new URLSearchParams();
|
|
538
542
|
if (params.organizationId)
|
|
539
|
-
searchParams.set(
|
|
543
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
540
544
|
if (params.limit)
|
|
541
|
-
searchParams.set(
|
|
545
|
+
searchParams.set("limit", params.limit.toString());
|
|
542
546
|
if (params.offset)
|
|
543
|
-
searchParams.set(
|
|
547
|
+
searchParams.set("offset", params.offset.toString());
|
|
544
548
|
const query = searchParams.toString();
|
|
545
|
-
const endpoint = query ? `/api/llm-judge/configs?${query}` :
|
|
549
|
+
const endpoint = query ? `/api/llm-judge/configs?${query}` : "/api/llm-judge/configs";
|
|
546
550
|
return this.client.request(endpoint);
|
|
547
551
|
}
|
|
548
552
|
/**
|
|
@@ -551,15 +555,15 @@ class LLMJudgeAPI {
|
|
|
551
555
|
async listResults(params = {}) {
|
|
552
556
|
const searchParams = new URLSearchParams();
|
|
553
557
|
if (params.configId)
|
|
554
|
-
searchParams.set(
|
|
558
|
+
searchParams.set("configId", params.configId.toString());
|
|
555
559
|
if (params.evaluationId)
|
|
556
|
-
searchParams.set(
|
|
560
|
+
searchParams.set("evaluationId", params.evaluationId.toString());
|
|
557
561
|
if (params.limit)
|
|
558
|
-
searchParams.set(
|
|
562
|
+
searchParams.set("limit", params.limit.toString());
|
|
559
563
|
if (params.offset)
|
|
560
|
-
searchParams.set(
|
|
564
|
+
searchParams.set("offset", params.offset.toString());
|
|
561
565
|
const query = searchParams.toString();
|
|
562
|
-
const endpoint = query ? `/api/llm-judge/results?${query}` :
|
|
566
|
+
const endpoint = query ? `/api/llm-judge/results?${query}` : "/api/llm-judge/results";
|
|
563
567
|
return this.client.request(endpoint);
|
|
564
568
|
}
|
|
565
569
|
/**
|
|
@@ -567,11 +571,11 @@ class LLMJudgeAPI {
|
|
|
567
571
|
*/
|
|
568
572
|
async getAlignment(params) {
|
|
569
573
|
const searchParams = new URLSearchParams();
|
|
570
|
-
searchParams.set(
|
|
574
|
+
searchParams.set("configId", params.configId.toString());
|
|
571
575
|
if (params.startDate)
|
|
572
|
-
searchParams.set(
|
|
576
|
+
searchParams.set("startDate", params.startDate);
|
|
573
577
|
if (params.endDate)
|
|
574
|
-
searchParams.set(
|
|
578
|
+
searchParams.set("endDate", params.endDate);
|
|
575
579
|
const query = searchParams.toString();
|
|
576
580
|
return this.client.request(`/api/llm-judge/alignment?${query}`);
|
|
577
581
|
}
|
|
@@ -588,10 +592,12 @@ class AnnotationsAPI {
|
|
|
588
592
|
* Create an annotation
|
|
589
593
|
*/
|
|
590
594
|
async create(params) {
|
|
591
|
-
return this.client
|
|
592
|
-
|
|
595
|
+
return this.client
|
|
596
|
+
.request("/api/annotations", {
|
|
597
|
+
method: "POST",
|
|
593
598
|
body: JSON.stringify(params),
|
|
594
|
-
})
|
|
599
|
+
})
|
|
600
|
+
.then((res) => res.annotation);
|
|
595
601
|
}
|
|
596
602
|
/**
|
|
597
603
|
* List annotations
|
|
@@ -599,16 +605,18 @@ class AnnotationsAPI {
|
|
|
599
605
|
async list(params = {}) {
|
|
600
606
|
const searchParams = new URLSearchParams();
|
|
601
607
|
if (params.evaluationRunId)
|
|
602
|
-
searchParams.set(
|
|
608
|
+
searchParams.set("evaluationRunId", params.evaluationRunId.toString());
|
|
603
609
|
if (params.testCaseId)
|
|
604
|
-
searchParams.set(
|
|
610
|
+
searchParams.set("testCaseId", params.testCaseId.toString());
|
|
605
611
|
if (params.limit)
|
|
606
|
-
searchParams.set(
|
|
612
|
+
searchParams.set("limit", params.limit.toString());
|
|
607
613
|
if (params.offset)
|
|
608
|
-
searchParams.set(
|
|
614
|
+
searchParams.set("offset", params.offset.toString());
|
|
609
615
|
const query = searchParams.toString();
|
|
610
|
-
const endpoint = query ? `/api/annotations?${query}` :
|
|
611
|
-
return this.client
|
|
616
|
+
const endpoint = query ? `/api/annotations?${query}` : "/api/annotations";
|
|
617
|
+
return this.client
|
|
618
|
+
.request(endpoint)
|
|
619
|
+
.then((res) => res.annotations);
|
|
612
620
|
}
|
|
613
621
|
}
|
|
614
622
|
/**
|
|
@@ -623,8 +631,8 @@ class AnnotationTasksAPI {
|
|
|
623
631
|
* Create an annotation task
|
|
624
632
|
*/
|
|
625
633
|
async create(params) {
|
|
626
|
-
return this.client.request(
|
|
627
|
-
method:
|
|
634
|
+
return this.client.request("/api/annotations/tasks", {
|
|
635
|
+
method: "POST",
|
|
628
636
|
body: JSON.stringify(params),
|
|
629
637
|
});
|
|
630
638
|
}
|
|
@@ -634,15 +642,15 @@ class AnnotationTasksAPI {
|
|
|
634
642
|
async list(params = {}) {
|
|
635
643
|
const searchParams = new URLSearchParams();
|
|
636
644
|
if (params.organizationId)
|
|
637
|
-
searchParams.set(
|
|
645
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
638
646
|
if (params.status)
|
|
639
|
-
searchParams.set(
|
|
647
|
+
searchParams.set("status", params.status);
|
|
640
648
|
if (params.limit)
|
|
641
|
-
searchParams.set(
|
|
649
|
+
searchParams.set("limit", params.limit.toString());
|
|
642
650
|
if (params.offset)
|
|
643
|
-
searchParams.set(
|
|
651
|
+
searchParams.set("offset", params.offset.toString());
|
|
644
652
|
const query = searchParams.toString();
|
|
645
|
-
const endpoint = query ? `/api/annotations/tasks?${query}` :
|
|
653
|
+
const endpoint = query ? `/api/annotations/tasks?${query}` : "/api/annotations/tasks";
|
|
646
654
|
return this.client.request(endpoint);
|
|
647
655
|
}
|
|
648
656
|
/**
|
|
@@ -664,7 +672,7 @@ class AnnotationTaskItemsAPI {
|
|
|
664
672
|
*/
|
|
665
673
|
async create(taskId, params) {
|
|
666
674
|
return this.client.request(`/api/annotations/tasks/${taskId}/items`, {
|
|
667
|
-
method:
|
|
675
|
+
method: "POST",
|
|
668
676
|
body: JSON.stringify(params),
|
|
669
677
|
});
|
|
670
678
|
}
|
|
@@ -674,11 +682,13 @@ class AnnotationTaskItemsAPI {
|
|
|
674
682
|
async list(taskId, params = {}) {
|
|
675
683
|
const searchParams = new URLSearchParams();
|
|
676
684
|
if (params.limit)
|
|
677
|
-
searchParams.set(
|
|
685
|
+
searchParams.set("limit", params.limit.toString());
|
|
678
686
|
if (params.offset)
|
|
679
|
-
searchParams.set(
|
|
687
|
+
searchParams.set("offset", params.offset.toString());
|
|
680
688
|
const query = searchParams.toString();
|
|
681
|
-
const endpoint = query
|
|
689
|
+
const endpoint = query
|
|
690
|
+
? `/api/annotations/tasks/${taskId}/items?${query}`
|
|
691
|
+
: `/api/annotations/tasks/${taskId}/items`;
|
|
682
692
|
return this.client.request(endpoint);
|
|
683
693
|
}
|
|
684
694
|
}
|
|
@@ -696,11 +706,11 @@ class DeveloperAPI {
|
|
|
696
706
|
*/
|
|
697
707
|
async getUsage(params) {
|
|
698
708
|
const searchParams = new URLSearchParams();
|
|
699
|
-
searchParams.set(
|
|
709
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
700
710
|
if (params.startDate)
|
|
701
|
-
searchParams.set(
|
|
711
|
+
searchParams.set("startDate", params.startDate);
|
|
702
712
|
if (params.endDate)
|
|
703
|
-
searchParams.set(
|
|
713
|
+
searchParams.set("endDate", params.endDate);
|
|
704
714
|
const query = searchParams.toString();
|
|
705
715
|
return this.client.request(`/api/developer/usage?${query}`);
|
|
706
716
|
}
|
|
@@ -722,8 +732,8 @@ class APIKeysAPI {
|
|
|
722
732
|
* Create an API key
|
|
723
733
|
*/
|
|
724
734
|
async create(params) {
|
|
725
|
-
return this.client.request(
|
|
726
|
-
method:
|
|
735
|
+
return this.client.request("/api/developer/api-keys", {
|
|
736
|
+
method: "POST",
|
|
727
737
|
body: JSON.stringify(params),
|
|
728
738
|
});
|
|
729
739
|
}
|
|
@@ -733,13 +743,13 @@ class APIKeysAPI {
|
|
|
733
743
|
async list(params = {}) {
|
|
734
744
|
const searchParams = new URLSearchParams();
|
|
735
745
|
if (params.organizationId)
|
|
736
|
-
searchParams.set(
|
|
746
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
737
747
|
if (params.limit)
|
|
738
|
-
searchParams.set(
|
|
748
|
+
searchParams.set("limit", params.limit.toString());
|
|
739
749
|
if (params.offset)
|
|
740
|
-
searchParams.set(
|
|
750
|
+
searchParams.set("offset", params.offset.toString());
|
|
741
751
|
const query = searchParams.toString();
|
|
742
|
-
const endpoint = query ? `/api/developer/api-keys?${query}` :
|
|
752
|
+
const endpoint = query ? `/api/developer/api-keys?${query}` : "/api/developer/api-keys";
|
|
743
753
|
return this.client.request(endpoint);
|
|
744
754
|
}
|
|
745
755
|
/**
|
|
@@ -747,7 +757,7 @@ class APIKeysAPI {
|
|
|
747
757
|
*/
|
|
748
758
|
async update(keyId, params) {
|
|
749
759
|
return this.client.request(`/api/developer/api-keys/${keyId}`, {
|
|
750
|
-
method:
|
|
760
|
+
method: "PATCH",
|
|
751
761
|
body: JSON.stringify(params),
|
|
752
762
|
});
|
|
753
763
|
}
|
|
@@ -756,7 +766,7 @@ class APIKeysAPI {
|
|
|
756
766
|
*/
|
|
757
767
|
async revoke(keyId) {
|
|
758
768
|
return this.client.request(`/api/developer/api-keys/${keyId}`, {
|
|
759
|
-
method:
|
|
769
|
+
method: "DELETE",
|
|
760
770
|
});
|
|
761
771
|
}
|
|
762
772
|
/**
|
|
@@ -777,8 +787,8 @@ class WebhooksAPI {
|
|
|
777
787
|
* Create a webhook
|
|
778
788
|
*/
|
|
779
789
|
async create(params) {
|
|
780
|
-
return this.client.request(
|
|
781
|
-
method:
|
|
790
|
+
return this.client.request("/api/developer/webhooks", {
|
|
791
|
+
method: "POST",
|
|
782
792
|
body: JSON.stringify(params),
|
|
783
793
|
});
|
|
784
794
|
}
|
|
@@ -787,13 +797,13 @@ class WebhooksAPI {
|
|
|
787
797
|
*/
|
|
788
798
|
async list(params) {
|
|
789
799
|
const searchParams = new URLSearchParams();
|
|
790
|
-
searchParams.set(
|
|
800
|
+
searchParams.set("organizationId", params.organizationId.toString());
|
|
791
801
|
if (params.status)
|
|
792
|
-
searchParams.set(
|
|
802
|
+
searchParams.set("status", params.status);
|
|
793
803
|
if (params.limit)
|
|
794
|
-
searchParams.set(
|
|
804
|
+
searchParams.set("limit", params.limit.toString());
|
|
795
805
|
if (params.offset)
|
|
796
|
-
searchParams.set(
|
|
806
|
+
searchParams.set("offset", params.offset.toString());
|
|
797
807
|
const query = searchParams.toString();
|
|
798
808
|
return this.client.request(`/api/developer/webhooks?${query}`);
|
|
799
809
|
}
|
|
@@ -808,7 +818,7 @@ class WebhooksAPI {
|
|
|
808
818
|
*/
|
|
809
819
|
async update(webhookId, params) {
|
|
810
820
|
return this.client.request(`/api/developer/webhooks/${webhookId}`, {
|
|
811
|
-
method:
|
|
821
|
+
method: "PATCH",
|
|
812
822
|
body: JSON.stringify(params),
|
|
813
823
|
});
|
|
814
824
|
}
|
|
@@ -817,7 +827,7 @@ class WebhooksAPI {
|
|
|
817
827
|
*/
|
|
818
828
|
async delete(webhookId) {
|
|
819
829
|
return this.client.request(`/api/developer/webhooks/${webhookId}`, {
|
|
820
|
-
method:
|
|
830
|
+
method: "DELETE",
|
|
821
831
|
});
|
|
822
832
|
}
|
|
823
833
|
/**
|
|
@@ -826,13 +836,15 @@ class WebhooksAPI {
|
|
|
826
836
|
async getDeliveries(webhookId, params = {}) {
|
|
827
837
|
const searchParams = new URLSearchParams();
|
|
828
838
|
if (params.limit)
|
|
829
|
-
searchParams.set(
|
|
839
|
+
searchParams.set("limit", params.limit.toString());
|
|
830
840
|
if (params.offset)
|
|
831
|
-
searchParams.set(
|
|
841
|
+
searchParams.set("offset", params.offset.toString());
|
|
832
842
|
if (params.success !== undefined)
|
|
833
|
-
searchParams.set(
|
|
843
|
+
searchParams.set("success", params.success.toString());
|
|
834
844
|
const query = searchParams.toString();
|
|
835
|
-
const endpoint = query
|
|
845
|
+
const endpoint = query
|
|
846
|
+
? `/api/developer/webhooks/${webhookId}/deliveries?${query}`
|
|
847
|
+
: `/api/developer/webhooks/${webhookId}/deliveries`;
|
|
836
848
|
return this.client.request(endpoint);
|
|
837
849
|
}
|
|
838
850
|
}
|
|
@@ -847,6 +859,6 @@ class OrganizationsAPI {
|
|
|
847
859
|
* Get current organization
|
|
848
860
|
*/
|
|
849
861
|
async getCurrent() {
|
|
850
|
-
return this.client.request(
|
|
862
|
+
return this.client.request("/api/organizations/current");
|
|
851
863
|
}
|
|
852
864
|
}
|