@evalgate/sdk 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/CHANGELOG.md +638 -0
  2. package/README.md +398 -0
  3. package/dist/assertions.d.ts +189 -0
  4. package/dist/assertions.js +662 -0
  5. package/dist/batch.d.ts +68 -0
  6. package/dist/batch.js +179 -0
  7. package/dist/cache.d.ts +65 -0
  8. package/dist/cache.js +131 -0
  9. package/dist/cli/api.d.ts +108 -0
  10. package/dist/cli/api.js +132 -0
  11. package/dist/cli/baseline.d.ts +10 -0
  12. package/dist/cli/baseline.js +172 -0
  13. package/dist/cli/check.d.ts +73 -0
  14. package/dist/cli/check.js +355 -0
  15. package/dist/cli/ci-context.d.ts +6 -0
  16. package/dist/cli/ci-context.js +112 -0
  17. package/dist/cli/ci.d.ts +45 -0
  18. package/dist/cli/ci.js +192 -0
  19. package/dist/cli/config.d.ts +30 -0
  20. package/dist/cli/config.js +230 -0
  21. package/dist/cli/constants.d.ts +15 -0
  22. package/dist/cli/constants.js +18 -0
  23. package/dist/cli/diff.d.ts +173 -0
  24. package/dist/cli/diff.js +685 -0
  25. package/dist/cli/discover.d.ts +84 -0
  26. package/dist/cli/discover.js +419 -0
  27. package/dist/cli/doctor.d.ts +88 -0
  28. package/dist/cli/doctor.js +675 -0
  29. package/dist/cli/env.d.ts +21 -0
  30. package/dist/cli/env.js +42 -0
  31. package/dist/cli/explain.d.ts +58 -0
  32. package/dist/cli/explain.js +561 -0
  33. package/dist/cli/formatters/github.d.ts +8 -0
  34. package/dist/cli/formatters/github.js +135 -0
  35. package/dist/cli/formatters/human.d.ts +6 -0
  36. package/dist/cli/formatters/human.js +110 -0
  37. package/dist/cli/formatters/json.d.ts +6 -0
  38. package/dist/cli/formatters/json.js +10 -0
  39. package/dist/cli/formatters/pr-comment.d.ts +12 -0
  40. package/dist/cli/formatters/pr-comment.js +103 -0
  41. package/dist/cli/formatters/types.d.ts +103 -0
  42. package/dist/cli/formatters/types.js +8 -0
  43. package/dist/cli/gate.d.ts +21 -0
  44. package/dist/cli/gate.js +179 -0
  45. package/dist/cli/impact-analysis.d.ts +63 -0
  46. package/dist/cli/impact-analysis.js +252 -0
  47. package/dist/cli/index.d.ts +9 -0
  48. package/dist/cli/index.js +332 -0
  49. package/dist/cli/init.d.ts +16 -0
  50. package/dist/cli/init.js +292 -0
  51. package/dist/cli/manifest.d.ts +103 -0
  52. package/dist/cli/manifest.js +282 -0
  53. package/dist/cli/migrate.d.ts +41 -0
  54. package/dist/cli/migrate.js +349 -0
  55. package/dist/cli/policy-packs.d.ts +23 -0
  56. package/dist/cli/policy-packs.js +89 -0
  57. package/dist/cli/print-config.d.ts +29 -0
  58. package/dist/cli/print-config.js +270 -0
  59. package/dist/cli/profiles.d.ts +28 -0
  60. package/dist/cli/profiles.js +30 -0
  61. package/dist/cli/reason-codes.d.ts +17 -0
  62. package/dist/cli/reason-codes.js +19 -0
  63. package/dist/cli/regression-gate.d.ts +15 -0
  64. package/dist/cli/regression-gate.js +341 -0
  65. package/dist/cli/render/snippet.d.ts +5 -0
  66. package/dist/cli/render/snippet.js +15 -0
  67. package/dist/cli/render/sort.d.ts +10 -0
  68. package/dist/cli/render/sort.js +24 -0
  69. package/dist/cli/report/build-check-report.d.ts +19 -0
  70. package/dist/cli/report/build-check-report.js +132 -0
  71. package/dist/cli/run.d.ts +101 -0
  72. package/dist/cli/run.js +395 -0
  73. package/dist/cli/share.d.ts +17 -0
  74. package/dist/cli/share.js +91 -0
  75. package/dist/cli/upgrade.d.ts +15 -0
  76. package/dist/cli/upgrade.js +492 -0
  77. package/dist/cli/workspace.d.ts +31 -0
  78. package/dist/cli/workspace.js +68 -0
  79. package/dist/client.d.ts +368 -0
  80. package/dist/client.js +893 -0
  81. package/dist/client.request.test.d.ts +1 -0
  82. package/dist/client.request.test.js +232 -0
  83. package/dist/context.d.ts +134 -0
  84. package/dist/context.js +215 -0
  85. package/dist/errors.d.ts +82 -0
  86. package/dist/errors.js +298 -0
  87. package/dist/export.d.ts +195 -0
  88. package/dist/export.js +344 -0
  89. package/dist/index.d.ts +44 -0
  90. package/dist/index.js +153 -0
  91. package/dist/integrations/anthropic.d.ts +91 -0
  92. package/dist/integrations/anthropic.js +163 -0
  93. package/dist/integrations/openai-eval.d.ts +57 -0
  94. package/dist/integrations/openai-eval.js +232 -0
  95. package/dist/integrations/openai.d.ts +92 -0
  96. package/dist/integrations/openai.js +160 -0
  97. package/dist/local.d.ts +39 -0
  98. package/dist/local.js +148 -0
  99. package/dist/logger.d.ts +128 -0
  100. package/dist/logger.js +227 -0
  101. package/dist/matchers/index.d.ts +1 -0
  102. package/dist/matchers/index.js +6 -0
  103. package/dist/matchers/to-pass-gate.d.ts +29 -0
  104. package/dist/matchers/to-pass-gate.js +35 -0
  105. package/dist/pagination.d.ts +74 -0
  106. package/dist/pagination.js +139 -0
  107. package/dist/regression.d.ts +100 -0
  108. package/dist/regression.js +44 -0
  109. package/dist/runtime/adapters/config-to-dsl.d.ts +33 -0
  110. package/dist/runtime/adapters/config-to-dsl.js +400 -0
  111. package/dist/runtime/adapters/testsuite-to-dsl.d.ts +63 -0
  112. package/dist/runtime/adapters/testsuite-to-dsl.js +276 -0
  113. package/dist/runtime/context.d.ts +26 -0
  114. package/dist/runtime/context.js +74 -0
  115. package/dist/runtime/eval.d.ts +46 -0
  116. package/dist/runtime/eval.js +244 -0
  117. package/dist/runtime/execution-mode.d.ts +80 -0
  118. package/dist/runtime/execution-mode.js +357 -0
  119. package/dist/runtime/executor.d.ts +16 -0
  120. package/dist/runtime/executor.js +152 -0
  121. package/dist/runtime/registry.d.ts +78 -0
  122. package/dist/runtime/registry.js +403 -0
  123. package/dist/runtime/run-report.d.ts +200 -0
  124. package/dist/runtime/run-report.js +222 -0
  125. package/dist/runtime/types.d.ts +356 -0
  126. package/dist/runtime/types.js +76 -0
  127. package/dist/snapshot.d.ts +176 -0
  128. package/dist/snapshot.js +322 -0
  129. package/dist/streaming.d.ts +173 -0
  130. package/dist/streaming.js +268 -0
  131. package/dist/testing.d.ts +273 -0
  132. package/dist/testing.js +317 -0
  133. package/dist/types.d.ts +754 -0
  134. package/dist/types.js +54 -0
  135. package/dist/utils/input-hash.d.ts +8 -0
  136. package/dist/utils/input-hash.js +41 -0
  137. package/dist/version.d.ts +7 -0
  138. package/dist/version.js +10 -0
  139. package/dist/workflows.d.ts +389 -0
  140. package/dist/workflows.js +671 -0
  141. package/package.json +117 -0
package/dist/client.js ADDED
@@ -0,0 +1,893 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.AIEvalClient = void 0;
4
+ const batch_1 = require("./batch");
5
+ const cache_1 = require("./cache");
6
+ const context_1 = require("./context");
7
+ const errors_1 = require("./errors");
8
+ const logger_1 = require("./logger");
9
+ const version_1 = require("./version");
10
+ /**
11
+ * Safe environment variable access (works in both Node.js and browsers).
12
+ * Prefers EVALGATE_* vars; falls back to EVALAI_* with deprecation warning.
13
+ */
14
+ function getEnvVar(newName, legacyName) {
15
+ if (typeof process === "undefined" || !process.env)
16
+ return undefined;
17
+ const val = process.env[newName];
18
+ if (val)
19
+ return val;
20
+ if (legacyName && process.env[legacyName]) {
21
+ if (typeof process !== "undefined" &&
22
+ !process.__EVALGATE_LEGACY_WARNED) {
23
+ console.warn(`[EvalGate] Deprecation: ${legacyName} is deprecated. Use ${newName} instead.`);
24
+ process.__EVALGATE_LEGACY_WARNED = true;
25
+ }
26
+ return process.env[legacyName];
27
+ }
28
+ return undefined;
29
+ }
30
+ /**
31
+ * EvalGate SDK Client
32
+ *
33
+ * @example
34
+ * ```typescript
35
+ * import { AIEvalClient } from '@ai-eval-platform/sdk';
36
+ *
37
+ * // Zero-config initialization (uses env variables)
38
+ * const client = AIEvalClient.init();
39
+ *
40
+ * // Or with explicit config
41
+ * const client = new AIEvalClient({
42
+ * apiKey: 'your-api-key',
43
+ * organizationId: 123,
44
+ * debug: true
45
+ * });
46
+ *
47
+ * // Create a trace with automatic context propagation
48
+ * const trace = await client.traces.create({
49
+ * name: 'User Query',
50
+ * traceId: 'trace-123'
51
+ * });
52
+ * ```
53
+ */
54
+ class AIEvalClient {
55
+ constructor(config = {}) {
56
+ // Tier 1.1: Zero-config with env variable detection (works in Node.js and browsers)
57
+ this.apiKey =
58
+ config.apiKey ||
59
+ getEnvVar("EVALGATE_API_KEY", "EVALAI_API_KEY") ||
60
+ getEnvVar("AI_EVAL_API_KEY") ||
61
+ "";
62
+ if (!this.apiKey) {
63
+ throw new errors_1.EvalGateError("API key is required. Provide via config.apiKey or EVALGATE_API_KEY environment variable.", "MISSING_API_KEY", 0);
64
+ }
65
+ // Auto-detect organization ID from env
66
+ const orgIdFromEnv = getEnvVar("EVALGATE_ORGANIZATION_ID", "EVALAI_ORGANIZATION_ID") ||
67
+ getEnvVar("AI_EVAL_ORGANIZATION_ID");
68
+ this.organizationId =
69
+ config.organizationId ||
70
+ (orgIdFromEnv ? parseInt(orgIdFromEnv, 10) : undefined);
71
+ const isBrowser = typeof globalThis.window !== "undefined";
72
+ this.baseUrl =
73
+ config.baseUrl ||
74
+ getEnvVar("EVALGATE_BASE_URL", "EVALAI_BASE_URL") ||
75
+ (isBrowser ? "" : "http://localhost:3000");
76
+ this.timeout = config.timeout || 30000;
77
+ // Tier 4.17: Debug mode with request logging
78
+ const logLevel = config.logLevel || (config.debug ? "debug" : "info");
79
+ this.logger = (0, logger_1.createLogger)({
80
+ level: logLevel,
81
+ pretty: config.debug,
82
+ prefix: "EvalGate",
83
+ });
84
+ this.requestLogger = new logger_1.RequestLogger(this.logger);
85
+ // Retry configuration
86
+ this.retryConfig = {
87
+ maxAttempts: config.retry?.maxAttempts || 3,
88
+ backoff: config.retry?.backoff || "exponential",
89
+ retryableErrors: config.retry?.retryableErrors || [
90
+ "RATE_LIMIT_EXCEEDED",
91
+ "TIMEOUT",
92
+ "NETWORK_ERROR",
93
+ "INTERNAL_SERVER_ERROR",
94
+ ],
95
+ };
96
+ // Initialize cache for GET requests
97
+ this.cache = new cache_1.RequestCache(config.cacheSize || 1000);
98
+ // Initialize request batcher if enabled (default: enabled)
99
+ if (config.enableBatching !== false) {
100
+ const MAX_CONCURRENCY = 5;
101
+ this.batcher = new batch_1.RequestBatcher(async (requests) => {
102
+ const results = [];
103
+ const executing = [];
104
+ for (const req of requests) {
105
+ const task = (async () => {
106
+ try {
107
+ const data = await this.request(req.endpoint, {
108
+ method: req.method,
109
+ body: req.body ? JSON.stringify(req.body) : undefined,
110
+ headers: req.headers,
111
+ });
112
+ results.push({ id: req.id, status: 200, data });
113
+ }
114
+ catch (err) {
115
+ const errorObj = err;
116
+ results.push({
117
+ id: req.id,
118
+ status: errorObj?.statusCode || 500,
119
+ data: null,
120
+ error: errorObj?.message || "Unknown error",
121
+ });
122
+ }
123
+ })();
124
+ executing.push(task);
125
+ if (executing.length >= MAX_CONCURRENCY) {
126
+ await Promise.race(executing);
127
+ // Remove settled promises
128
+ for (let i = executing.length - 1; i >= 0; i--) {
129
+ const settled = await Promise.race([
130
+ executing[i].then(() => true),
131
+ Promise.resolve(false),
132
+ ]);
133
+ if (settled)
134
+ executing.splice(i, 1);
135
+ }
136
+ }
137
+ }
138
+ await Promise.allSettled(executing);
139
+ return results;
140
+ }, {
141
+ maxBatchSize: config.batchSize || 10,
142
+ batchDelay: config.batchDelay || 50,
143
+ });
144
+ }
145
+ else {
146
+ this.batcher = null;
147
+ }
148
+ // Initialize API modules
149
+ this.traces = new TraceAPI(this);
150
+ this.evaluations = new EvaluationAPI(this);
151
+ this.llmJudge = new LLMJudgeAPI(this);
152
+ this.annotations = new AnnotationsAPI(this);
153
+ this.developer = new DeveloperAPI(this);
154
+ this.organizations = new OrganizationsAPI(this);
155
+ this.logger.info("SDK initialized", {
156
+ hasOrganizationId: !!this.organizationId,
157
+ baseUrl: this.baseUrl,
158
+ });
159
+ }
160
+ /**
161
+ * Zero-config initialization using environment variables
162
+ *
163
+ * Works in both Node.js and browsers. In Node.js, reads from environment variables.
164
+ * In browsers, you must provide config explicitly.
165
+ *
166
+ * Environment variables (Node.js only):
167
+ * - EVALGATE_API_KEY (or EVALAI_API_KEY): Your API key
168
+ * - EVALGATE_ORGANIZATION_ID (or EVALAI_ORGANIZATION_ID): Your organization ID
169
+ * - EVALGATE_BASE_URL (or EVALAI_BASE_URL): Custom API base URL (optional)
170
+ *
171
+ * @example
172
+ * ```typescript
173
+ * // Node.js - reads from env vars:
174
+ * // EVALGATE_API_KEY=your-key
175
+ * // EVALGATE_ORGANIZATION_ID=123
176
+ * const client = AIEvalClient.init();
177
+ *
178
+ * // Browser - must provide config:
179
+ * const client = AIEvalClient.init({
180
+ * apiKey: 'your-key',
181
+ * organizationId: 123
182
+ * });
183
+ * ```
184
+ */
185
+ static init(config = {}) {
186
+ return new AIEvalClient({
187
+ baseUrl: getEnvVar("EVALGATE_BASE_URL", "EVALAI_BASE_URL"),
188
+ ...config,
189
+ });
190
+ }
191
+ /**
192
+ * Internal method to make HTTP requests with retry logic and error handling
193
+ */
194
+ async request(endpoint, options = {}, attempt = 1) {
195
+ const method = (options.method || "GET").toUpperCase();
196
+ const url = `${this.baseUrl}${endpoint}`;
197
+ // Check cache for GET requests
198
+ if (method === "GET" && (0, cache_1.shouldCache)(method, endpoint)) {
199
+ const cached = this.cache.get(method, endpoint, options.body);
200
+ if (cached !== null) {
201
+ this.logger.debug("Cache hit", { endpoint });
202
+ return cached;
203
+ }
204
+ }
205
+ const controller = new AbortController();
206
+ const timeoutId = setTimeout(() => controller.abort(), this.timeout);
207
+ const startTime = Date.now();
208
+ // Log request
209
+ this.requestLogger.logRequest({
210
+ method: options.method || "GET",
211
+ url,
212
+ headers: options.headers,
213
+ body: options.body,
214
+ });
215
+ try {
216
+ const response = await fetch(url, {
217
+ ...options,
218
+ headers: {
219
+ "Content-Type": "application/json",
220
+ Authorization: `Bearer ${this.apiKey}`,
221
+ "X-EvalGate-SDK-Version": version_1.SDK_VERSION,
222
+ "X-EvalGate-Spec-Version": version_1.SPEC_VERSION,
223
+ ...options.headers,
224
+ },
225
+ signal: controller.signal,
226
+ });
227
+ clearTimeout(timeoutId);
228
+ const duration = Date.now() - startTime;
229
+ let data;
230
+ try {
231
+ data = await response.json();
232
+ }
233
+ catch (_e) {
234
+ data = {};
235
+ }
236
+ // Log response
237
+ this.requestLogger.logResponse({
238
+ method: options.method || "GET",
239
+ url,
240
+ status: response.status,
241
+ duration,
242
+ body: data,
243
+ });
244
+ if (!response.ok) {
245
+ const error = (0, errors_1.createErrorFromResponse)(response, data);
246
+ // Retry logic
247
+ if (attempt < this.retryConfig.maxAttempts &&
248
+ this.retryConfig.retryableErrors.includes(error.code)) {
249
+ const delay = this.calculateBackoff(attempt);
250
+ this.logger.warn(`Retrying request (attempt ${attempt + 1}/${this.retryConfig.maxAttempts}) after ${delay}ms`, {
251
+ error: error.code,
252
+ url,
253
+ });
254
+ await new Promise((resolve) => setTimeout(resolve, delay));
255
+ return this.request(endpoint, options, attempt + 1);
256
+ }
257
+ throw error;
258
+ }
259
+ // Cache successful GET responses
260
+ if (method === "GET" && (0, cache_1.shouldCache)(method, endpoint)) {
261
+ const ttl = (0, cache_1.getTTL)(endpoint);
262
+ this.cache.set(method, endpoint, data, ttl, options.body);
263
+ this.logger.debug("Cached response", { endpoint, ttl });
264
+ }
265
+ // Invalidate cache for mutation operations
266
+ if (["POST", "PUT", "DELETE", "PATCH"].includes(method)) {
267
+ // Invalidate related cached entries
268
+ const resourceMatch = endpoint.match(/\/api\/(\w+)/);
269
+ if (resourceMatch) {
270
+ this.cache.invalidatePattern(resourceMatch[1]);
271
+ }
272
+ }
273
+ return data;
274
+ }
275
+ catch (error) {
276
+ clearTimeout(timeoutId);
277
+ if (error instanceof errors_1.EvalGateError) {
278
+ throw error;
279
+ }
280
+ if (error instanceof Error) {
281
+ if (error.name === "AbortError") {
282
+ throw new errors_1.EvalGateError("Request timeout", "TIMEOUT", 408);
283
+ }
284
+ throw new errors_1.EvalGateError(error.message, "NETWORK_ERROR", 0);
285
+ }
286
+ throw error;
287
+ }
288
+ }
289
+ /**
290
+ * Calculate backoff delay for retries
291
+ */
292
+ calculateBackoff(attempt) {
293
+ const baseDelay = 1000; // 1 second
294
+ switch (this.retryConfig.backoff) {
295
+ case "exponential":
296
+ return baseDelay * 2 ** (attempt - 1);
297
+ case "linear":
298
+ return baseDelay * attempt;
299
+ default:
300
+ return baseDelay;
301
+ }
302
+ }
303
+ getOrganizationId() {
304
+ return this.organizationId;
305
+ }
306
+ /**
307
+ * Get the logger instance for custom logging
308
+ */
309
+ getLogger() {
310
+ return this.logger;
311
+ }
312
+ /**
313
+ * @deprecated The /api/organizations/:id/limits endpoint does not exist.
314
+ * Use `organizations.getCurrent()` to get org info instead.
315
+ */
316
+ async getOrganizationLimits() {
317
+ return {};
318
+ }
319
+ }
320
+ exports.AIEvalClient = AIEvalClient;
321
+ /**
322
+ * Trace API methods
323
+ */
324
+ class TraceAPI {
325
+ constructor(client) {
326
+ this.client = client;
327
+ }
328
+ /**
329
+ * Create a new trace with automatic context propagation
330
+ *
331
+ * @example
332
+ * ```typescript
333
+ * const trace = await client.traces.create({
334
+ * name: 'User Query',
335
+ * traceId: 'trace-123',
336
+ * metadata: { userId: '456' }
337
+ * });
338
+ * ```
339
+ */
340
+ async create(params) {
341
+ const orgId = params.organizationId || this.client.getOrganizationId();
342
+ if (!orgId) {
343
+ throw new errors_1.EvalGateError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
344
+ }
345
+ // Merge with context
346
+ const metadata = (0, context_1.mergeWithContext)(params.metadata || {});
347
+ return this.client.request("/api/traces", {
348
+ method: "POST",
349
+ body: JSON.stringify({ ...params, organizationId: orgId, metadata }),
350
+ });
351
+ }
352
+ /**
353
+ * List traces with optional filtering
354
+ */
355
+ async list(params = {}) {
356
+ const searchParams = new URLSearchParams();
357
+ if (params.limit)
358
+ searchParams.set("limit", params.limit.toString());
359
+ if (params.offset)
360
+ searchParams.set("offset", params.offset.toString());
361
+ if (params.organizationId)
362
+ searchParams.set("organizationId", params.organizationId.toString());
363
+ if (params.status)
364
+ searchParams.set("status", params.status);
365
+ if (params.search)
366
+ searchParams.set("search", params.search);
367
+ const query = searchParams.toString();
368
+ const endpoint = query ? `/api/traces?${query}` : "/api/traces";
369
+ return this.client.request(endpoint);
370
+ }
371
+ /**
372
+ * Delete a trace by ID
373
+ */
374
+ async delete(id) {
375
+ return this.client.request(`/api/traces?id=${id}`, {
376
+ method: "DELETE",
377
+ });
378
+ }
379
+ /**
380
+ * Get a single trace by ID, including its spans
381
+ */
382
+ async get(id) {
383
+ return this.client.request(`/api/traces/${id}`);
384
+ }
385
+ /**
386
+ * Update an existing trace (e.g. set status, duration, metadata on completion)
387
+ *
388
+ * @example
389
+ * ```typescript
390
+ * await client.traces.update(42, {
391
+ * status: 'success',
392
+ * durationMs: 1234,
393
+ * metadata: { output: 'done' }
394
+ * });
395
+ * ```
396
+ */
397
+ async update(id, params) {
398
+ return this.client.request(`/api/traces/${id}`, {
399
+ method: "PATCH",
400
+ body: JSON.stringify(params),
401
+ });
402
+ }
403
+ /**
404
+ * Create a span for a trace
405
+ */
406
+ async createSpan(traceId, params) {
407
+ return this.client.request(`/api/traces/${traceId}/spans`, {
408
+ method: "POST",
409
+ body: JSON.stringify(params),
410
+ });
411
+ }
412
+ /**
413
+ * List spans for a trace
414
+ */
415
+ async listSpans(traceId) {
416
+ return this.client.request(`/api/traces/${traceId}/spans`);
417
+ }
418
+ }
419
+ /**
420
+ * Evaluation API methods
421
+ */
422
+ class EvaluationAPI {
423
+ constructor(client) {
424
+ this.client = client;
425
+ }
426
+ /**
427
+ * Create a new evaluation
428
+ */
429
+ async create(params) {
430
+ const orgId = params.organizationId || this.client.getOrganizationId();
431
+ if (!orgId) {
432
+ throw new errors_1.EvalGateError("Organization ID is required", "MISSING_ORGANIZATION_ID", 0);
433
+ }
434
+ return this.client.request("/api/evaluations", {
435
+ method: "POST",
436
+ body: JSON.stringify({ ...params, organizationId: orgId }),
437
+ });
438
+ }
439
+ /**
440
+ * Get a single evaluation by ID
441
+ */
442
+ async get(id) {
443
+ return this.client.request(`/api/evaluations?id=${id}`);
444
+ }
445
+ /**
446
+ * List evaluations with optional filtering
447
+ */
448
+ async list(params = {}) {
449
+ const searchParams = new URLSearchParams();
450
+ if (params.limit)
451
+ searchParams.set("limit", params.limit.toString());
452
+ if (params.offset)
453
+ searchParams.set("offset", params.offset.toString());
454
+ if (params.organizationId)
455
+ searchParams.set("organizationId", params.organizationId.toString());
456
+ if (params.type)
457
+ searchParams.set("type", params.type);
458
+ if (params.status)
459
+ searchParams.set("status", params.status);
460
+ if (params.search)
461
+ searchParams.set("search", params.search);
462
+ const query = searchParams.toString();
463
+ const endpoint = query ? `/api/evaluations?${query}` : "/api/evaluations";
464
+ return this.client.request(endpoint);
465
+ }
466
+ /**
467
+ * Update an evaluation
468
+ */
469
+ async update(id, params) {
470
+ return this.client.request(`/api/evaluations?id=${id}`, {
471
+ method: "PUT",
472
+ body: JSON.stringify(params),
473
+ });
474
+ }
475
+ /**
476
+ * Delete an evaluation
477
+ */
478
+ async delete(id) {
479
+ return this.client.request(`/api/evaluations?id=${id}`, {
480
+ method: "DELETE",
481
+ });
482
+ }
483
+ /**
484
+ * Create a test case for an evaluation
485
+ */
486
+ async createTestCase(evaluationId, params) {
487
+ return this.client.request(`/api/evaluations/${evaluationId}/test-cases`, {
488
+ method: "POST",
489
+ body: JSON.stringify(params),
490
+ });
491
+ }
492
+ /**
493
+ * List test cases for an evaluation
494
+ */
495
+ async listTestCases(evaluationId) {
496
+ return this.client.request(`/api/evaluations/${evaluationId}/test-cases`);
497
+ }
498
+ /**
499
+ * Create a run for an evaluation
500
+ */
501
+ async createRun(evaluationId, params) {
502
+ return this.client.request(`/api/evaluations/${evaluationId}/runs`, {
503
+ method: "POST",
504
+ body: JSON.stringify(params),
505
+ });
506
+ }
507
+ /**
508
+ * List runs for an evaluation
509
+ */
510
+ async listRuns(evaluationId) {
511
+ return this.client.request(`/api/evaluations/${evaluationId}/runs`);
512
+ }
513
+ /**
514
+ * Get a specific run with its results
515
+ */
516
+ async getRun(evaluationId, runId) {
517
+ return this.client.request(`/api/evaluations/${evaluationId}/runs/${runId}`);
518
+ }
519
+ }
520
+ /**
521
+ * LLM Judge API methods
522
+ */
523
+ class LLMJudgeAPI {
524
+ constructor(client) {
525
+ this.client = client;
526
+ }
527
+ /**
528
+ * Run an LLM judge evaluation
529
+ */
530
+ async evaluate(params) {
531
+ return this.client.request("/api/llm-judge/evaluate", {
532
+ method: "POST",
533
+ body: JSON.stringify(params),
534
+ });
535
+ }
536
+ /**
537
+ * Create an LLM judge configuration
538
+ */
539
+ async createConfig(params) {
540
+ return this.client.request("/api/llm-judge/configs", {
541
+ method: "POST",
542
+ body: JSON.stringify(params),
543
+ });
544
+ }
545
+ /**
546
+ * List LLM judge configurations
547
+ */
548
+ async listConfigs(params = {}) {
549
+ const searchParams = new URLSearchParams();
550
+ if (params.organizationId)
551
+ searchParams.set("organizationId", params.organizationId.toString());
552
+ if (params.limit)
553
+ searchParams.set("limit", params.limit.toString());
554
+ if (params.offset)
555
+ searchParams.set("offset", params.offset.toString());
556
+ const query = searchParams.toString();
557
+ const endpoint = query
558
+ ? `/api/llm-judge/configs?${query}`
559
+ : "/api/llm-judge/configs";
560
+ return this.client.request(endpoint);
561
+ }
562
+ /**
563
+ * List LLM judge results
564
+ */
565
+ async listResults(params = {}) {
566
+ const searchParams = new URLSearchParams();
567
+ if (params.configId)
568
+ searchParams.set("configId", params.configId.toString());
569
+ if (params.evaluationId)
570
+ searchParams.set("evaluationId", params.evaluationId.toString());
571
+ if (params.limit)
572
+ searchParams.set("limit", params.limit.toString());
573
+ if (params.offset)
574
+ searchParams.set("offset", params.offset.toString());
575
+ const query = searchParams.toString();
576
+ const endpoint = query
577
+ ? `/api/llm-judge/results?${query}`
578
+ : "/api/llm-judge/results";
579
+ return this.client.request(endpoint);
580
+ }
581
+ /**
582
+ * Get alignment analysis
583
+ */
584
+ async getAlignment(params) {
585
+ const searchParams = new URLSearchParams();
586
+ searchParams.set("evaluationRunId", params.evaluationRunId.toString());
587
+ const query = searchParams.toString();
588
+ return this.client.request(`/api/llm-judge/alignment?${query}`);
589
+ }
590
+ }
591
+ /**
592
+ * Annotations API methods
593
+ */
594
+ class AnnotationsAPI {
595
+ constructor(client) {
596
+ this.client = client;
597
+ this.tasks = new AnnotationTasksAPI(client);
598
+ }
599
+ /**
600
+ * Create an annotation
601
+ */
602
+ async create(params) {
603
+ return this.client
604
+ .request("/api/annotations", {
605
+ method: "POST",
606
+ body: JSON.stringify(params),
607
+ })
608
+ .then((res) => res.annotation);
609
+ }
610
+ /**
611
+ * List annotations
612
+ */
613
+ async list(params = {}) {
614
+ const searchParams = new URLSearchParams();
615
+ if (params.evaluationRunId)
616
+ searchParams.set("evaluationRunId", params.evaluationRunId.toString());
617
+ if (params.testCaseId)
618
+ searchParams.set("testCaseId", params.testCaseId.toString());
619
+ if (params.limit)
620
+ searchParams.set("limit", params.limit.toString());
621
+ if (params.offset)
622
+ searchParams.set("offset", params.offset.toString());
623
+ const query = searchParams.toString();
624
+ const endpoint = query ? `/api/annotations?${query}` : "/api/annotations";
625
+ return this.client
626
+ .request(endpoint)
627
+ .then((res) => res.annotations);
628
+ }
629
+ }
630
+ /**
631
+ * Annotation Tasks API methods
632
+ */
633
+ class AnnotationTasksAPI {
634
+ constructor(client) {
635
+ this.client = client;
636
+ this.items = new AnnotationTaskItemsAPI(client);
637
+ }
638
+ /**
639
+ * Create an annotation task
640
+ */
641
+ async create(params) {
642
+ return this.client.request("/api/annotations/tasks", {
643
+ method: "POST",
644
+ body: JSON.stringify(params),
645
+ });
646
+ }
647
+ /**
648
+ * List annotation tasks
649
+ */
650
+ async list(params = {}) {
651
+ const searchParams = new URLSearchParams();
652
+ if (params.organizationId)
653
+ searchParams.set("organizationId", params.organizationId.toString());
654
+ if (params.status)
655
+ searchParams.set("status", params.status);
656
+ if (params.limit)
657
+ searchParams.set("limit", params.limit.toString());
658
+ if (params.offset)
659
+ searchParams.set("offset", params.offset.toString());
660
+ const query = searchParams.toString();
661
+ const endpoint = query
662
+ ? `/api/annotations/tasks?${query}`
663
+ : "/api/annotations/tasks";
664
+ return this.client.request(endpoint);
665
+ }
666
+ /**
667
+ * Get an annotation task
668
+ */
669
+ async get(taskId) {
670
+ return this.client
671
+ .request(`/api/annotations/tasks/${taskId}`)
672
+ .then((res) => res.task);
673
+ }
674
+ }
675
+ /**
676
+ * Annotation Task Items API methods
677
+ */
678
+ class AnnotationTaskItemsAPI {
679
+ constructor(client) {
680
+ this.client = client;
681
+ }
682
+ /**
683
+ * Create an annotation item
684
+ */
685
+ async create(taskId, params) {
686
+ return this.client.request(`/api/annotations/tasks/${taskId}/items`, {
687
+ method: "POST",
688
+ body: JSON.stringify(params),
689
+ });
690
+ }
691
+ /**
692
+ * List annotation items
693
+ */
694
+ async list(taskId, params = {}) {
695
+ const searchParams = new URLSearchParams();
696
+ if (params.limit)
697
+ searchParams.set("limit", params.limit.toString());
698
+ if (params.offset)
699
+ searchParams.set("offset", params.offset.toString());
700
+ const query = searchParams.toString();
701
+ const endpoint = query
702
+ ? `/api/annotations/tasks/${taskId}/items?${query}`
703
+ : `/api/annotations/tasks/${taskId}/items`;
704
+ return this.client.request(endpoint);
705
+ }
706
+ }
707
+ /**
708
+ * Developer API methods
709
+ */
710
+ class DeveloperAPI {
711
+ constructor(client) {
712
+ this.client = client;
713
+ this.apiKeys = new APIKeysAPI(client);
714
+ this.webhooks = new WebhooksAPI(client);
715
+ }
716
+ /**
717
+ * Get usage statistics
718
+ */
719
+ async getUsage(params = {}) {
720
+ const searchParams = new URLSearchParams();
721
+ if (params.period)
722
+ searchParams.set("period", params.period);
723
+ if (params.groupBy)
724
+ searchParams.set("groupBy", params.groupBy);
725
+ if (params.limit)
726
+ searchParams.set("limit", params.limit.toString());
727
+ if (params.offset)
728
+ searchParams.set("offset", params.offset.toString());
729
+ const query = searchParams.toString();
730
+ const endpoint = query
731
+ ? `/api/developer/usage?${query}`
732
+ : "/api/developer/usage";
733
+ return this.client.request(endpoint);
734
+ }
735
+ /**
736
+ * Get usage summary
737
+ */
738
+ async getUsageSummary(params = {}) {
739
+ const searchParams = new URLSearchParams();
740
+ if (params.period)
741
+ searchParams.set("period", params.period);
742
+ const query = searchParams.toString();
743
+ const endpoint = query
744
+ ? `/api/developer/usage/summary?${query}`
745
+ : "/api/developer/usage/summary";
746
+ return this.client.request(endpoint);
747
+ }
748
+ }
749
+ /**
750
+ * API Keys API methods
751
+ */
752
+ class APIKeysAPI {
753
+ constructor(client) {
754
+ this.client = client;
755
+ }
756
+ /**
757
+ * Create an API key
758
+ */
759
+ async create(params) {
760
+ return this.client.request("/api/developer/api-keys", {
761
+ method: "POST",
762
+ body: JSON.stringify(params),
763
+ });
764
+ }
765
+ /**
766
+ * List API keys
767
+ */
768
+ async list(params = {}) {
769
+ const searchParams = new URLSearchParams();
770
+ if (params.organizationId)
771
+ searchParams.set("organizationId", params.organizationId.toString());
772
+ if (params.limit)
773
+ searchParams.set("limit", params.limit.toString());
774
+ if (params.offset)
775
+ searchParams.set("offset", params.offset.toString());
776
+ const query = searchParams.toString();
777
+ const endpoint = query
778
+ ? `/api/developer/api-keys?${query}`
779
+ : "/api/developer/api-keys";
780
+ return this.client.request(endpoint);
781
+ }
782
+ /**
783
+ * Update an API key
784
+ */
785
+ async update(keyId, params) {
786
+ return this.client.request(`/api/developer/api-keys/${keyId}`, {
787
+ method: "PATCH",
788
+ body: JSON.stringify(params),
789
+ });
790
+ }
791
+ /**
792
+ * Revoke an API key
793
+ */
794
+ async revoke(keyId) {
795
+ return this.client.request(`/api/developer/api-keys/${keyId}`, {
796
+ method: "DELETE",
797
+ });
798
+ }
799
+ /**
800
+ * Get API key usage
801
+ */
802
+ async getUsage(keyId) {
803
+ return this.client.request(`/api/developer/api-keys/${keyId}/usage`);
804
+ }
805
+ }
806
+ /**
807
+ * Webhooks API methods
808
+ */
809
+ class WebhooksAPI {
810
+ constructor(client) {
811
+ this.client = client;
812
+ }
813
+ /**
814
+ * Create a webhook
815
+ */
816
+ async create(params) {
817
+ return this.client.request("/api/developer/webhooks", {
818
+ method: "POST",
819
+ body: JSON.stringify(params),
820
+ });
821
+ }
822
+ /**
823
+ * List webhooks
824
+ */
825
+ async list(params) {
826
+ const searchParams = new URLSearchParams();
827
+ searchParams.set("organizationId", params.organizationId.toString());
828
+ if (params.status)
829
+ searchParams.set("status", params.status);
830
+ if (params.limit)
831
+ searchParams.set("limit", params.limit.toString());
832
+ if (params.offset)
833
+ searchParams.set("offset", params.offset.toString());
834
+ const query = searchParams.toString();
835
+ return this.client.request(`/api/developer/webhooks?${query}`);
836
+ }
837
+ /**
838
+ * Get a webhook
839
+ */
840
+ async get(webhookId) {
841
+ return this.client.request(`/api/developer/webhooks/${webhookId}`);
842
+ }
843
+ /**
844
+ * Update a webhook
845
+ */
846
+ async update(webhookId, params) {
847
+ return this.client.request(`/api/developer/webhooks/${webhookId}`, {
848
+ method: "PATCH",
849
+ body: JSON.stringify(params),
850
+ });
851
+ }
852
+ /**
853
+ * Delete a webhook
854
+ */
855
+ async delete(webhookId) {
856
+ return this.client.request(`/api/developer/webhooks/${webhookId}`, {
857
+ method: "DELETE",
858
+ });
859
+ }
860
+ /**
861
+ * Get webhook deliveries
862
+ */
863
+ async getDeliveries(webhookId, params = {}) {
864
+ const searchParams = new URLSearchParams();
865
+ if (params.limit)
866
+ searchParams.set("limit", params.limit.toString());
867
+ if (params.offset)
868
+ searchParams.set("offset", params.offset.toString());
869
+ if (params.status)
870
+ searchParams.set("status", params.status);
871
+ const query = searchParams.toString();
872
+ const endpoint = query
873
+ ? `/api/developer/webhooks/${webhookId}/deliveries?${query}`
874
+ : `/api/developer/webhooks/${webhookId}/deliveries`;
875
+ return this.client.request(endpoint);
876
+ }
877
+ }
878
+ /**
879
+ * Organizations API methods
880
+ */
881
+ class OrganizationsAPI {
882
+ constructor(client) {
883
+ this.client = client;
884
+ }
885
+ /**
886
+ * Get current organization
887
+ */
888
+ async getCurrent() {
889
+ return this.client
890
+ .request("/api/organizations/current")
891
+ .then((res) => res.organization);
892
+ }
893
+ }