@pauly4010/evalai-sdk 1.4.1 → 1.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +205 -543
  3. package/dist/assertions.d.ts +2 -2
  4. package/dist/assertions.js +104 -71
  5. package/dist/batch.js +12 -17
  6. package/dist/cache.js +7 -11
  7. package/dist/cli/api.d.ts +108 -0
  8. package/dist/cli/api.js +130 -0
  9. package/dist/cli/check.d.ts +28 -13
  10. package/dist/cli/check.js +249 -142
  11. package/dist/cli/ci-context.d.ts +6 -0
  12. package/dist/cli/ci-context.js +110 -0
  13. package/dist/cli/config.d.ts +30 -0
  14. package/dist/cli/config.js +207 -0
  15. package/dist/cli/constants.d.ts +15 -0
  16. package/dist/cli/constants.js +18 -0
  17. package/dist/cli/doctor.d.ts +11 -0
  18. package/dist/cli/doctor.js +82 -0
  19. package/dist/cli/formatters/github.d.ts +8 -0
  20. package/dist/cli/formatters/github.js +130 -0
  21. package/dist/cli/formatters/human.d.ts +6 -0
  22. package/dist/cli/formatters/human.js +107 -0
  23. package/dist/cli/formatters/json.d.ts +6 -0
  24. package/dist/cli/formatters/json.js +10 -0
  25. package/dist/cli/formatters/pr-comment.d.ts +12 -0
  26. package/dist/cli/formatters/pr-comment.js +101 -0
  27. package/dist/cli/formatters/types.d.ts +100 -0
  28. package/dist/cli/formatters/types.js +5 -0
  29. package/dist/cli/gate.d.ts +21 -0
  30. package/dist/cli/gate.js +175 -0
  31. package/dist/cli/index.d.ts +1 -0
  32. package/dist/cli/index.js +67 -23
  33. package/dist/cli/init.d.ts +7 -0
  34. package/dist/cli/init.js +69 -0
  35. package/dist/cli/policy-packs.d.ts +23 -0
  36. package/dist/cli/policy-packs.js +83 -0
  37. package/dist/cli/profiles.d.ts +28 -0
  38. package/dist/cli/profiles.js +30 -0
  39. package/dist/cli/reason-codes.d.ts +17 -0
  40. package/dist/cli/reason-codes.js +19 -0
  41. package/dist/cli/render/snippet.d.ts +5 -0
  42. package/dist/cli/render/snippet.js +15 -0
  43. package/dist/cli/render/sort.d.ts +10 -0
  44. package/dist/cli/render/sort.js +24 -0
  45. package/dist/cli/report/build-check-report.d.ts +19 -0
  46. package/dist/cli/report/build-check-report.js +124 -0
  47. package/dist/cli/share.d.ts +17 -0
  48. package/dist/cli/share.js +83 -0
  49. package/dist/client.d.ts +2 -2
  50. package/dist/client.js +144 -132
  51. package/dist/context.d.ts +1 -1
  52. package/dist/context.js +4 -6
  53. package/dist/errors.d.ts +2 -0
  54. package/dist/errors.js +116 -107
  55. package/dist/export.d.ts +6 -6
  56. package/dist/export.js +39 -33
  57. package/dist/index.d.ts +25 -24
  58. package/dist/index.js +62 -56
  59. package/dist/integrations/anthropic.d.ts +1 -1
  60. package/dist/integrations/anthropic.js +23 -19
  61. package/dist/integrations/openai-eval.d.ts +57 -0
  62. package/dist/integrations/openai-eval.js +230 -0
  63. package/dist/integrations/openai.d.ts +1 -1
  64. package/dist/integrations/openai.js +23 -19
  65. package/dist/local.d.ts +2 -2
  66. package/dist/local.js +25 -25
  67. package/dist/logger.d.ts +1 -1
  68. package/dist/logger.js +24 -28
  69. package/dist/matchers/index.d.ts +1 -0
  70. package/dist/matchers/index.js +6 -0
  71. package/dist/matchers/to-pass-gate.d.ts +29 -0
  72. package/dist/matchers/to-pass-gate.js +35 -0
  73. package/dist/pagination.d.ts +1 -1
  74. package/dist/pagination.js +6 -6
  75. package/dist/snapshot.js +24 -24
  76. package/dist/streaming.js +11 -11
  77. package/dist/testing.d.ts +6 -2
  78. package/dist/testing.js +30 -12
  79. package/dist/types.d.ts +22 -22
  80. package/dist/types.js +13 -13
  81. package/dist/utils/input-hash.d.ts +8 -0
  82. package/dist/utils/input-hash.js +38 -0
  83. package/dist/version.d.ts +7 -0
  84. package/dist/version.js +10 -0
  85. package/dist/workflows.d.ts +7 -7
  86. package/dist/workflows.js +44 -44
  87. package/package.json +102 -90
  88. package/dist/__tests__/assertions.test.d.ts +0 -1
  89. package/dist/__tests__/assertions.test.js +0 -288
  90. package/dist/__tests__/client.test.d.ts +0 -1
  91. package/dist/__tests__/client.test.js +0 -185
  92. package/dist/__tests__/testing.test.d.ts +0 -1
  93. package/dist/__tests__/testing.test.js +0 -230
  94. package/dist/__tests__/workflows.test.d.ts +0 -1
  95. package/dist/__tests__/workflows.test.js +0 -222
@@ -0,0 +1,230 @@
1
+ "use strict";
2
+ /**
3
+ * openAIChatEval — One-function OpenAI chat regression testing
4
+ *
5
+ * Run local regression tests with OpenAI. No EvalAI account required.
6
+ * CI-friendly output. Optional reportToEvalAI in v1.5.
7
+ *
8
+ * @example
9
+ * ```typescript
10
+ * import { openAIChatEval } from '@pauly4010/evalai-sdk';
11
+ *
12
+ * await openAIChatEval({
13
+ * name: 'chat-regression',
14
+ * cases: [
15
+ * { input: 'Hello', expectedOutput: 'greeting' },
16
+ * { input: '2 + 2 = ?', expectedOutput: '4' }
17
+ * ]
18
+ * });
19
+ * ```
20
+ */
21
+ Object.defineProperty(exports, "__esModule", { value: true });
22
+ exports.openAIChatEval = openAIChatEval;
23
+ const assertions_1 = require("../assertions");
24
+ const config_1 = require("../cli/config");
25
+ const testing_1 = require("../testing");
26
+ const input_hash_1 = require("../utils/input-hash");
27
+ const MAX_FAILED_CASES_TO_SHOW = 5;
28
+ function getOpenAI() {
29
+ try {
30
+ const OpenAI = require("openai");
31
+ return OpenAI;
32
+ }
33
+ catch {
34
+ throw new Error("openai package is required for openAIChatEval. Install with: npm install openai");
35
+ }
36
+ }
37
+ function createExecutor(model, apiKey) {
38
+ const OpenAI = getOpenAI();
39
+ const openai = new OpenAI({ apiKey });
40
+ return async (input) => {
41
+ const response = await openai.chat.completions.create({
42
+ model,
43
+ messages: [{ role: "user", content: input }],
44
+ temperature: 0.1,
45
+ });
46
+ return response.choices[0]?.message?.content ?? "";
47
+ };
48
+ }
49
+ function printSummary(result) {
50
+ const { passed, total, results } = result;
51
+ const score = total > 0 ? Math.round((passed / total) * 100) : 0;
52
+ const failed = results.filter((r) => !r.passed);
53
+ const status = failed.length === 0 ? "PASS" : "FAIL";
54
+ console.log(`\n${status} ${passed}/${total} (score: ${score})\n`);
55
+ if (failed.length > 0) {
56
+ const toShow = failed.slice(0, MAX_FAILED_CASES_TO_SHOW);
57
+ const more = failed.length - toShow.length;
58
+ console.log(`${failed.length} failing case${failed.length === 1 ? "" : "s"}:`);
59
+ for (const r of toShow) {
60
+ const expected = r.expected ?? "(no expected)";
61
+ console.log(`- "${r.input}" → expected: ${expected}`);
62
+ }
63
+ if (more > 0) {
64
+ console.log(`+ ${more} more`);
65
+ }
66
+ console.log("\nGate this in CI:");
67
+ console.log(" npx -y @pauly4010/evalai-sdk@^1 init");
68
+ }
69
+ else {
70
+ console.log("Tip: Want dashboards and history?");
71
+ console.log("Set EVALAI_API_KEY and connect this to the platform.");
72
+ }
73
+ }
74
+ /**
75
+ * Run OpenAI chat regression tests locally.
76
+ * No EvalAI account required. Returns score and prints CI-friendly summary.
77
+ */
78
+ async function openAIChatEval(options) {
79
+ const { name, model = "gpt-4o-mini", apiKey, cases, retries = 0 } = options;
80
+ const resolvedApiKey = apiKey ?? (typeof process !== "undefined" && process.env?.OPENAI_API_KEY);
81
+ if (!resolvedApiKey) {
82
+ throw new Error("OPENAI_API_KEY is required. Set it in the environment or pass apiKey to openAIChatEval.");
83
+ }
84
+ const executor = createExecutor(model, resolvedApiKey);
85
+ const suiteCases = cases.map((c) => {
86
+ const assertions = c.assertions
87
+ ? [...c.assertions]
88
+ : c.expectedOutput
89
+ ? [
90
+ (output) => (0, assertions_1.expect)(output).toContainKeywords(c.expectedOutput.split(/\s+/).filter(Boolean)),
91
+ ]
92
+ : undefined;
93
+ return {
94
+ input: c.input,
95
+ expected: c.expectedOutput,
96
+ assertions,
97
+ };
98
+ });
99
+ const suite = (0, testing_1.createTestSuite)(name, {
100
+ cases: suiteCases,
101
+ executor,
102
+ parallel: true,
103
+ retries,
104
+ });
105
+ const result = await suite.run();
106
+ const score = result.total > 0 ? Math.round((result.passed / result.total) * 100) : 0;
107
+ const evalResult = {
108
+ passed: result.passed,
109
+ total: result.total,
110
+ score,
111
+ results: result.results,
112
+ durationMs: result.durationMs,
113
+ ...(result.retriedCases &&
114
+ result.retriedCases.length > 0 && { retriedCases: result.retriedCases }),
115
+ };
116
+ printSummary(evalResult);
117
+ // v1.5: Optional report to EvalAI platform
118
+ if (options.reportToEvalAI) {
119
+ const config = typeof process !== "undefined" && process.cwd ? (0, config_1.loadConfig)(process.cwd()) : null;
120
+ const evalId = options.evaluationId || config?.evaluationId;
121
+ if (!evalId || String(evalId).trim() === "") {
122
+ console.log("Run evalai init and set evaluationId to upload results.");
123
+ return evalResult;
124
+ }
125
+ const evalaiKey = (typeof process !== "undefined" && process.env?.EVALAI_API_KEY) || "";
126
+ if (!evalaiKey) {
127
+ console.log("Set EVALAI_API_KEY to upload results.");
128
+ return evalResult;
129
+ }
130
+ const baseUrl = options.baseUrl ||
131
+ config?.baseUrl ||
132
+ (typeof process !== "undefined" && process.env?.EVALAI_BASE_URL) ||
133
+ "http://localhost:3000";
134
+ const url = String(baseUrl).replace(/\/$/, "");
135
+ try {
136
+ // Resolve testCaseId for each result: explicit testCaseId in cases, or match by inputHash
137
+ const importResults = [];
138
+ const hasExplicitIds = cases.some((c) => c.testCaseId != null);
139
+ if (hasExplicitIds) {
140
+ // Use testCaseId from cases (same order as results)
141
+ for (let i = 0; i < result.results.length; i++) {
142
+ const tcId = cases[i]?.testCaseId;
143
+ if (tcId == null) {
144
+ console.log("reportToEvalAI: All cases must have testCaseId when any has it.");
145
+ return evalResult;
146
+ }
147
+ importResults.push({
148
+ testCaseId: tcId,
149
+ status: result.results[i].passed ? "passed" : "failed",
150
+ output: result.results[i].actual ?? "",
151
+ latencyMs: result.results[i].durationMs,
152
+ });
153
+ }
154
+ }
155
+ else {
156
+ // Match by inputHash (same canonicalization as platform)
157
+ const tcRes = await fetch(`${url}/api/evaluations/${evalId}/test-cases?limit=500`, {
158
+ headers: { Authorization: `Bearer ${evalaiKey}` },
159
+ });
160
+ if (!tcRes.ok) {
161
+ console.log("Could not fetch test cases. Check evaluationId and EVALAI_API_KEY.");
162
+ return evalResult;
163
+ }
164
+ const platformCases = (await tcRes.json());
165
+ const hashToIds = new Map();
166
+ for (const tc of platformCases) {
167
+ const input = tc.input ?? "";
168
+ if (!input.trim())
169
+ continue;
170
+ const hash = (0, input_hash_1.sha256Input)(input);
171
+ const existing = hashToIds.get(hash) ?? [];
172
+ existing.push(tc.id);
173
+ hashToIds.set(hash, existing);
174
+ }
175
+ for (const r of result.results) {
176
+ const hash = (0, input_hash_1.sha256Input)(r.input ?? "");
177
+ const ids = hashToIds.get(hash);
178
+ if (ids == null || ids.length === 0) {
179
+ console.log(`No platform test case matches input: "${(r.input ?? "").slice(0, 50)}…"`);
180
+ return evalResult;
181
+ }
182
+ if (ids.length > 1) {
183
+ console.log(`Multiple platform test cases share the same input (hash collision). Use testCaseId in cases.`);
184
+ return evalResult;
185
+ }
186
+ importResults.push({
187
+ testCaseId: ids[0],
188
+ status: r.passed ? "passed" : "failed",
189
+ output: r.actual ?? "",
190
+ latencyMs: r.durationMs,
191
+ });
192
+ }
193
+ }
194
+ if (importResults.length !== result.results.length) {
195
+ console.log("Could not match all results to platform test cases.");
196
+ return evalResult;
197
+ }
198
+ const sdkVersion = "1.4.1";
199
+ const headers = {
200
+ "Content-Type": "application/json",
201
+ Authorization: `Bearer ${evalaiKey}`,
202
+ };
203
+ if (options.idempotencyKey) {
204
+ headers["Idempotency-Key"] = options.idempotencyKey;
205
+ }
206
+ const importRes = await fetch(`${url}/api/evaluations/${evalId}/runs/import`, {
207
+ method: "POST",
208
+ headers,
209
+ body: JSON.stringify({
210
+ environment: "dev",
211
+ results: importResults,
212
+ importClientVersion: sdkVersion,
213
+ }),
214
+ });
215
+ if (!importRes.ok) {
216
+ const body = await importRes.text();
217
+ console.log(`Upload failed: ${importRes.status} — ${body}`);
218
+ return evalResult;
219
+ }
220
+ const importData = (await importRes.json());
221
+ if (importData.dashboardUrl) {
222
+ console.log(`Dashboard: ${importData.dashboardUrl}`);
223
+ }
224
+ }
225
+ catch (err) {
226
+ console.log("Upload failed:", err instanceof Error ? err.message : String(err));
227
+ }
228
+ }
229
+ return evalResult;
230
+ }
@@ -17,7 +17,7 @@
17
17
  * });
18
18
  * ```
19
19
  */
20
- import type { AIEvalClient } from '../client';
20
+ import type { AIEvalClient } from "../client";
21
21
  export interface OpenAITraceOptions {
22
22
  /** Whether to capture input (default: true) */
23
23
  captureInput?: boolean;
@@ -41,7 +41,7 @@ const context_1 = require("../context");
41
41
  * ```
42
42
  */
43
43
  function traceOpenAI(openai, evalClient, options = {}) {
44
- const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = 'openai' } = options;
44
+ const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = "openai", } = options;
45
45
  // Create proxy for chat.completions.create
46
46
  const originalCreate = openai.chat.completions.create.bind(openai.chat.completions);
47
47
  openai.chat.completions.create = async (params, requestOptions) => {
@@ -58,18 +58,20 @@ function traceOpenAI(openai, evalClient, options = {}) {
58
58
  max_tokens: params.max_tokens,
59
59
  ...(captureInput ? { input: params.messages } : {}),
60
60
  ...(captureOutput ? { output: response.choices[0]?.message } : {}),
61
- ...(captureMetadata ? {
62
- usage: response.usage,
63
- finish_reason: response.choices[0]?.finish_reason
64
- } : {})
61
+ ...(captureMetadata
62
+ ? {
63
+ usage: response.usage,
64
+ finish_reason: response.choices[0]?.finish_reason,
65
+ }
66
+ : {}),
65
67
  });
66
68
  await evalClient.traces.create({
67
69
  name: `OpenAI: ${params.model}`,
68
70
  traceId,
69
71
  organizationId: organizationId || evalClient.getOrganizationId(),
70
- status: 'success',
72
+ status: "success",
71
73
  durationMs,
72
- metadata: traceMetadata
74
+ metadata: traceMetadata,
73
75
  });
74
76
  return response;
75
77
  }
@@ -82,16 +84,18 @@ function traceOpenAI(openai, evalClient, options = {}) {
82
84
  max_tokens: params.max_tokens,
83
85
  ...(captureInput ? { input: params.messages } : {}),
84
86
  ...(captureMetadata ? { params } : {}),
85
- error: error instanceof Error ? error.message : String(error)
87
+ error: error instanceof Error ? error.message : String(error),
86
88
  });
87
- await evalClient.traces.create({
89
+ await evalClient.traces
90
+ .create({
88
91
  name: `OpenAI: ${params.model}`,
89
92
  traceId,
90
93
  organizationId: organizationId || evalClient.getOrganizationId(),
91
- status: 'error',
94
+ status: "error",
92
95
  durationMs,
93
- metadata: errorMetadata
94
- }).catch(() => {
96
+ metadata: errorMetadata,
97
+ })
98
+ .catch(() => {
95
99
  // Ignore errors in trace creation to avoid masking the original error
96
100
  });
97
101
  throw error;
@@ -124,8 +128,8 @@ async function traceOpenAICall(evalClient, name, fn, options = {}) {
124
128
  name,
125
129
  traceId,
126
130
  organizationId: options.organizationId || evalClient.getOrganizationId(),
127
- status: 'pending',
128
- metadata: (0, context_1.mergeWithContext)({})
131
+ status: "pending",
132
+ metadata: (0, context_1.mergeWithContext)({}),
129
133
  });
130
134
  const result = await fn();
131
135
  const durationMs = Date.now() - startTime;
@@ -133,9 +137,9 @@ async function traceOpenAICall(evalClient, name, fn, options = {}) {
133
137
  name,
134
138
  traceId,
135
139
  organizationId: options.organizationId || evalClient.getOrganizationId(),
136
- status: 'success',
140
+ status: "success",
137
141
  durationMs,
138
- metadata: (0, context_1.mergeWithContext)({})
142
+ metadata: (0, context_1.mergeWithContext)({}),
139
143
  });
140
144
  return result;
141
145
  }
@@ -145,11 +149,11 @@ async function traceOpenAICall(evalClient, name, fn, options = {}) {
145
149
  name,
146
150
  traceId,
147
151
  organizationId: options.organizationId || evalClient.getOrganizationId(),
148
- status: 'error',
152
+ status: "error",
149
153
  durationMs,
150
154
  metadata: (0, context_1.mergeWithContext)({
151
- error: error instanceof Error ? error.message : String(error)
152
- })
155
+ error: error instanceof Error ? error.message : String(error),
156
+ }),
153
157
  });
154
158
  throw error;
155
159
  }
package/dist/local.d.ts CHANGED
@@ -4,7 +4,7 @@
4
4
  *
5
5
  * ⚠️ NOTE: This module requires Node.js and will not work in browsers.
6
6
  */
7
- import type { Trace, Evaluation, Span } from './types';
7
+ import type { Evaluation, Span, Trace } from "./types";
8
8
  export interface LocalStorageOptions {
9
9
  directory?: string;
10
10
  autoSave?: boolean;
@@ -30,7 +30,7 @@ export declare class LocalStorage {
30
30
  saveSpans(traceId: string, spans: Span[]): Promise<void>;
31
31
  getSpans(traceId: string): Promise<Span[] | undefined>;
32
32
  clear(): Promise<void>;
33
- export(format: 'json'): Promise<string>;
33
+ export(_format: "json"): Promise<string>;
34
34
  getStats(): {
35
35
  traces: number;
36
36
  evaluations: number;
package/dist/local.js CHANGED
@@ -11,64 +11,64 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
12
  exports.LocalStorage = void 0;
13
13
  // Environment check
14
- const isNode = typeof process !== 'undefined' && process.versions?.node;
14
+ const isNode = typeof process !== "undefined" && process.versions?.node;
15
15
  if (!isNode) {
16
- throw new Error('Local storage mode requires Node.js and cannot run in browsers. ' +
17
- 'This feature uses the filesystem for storing data.');
16
+ throw new Error("Local storage mode requires Node.js and cannot run in browsers. " +
17
+ "This feature uses the filesystem for storing data.");
18
18
  }
19
- const promises_1 = __importDefault(require("fs/promises"));
20
- const path_1 = __importDefault(require("path"));
19
+ const promises_1 = __importDefault(require("node:fs/promises"));
20
+ const node_path_1 = __importDefault(require("node:path"));
21
21
  class LocalStorage {
22
22
  constructor(options = {}) {
23
23
  this.traces = new Map();
24
24
  this.evaluations = new Map();
25
25
  this.spans = new Map();
26
- this.directory = options.directory || './.evalai-data';
26
+ this.directory = options.directory || "./.evalai-data";
27
27
  this.autoSave = options.autoSave !== false;
28
28
  this.initialize();
29
29
  }
30
30
  async initialize() {
31
31
  try {
32
32
  await promises_1.default.mkdir(this.directory, { recursive: true });
33
- await promises_1.default.mkdir(path_1.default.join(this.directory, 'traces'), { recursive: true });
34
- await promises_1.default.mkdir(path_1.default.join(this.directory, 'evaluations'), { recursive: true });
35
- await promises_1.default.mkdir(path_1.default.join(this.directory, 'spans'), { recursive: true });
33
+ await promises_1.default.mkdir(node_path_1.default.join(this.directory, "traces"), { recursive: true });
34
+ await promises_1.default.mkdir(node_path_1.default.join(this.directory, "evaluations"), { recursive: true });
35
+ await promises_1.default.mkdir(node_path_1.default.join(this.directory, "spans"), { recursive: true });
36
36
  // Load existing data
37
37
  await this.loadAllData();
38
38
  }
39
39
  catch (error) {
40
- console.warn('Failed to initialize local storage:', error);
40
+ console.warn("Failed to initialize local storage:", error);
41
41
  }
42
42
  }
43
43
  async loadAllData() {
44
44
  try {
45
45
  // Load traces
46
- const tracesDir = path_1.default.join(this.directory, 'traces');
46
+ const tracesDir = node_path_1.default.join(this.directory, "traces");
47
47
  const traceFiles = await promises_1.default.readdir(tracesDir);
48
48
  for (const file of traceFiles) {
49
- if (file.endsWith('.json')) {
50
- const content = await promises_1.default.readFile(path_1.default.join(tracesDir, file), 'utf-8');
49
+ if (file.endsWith(".json")) {
50
+ const content = await promises_1.default.readFile(node_path_1.default.join(tracesDir, file), "utf-8");
51
51
  const trace = JSON.parse(content);
52
52
  this.traces.set(trace.id.toString(), trace);
53
53
  }
54
54
  }
55
55
  // Load evaluations
56
- const evalsDir = path_1.default.join(this.directory, 'evaluations');
56
+ const evalsDir = node_path_1.default.join(this.directory, "evaluations");
57
57
  const evalFiles = await promises_1.default.readdir(evalsDir);
58
58
  for (const file of evalFiles) {
59
- if (file.endsWith('.json')) {
60
- const content = await promises_1.default.readFile(path_1.default.join(evalsDir, file), 'utf-8');
59
+ if (file.endsWith(".json")) {
60
+ const content = await promises_1.default.readFile(node_path_1.default.join(evalsDir, file), "utf-8");
61
61
  const evaluation = JSON.parse(content);
62
62
  this.evaluations.set(evaluation.id.toString(), evaluation);
63
63
  }
64
64
  }
65
65
  }
66
- catch (error) {
66
+ catch (_error) {
67
67
  // Directories might not exist yet, that's fine
68
68
  }
69
69
  }
70
70
  async saveTraceToDisk(trace) {
71
- const filePath = path_1.default.join(this.directory, 'traces', `${trace.id}.json`);
71
+ const filePath = node_path_1.default.join(this.directory, "traces", `${trace.id}.json`);
72
72
  await promises_1.default.writeFile(filePath, JSON.stringify(trace, null, 2));
73
73
  }
74
74
  async saveTrace(trace) {
@@ -84,7 +84,7 @@ class LocalStorage {
84
84
  return Array.from(this.traces.values());
85
85
  }
86
86
  async saveEvaluationToDisk(evaluation) {
87
- const filePath = path_1.default.join(this.directory, 'evaluations', `${evaluation.id}.json`);
87
+ const filePath = node_path_1.default.join(this.directory, "evaluations", `${evaluation.id}.json`);
88
88
  await promises_1.default.writeFile(filePath, JSON.stringify(evaluation, null, 2));
89
89
  }
90
90
  async saveEvaluation(evaluation) {
@@ -100,7 +100,7 @@ class LocalStorage {
100
100
  return Array.from(this.evaluations.values());
101
101
  }
102
102
  async saveSpansToDisk(traceId, spans) {
103
- const filePath = path_1.default.join(this.directory, 'spans', `${traceId}.json`);
103
+ const filePath = node_path_1.default.join(this.directory, "spans", `${traceId}.json`);
104
104
  await promises_1.default.writeFile(filePath, JSON.stringify(spans, null, 2));
105
105
  }
106
106
  async saveSpans(traceId, spans) {
@@ -122,16 +122,16 @@ class LocalStorage {
122
122
  await this.initialize();
123
123
  }
124
124
  catch (error) {
125
- console.warn('Failed to clear local storage:', error);
125
+ console.warn("Failed to clear local storage:", error);
126
126
  }
127
127
  }
128
- async export(format) {
128
+ async export(_format) {
129
129
  const data = {
130
130
  traces: Array.from(this.traces.values()),
131
131
  evaluations: Array.from(this.evaluations.values()),
132
- spans: Object.fromEntries(this.spans)
132
+ spans: Object.fromEntries(this.spans),
133
133
  };
134
- const exportPath = path_1.default.join(this.directory, `export-${Date.now()}.json`);
134
+ const exportPath = node_path_1.default.join(this.directory, `export-${Date.now()}.json`);
135
135
  await promises_1.default.writeFile(exportPath, JSON.stringify(data, null, 2));
136
136
  return exportPath;
137
137
  }
@@ -139,7 +139,7 @@ class LocalStorage {
139
139
  return {
140
140
  traces: this.traces.size,
141
141
  evaluations: this.evaluations.size,
142
- totalSpans: Array.from(this.spans.values()).reduce((sum, spans) => sum + spans.length, 0)
142
+ totalSpans: Array.from(this.spans.values()).reduce((sum, spans) => sum + spans.length, 0),
143
143
  };
144
144
  }
145
145
  }
package/dist/logger.d.ts CHANGED
@@ -12,7 +12,7 @@
12
12
  * logger.error('Request failed', { error: err });
13
13
  * ```
14
14
  */
15
- export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error';
15
+ export type LogLevel = "trace" | "debug" | "info" | "warn" | "error";
16
16
  export interface LoggerOptions {
17
17
  /** Log level (default: 'info') */
18
18
  level?: LogLevel;
package/dist/logger.js CHANGED
@@ -23,58 +23,58 @@ const LOG_LEVELS = {
23
23
  debug: 1,
24
24
  info: 2,
25
25
  warn: 3,
26
- error: 4
26
+ error: 4,
27
27
  };
28
28
  const LOG_COLORS = {
29
- trace: '\x1b[90m', // gray
30
- debug: '\x1b[36m', // cyan
31
- info: '\x1b[32m', // green
32
- warn: '\x1b[33m', // yellow
33
- error: '\x1b[31m' // red
29
+ trace: "\x1b[90m", // gray
30
+ debug: "\x1b[36m", // cyan
31
+ info: "\x1b[32m", // green
32
+ warn: "\x1b[33m", // yellow
33
+ error: "\x1b[31m", // red
34
34
  };
35
- const COLOR_RESET = '\x1b[0m';
35
+ const COLOR_RESET = "\x1b[0m";
36
36
  /**
37
37
  * Logger for SDK debugging and troubleshooting
38
38
  */
39
39
  class Logger {
40
40
  constructor(options = {}) {
41
41
  this.options = {
42
- level: options.level || 'info',
42
+ level: options.level || "info",
43
43
  pretty: options.pretty ?? false,
44
44
  timestamps: options.timestamps ?? true,
45
45
  handler: options.handler || this.defaultHandler.bind(this),
46
- prefix: options.prefix || 'EvalAI'
46
+ prefix: options.prefix || "EvalAI",
47
47
  };
48
48
  }
49
49
  /**
50
50
  * Log a trace message
51
51
  */
52
52
  trace(message, data) {
53
- this.log('trace', message, data);
53
+ this.log("trace", message, data);
54
54
  }
55
55
  /**
56
56
  * Log a debug message
57
57
  */
58
58
  debug(message, data) {
59
- this.log('debug', message, data);
59
+ this.log("debug", message, data);
60
60
  }
61
61
  /**
62
62
  * Log an info message
63
63
  */
64
64
  info(message, data) {
65
- this.log('info', message, data);
65
+ this.log("info", message, data);
66
66
  }
67
67
  /**
68
68
  * Log a warning message
69
69
  */
70
70
  warn(message, data) {
71
- this.log('warn', message, data);
71
+ this.log("warn", message, data);
72
72
  }
73
73
  /**
74
74
  * Log an error message
75
75
  */
76
76
  error(message, data) {
77
- this.log('error', message, data);
77
+ this.log("error", message, data);
78
78
  }
79
79
  /**
80
80
  * Log HTTP request
@@ -86,7 +86,7 @@ class Logger {
86
86
  * Log HTTP response
87
87
  */
88
88
  logResponse(method, url, status, duration, data) {
89
- const level = status >= 400 ? 'error' : status >= 300 ? 'warn' : 'debug';
89
+ const level = status >= 400 ? "error" : status >= 300 ? "warn" : "debug";
90
90
  this.log(level, `← ${method} ${url} ${status} (${duration}ms)`, data);
91
91
  }
92
92
  /**
@@ -95,7 +95,7 @@ class Logger {
95
95
  child(prefix) {
96
96
  return new Logger({
97
97
  ...this.options,
98
- prefix: `${this.options.prefix}:${prefix}`
98
+ prefix: `${this.options.prefix}:${prefix}`,
99
99
  });
100
100
  }
101
101
  /**
@@ -118,7 +118,7 @@ class Logger {
118
118
  message,
119
119
  timestamp: new Date().toISOString(),
120
120
  data,
121
- prefix: this.options.prefix
121
+ prefix: this.options.prefix,
122
122
  };
123
123
  this.options.handler(entry);
124
124
  }
@@ -133,23 +133,19 @@ class Logger {
133
133
  }
134
134
  // Level
135
135
  const levelStr = entry.level.toUpperCase().padEnd(5);
136
- parts.push(this.options.pretty
137
- ? `${LOG_COLORS[entry.level]}${levelStr}${COLOR_RESET}`
138
- : levelStr);
136
+ parts.push(this.options.pretty ? `${LOG_COLORS[entry.level]}${levelStr}${COLOR_RESET}` : levelStr);
139
137
  // Prefix
140
138
  if (entry.prefix) {
141
- parts.push(this.options.pretty
142
- ? `\x1b[35m[${entry.prefix}]${COLOR_RESET}`
143
- : `[${entry.prefix}]`);
139
+ parts.push(this.options.pretty ? `\x1b[35m[${entry.prefix}]${COLOR_RESET}` : `[${entry.prefix}]`);
144
140
  }
145
141
  // Message
146
142
  parts.push(entry.message);
147
143
  // Log
148
- const logLine = parts.join(' ');
149
- if (entry.level === 'error') {
144
+ const logLine = parts.join(" ");
145
+ if (entry.level === "error") {
150
146
  console.error(logLine);
151
147
  }
152
- else if (entry.level === 'warn') {
148
+ else if (entry.level === "warn") {
153
149
  console.warn(logLine);
154
150
  }
155
151
  else {
@@ -211,7 +207,7 @@ class RequestLogger {
211
207
  logRequest(request) {
212
208
  this.logger.logRequest(request.method, request.url, {
213
209
  headers: request.headers,
214
- body: request.body
210
+ body: request.body,
215
211
  });
216
212
  }
217
213
  /**
@@ -220,7 +216,7 @@ class RequestLogger {
220
216
  logResponse(response) {
221
217
  this.logger.logResponse(response.method, response.url, response.status, response.duration, {
222
218
  headers: response.headers,
223
- body: response.body
219
+ body: response.body,
224
220
  });
225
221
  }
226
222
  }
@@ -0,0 +1 @@
1
+ export { extendExpectWithToPassGate, toPassGate } from "./to-pass-gate";
@@ -0,0 +1,6 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.toPassGate = exports.extendExpectWithToPassGate = void 0;
4
+ var to_pass_gate_1 = require("./to-pass-gate");
5
+ Object.defineProperty(exports, "extendExpectWithToPassGate", { enumerable: true, get: function () { return to_pass_gate_1.extendExpectWithToPassGate; } });
6
+ Object.defineProperty(exports, "toPassGate", { enumerable: true, get: function () { return to_pass_gate_1.toPassGate; } });