@pauly4010/evalai-sdk 1.4.1 → 1.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +205 -543
- package/dist/assertions.d.ts +2 -2
- package/dist/assertions.js +104 -71
- package/dist/batch.js +12 -17
- package/dist/cache.js +7 -11
- package/dist/cli/api.d.ts +108 -0
- package/dist/cli/api.js +130 -0
- package/dist/cli/check.d.ts +28 -13
- package/dist/cli/check.js +249 -142
- package/dist/cli/ci-context.d.ts +6 -0
- package/dist/cli/ci-context.js +110 -0
- package/dist/cli/config.d.ts +30 -0
- package/dist/cli/config.js +207 -0
- package/dist/cli/constants.d.ts +15 -0
- package/dist/cli/constants.js +18 -0
- package/dist/cli/doctor.d.ts +11 -0
- package/dist/cli/doctor.js +82 -0
- package/dist/cli/formatters/github.d.ts +8 -0
- package/dist/cli/formatters/github.js +130 -0
- package/dist/cli/formatters/human.d.ts +6 -0
- package/dist/cli/formatters/human.js +107 -0
- package/dist/cli/formatters/json.d.ts +6 -0
- package/dist/cli/formatters/json.js +10 -0
- package/dist/cli/formatters/pr-comment.d.ts +12 -0
- package/dist/cli/formatters/pr-comment.js +101 -0
- package/dist/cli/formatters/types.d.ts +100 -0
- package/dist/cli/formatters/types.js +5 -0
- package/dist/cli/gate.d.ts +21 -0
- package/dist/cli/gate.js +175 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +67 -23
- package/dist/cli/init.d.ts +7 -0
- package/dist/cli/init.js +69 -0
- package/dist/cli/policy-packs.d.ts +23 -0
- package/dist/cli/policy-packs.js +83 -0
- package/dist/cli/profiles.d.ts +28 -0
- package/dist/cli/profiles.js +30 -0
- package/dist/cli/reason-codes.d.ts +17 -0
- package/dist/cli/reason-codes.js +19 -0
- package/dist/cli/render/snippet.d.ts +5 -0
- package/dist/cli/render/snippet.js +15 -0
- package/dist/cli/render/sort.d.ts +10 -0
- package/dist/cli/render/sort.js +24 -0
- package/dist/cli/report/build-check-report.d.ts +19 -0
- package/dist/cli/report/build-check-report.js +124 -0
- package/dist/cli/share.d.ts +17 -0
- package/dist/cli/share.js +83 -0
- package/dist/client.d.ts +2 -2
- package/dist/client.js +144 -132
- package/dist/context.d.ts +1 -1
- package/dist/context.js +4 -6
- package/dist/errors.d.ts +2 -0
- package/dist/errors.js +116 -107
- package/dist/export.d.ts +6 -6
- package/dist/export.js +39 -33
- package/dist/index.d.ts +25 -24
- package/dist/index.js +62 -56
- package/dist/integrations/anthropic.d.ts +1 -1
- package/dist/integrations/anthropic.js +23 -19
- package/dist/integrations/openai-eval.d.ts +57 -0
- package/dist/integrations/openai-eval.js +230 -0
- package/dist/integrations/openai.d.ts +1 -1
- package/dist/integrations/openai.js +23 -19
- package/dist/local.d.ts +2 -2
- package/dist/local.js +25 -25
- package/dist/logger.d.ts +1 -1
- package/dist/logger.js +24 -28
- package/dist/matchers/index.d.ts +1 -0
- package/dist/matchers/index.js +6 -0
- package/dist/matchers/to-pass-gate.d.ts +29 -0
- package/dist/matchers/to-pass-gate.js +35 -0
- package/dist/pagination.d.ts +1 -1
- package/dist/pagination.js +6 -6
- package/dist/snapshot.js +24 -24
- package/dist/streaming.js +11 -11
- package/dist/testing.d.ts +6 -2
- package/dist/testing.js +30 -12
- package/dist/types.d.ts +22 -22
- package/dist/types.js +13 -13
- package/dist/utils/input-hash.d.ts +8 -0
- package/dist/utils/input-hash.js +38 -0
- package/dist/version.d.ts +7 -0
- package/dist/version.js +10 -0
- package/dist/workflows.d.ts +7 -7
- package/dist/workflows.js +44 -44
- package/package.json +102 -90
- package/dist/__tests__/assertions.test.d.ts +0 -1
- package/dist/__tests__/assertions.test.js +0 -288
- package/dist/__tests__/client.test.d.ts +0 -1
- package/dist/__tests__/client.test.js +0 -185
- package/dist/__tests__/testing.test.d.ts +0 -1
- package/dist/__tests__/testing.test.js +0 -230
- package/dist/__tests__/workflows.test.d.ts +0 -1
- package/dist/__tests__/workflows.test.js +0 -222
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* openAIChatEval — One-function OpenAI chat regression testing
|
|
4
|
+
*
|
|
5
|
+
* Run local regression tests with OpenAI. No EvalAI account required.
|
|
6
|
+
* CI-friendly output. Optional reportToEvalAI in v1.5.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { openAIChatEval } from '@pauly4010/evalai-sdk';
|
|
11
|
+
*
|
|
12
|
+
* await openAIChatEval({
|
|
13
|
+
* name: 'chat-regression',
|
|
14
|
+
* cases: [
|
|
15
|
+
* { input: 'Hello', expectedOutput: 'greeting' },
|
|
16
|
+
* { input: '2 + 2 = ?', expectedOutput: '4' }
|
|
17
|
+
* ]
|
|
18
|
+
* });
|
|
19
|
+
* ```
|
|
20
|
+
*/
|
|
21
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
+
exports.openAIChatEval = openAIChatEval;
|
|
23
|
+
const assertions_1 = require("../assertions");
|
|
24
|
+
const config_1 = require("../cli/config");
|
|
25
|
+
const testing_1 = require("../testing");
|
|
26
|
+
const input_hash_1 = require("../utils/input-hash");
|
|
27
|
+
const MAX_FAILED_CASES_TO_SHOW = 5;
|
|
28
|
+
function getOpenAI() {
|
|
29
|
+
try {
|
|
30
|
+
const OpenAI = require("openai");
|
|
31
|
+
return OpenAI;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
throw new Error("openai package is required for openAIChatEval. Install with: npm install openai");
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function createExecutor(model, apiKey) {
|
|
38
|
+
const OpenAI = getOpenAI();
|
|
39
|
+
const openai = new OpenAI({ apiKey });
|
|
40
|
+
return async (input) => {
|
|
41
|
+
const response = await openai.chat.completions.create({
|
|
42
|
+
model,
|
|
43
|
+
messages: [{ role: "user", content: input }],
|
|
44
|
+
temperature: 0.1,
|
|
45
|
+
});
|
|
46
|
+
return response.choices[0]?.message?.content ?? "";
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
function printSummary(result) {
|
|
50
|
+
const { passed, total, results } = result;
|
|
51
|
+
const score = total > 0 ? Math.round((passed / total) * 100) : 0;
|
|
52
|
+
const failed = results.filter((r) => !r.passed);
|
|
53
|
+
const status = failed.length === 0 ? "PASS" : "FAIL";
|
|
54
|
+
console.log(`\n${status} ${passed}/${total} (score: ${score})\n`);
|
|
55
|
+
if (failed.length > 0) {
|
|
56
|
+
const toShow = failed.slice(0, MAX_FAILED_CASES_TO_SHOW);
|
|
57
|
+
const more = failed.length - toShow.length;
|
|
58
|
+
console.log(`${failed.length} failing case${failed.length === 1 ? "" : "s"}:`);
|
|
59
|
+
for (const r of toShow) {
|
|
60
|
+
const expected = r.expected ?? "(no expected)";
|
|
61
|
+
console.log(`- "${r.input}" → expected: ${expected}`);
|
|
62
|
+
}
|
|
63
|
+
if (more > 0) {
|
|
64
|
+
console.log(`+ ${more} more`);
|
|
65
|
+
}
|
|
66
|
+
console.log("\nGate this in CI:");
|
|
67
|
+
console.log(" npx -y @pauly4010/evalai-sdk@^1 init");
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
console.log("Tip: Want dashboards and history?");
|
|
71
|
+
console.log("Set EVALAI_API_KEY and connect this to the platform.");
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Run OpenAI chat regression tests locally.
|
|
76
|
+
* No EvalAI account required. Returns score and prints CI-friendly summary.
|
|
77
|
+
*/
|
|
78
|
+
async function openAIChatEval(options) {
|
|
79
|
+
const { name, model = "gpt-4o-mini", apiKey, cases, retries = 0 } = options;
|
|
80
|
+
const resolvedApiKey = apiKey ?? (typeof process !== "undefined" && process.env?.OPENAI_API_KEY);
|
|
81
|
+
if (!resolvedApiKey) {
|
|
82
|
+
throw new Error("OPENAI_API_KEY is required. Set it in the environment or pass apiKey to openAIChatEval.");
|
|
83
|
+
}
|
|
84
|
+
const executor = createExecutor(model, resolvedApiKey);
|
|
85
|
+
const suiteCases = cases.map((c) => {
|
|
86
|
+
const assertions = c.assertions
|
|
87
|
+
? [...c.assertions]
|
|
88
|
+
: c.expectedOutput
|
|
89
|
+
? [
|
|
90
|
+
(output) => (0, assertions_1.expect)(output).toContainKeywords(c.expectedOutput.split(/\s+/).filter(Boolean)),
|
|
91
|
+
]
|
|
92
|
+
: undefined;
|
|
93
|
+
return {
|
|
94
|
+
input: c.input,
|
|
95
|
+
expected: c.expectedOutput,
|
|
96
|
+
assertions,
|
|
97
|
+
};
|
|
98
|
+
});
|
|
99
|
+
const suite = (0, testing_1.createTestSuite)(name, {
|
|
100
|
+
cases: suiteCases,
|
|
101
|
+
executor,
|
|
102
|
+
parallel: true,
|
|
103
|
+
retries,
|
|
104
|
+
});
|
|
105
|
+
const result = await suite.run();
|
|
106
|
+
const score = result.total > 0 ? Math.round((result.passed / result.total) * 100) : 0;
|
|
107
|
+
const evalResult = {
|
|
108
|
+
passed: result.passed,
|
|
109
|
+
total: result.total,
|
|
110
|
+
score,
|
|
111
|
+
results: result.results,
|
|
112
|
+
durationMs: result.durationMs,
|
|
113
|
+
...(result.retriedCases &&
|
|
114
|
+
result.retriedCases.length > 0 && { retriedCases: result.retriedCases }),
|
|
115
|
+
};
|
|
116
|
+
printSummary(evalResult);
|
|
117
|
+
// v1.5: Optional report to EvalAI platform
|
|
118
|
+
if (options.reportToEvalAI) {
|
|
119
|
+
const config = typeof process !== "undefined" && process.cwd ? (0, config_1.loadConfig)(process.cwd()) : null;
|
|
120
|
+
const evalId = options.evaluationId || config?.evaluationId;
|
|
121
|
+
if (!evalId || String(evalId).trim() === "") {
|
|
122
|
+
console.log("Run evalai init and set evaluationId to upload results.");
|
|
123
|
+
return evalResult;
|
|
124
|
+
}
|
|
125
|
+
const evalaiKey = (typeof process !== "undefined" && process.env?.EVALAI_API_KEY) || "";
|
|
126
|
+
if (!evalaiKey) {
|
|
127
|
+
console.log("Set EVALAI_API_KEY to upload results.");
|
|
128
|
+
return evalResult;
|
|
129
|
+
}
|
|
130
|
+
const baseUrl = options.baseUrl ||
|
|
131
|
+
config?.baseUrl ||
|
|
132
|
+
(typeof process !== "undefined" && process.env?.EVALAI_BASE_URL) ||
|
|
133
|
+
"http://localhost:3000";
|
|
134
|
+
const url = String(baseUrl).replace(/\/$/, "");
|
|
135
|
+
try {
|
|
136
|
+
// Resolve testCaseId for each result: explicit testCaseId in cases, or match by inputHash
|
|
137
|
+
const importResults = [];
|
|
138
|
+
const hasExplicitIds = cases.some((c) => c.testCaseId != null);
|
|
139
|
+
if (hasExplicitIds) {
|
|
140
|
+
// Use testCaseId from cases (same order as results)
|
|
141
|
+
for (let i = 0; i < result.results.length; i++) {
|
|
142
|
+
const tcId = cases[i]?.testCaseId;
|
|
143
|
+
if (tcId == null) {
|
|
144
|
+
console.log("reportToEvalAI: All cases must have testCaseId when any has it.");
|
|
145
|
+
return evalResult;
|
|
146
|
+
}
|
|
147
|
+
importResults.push({
|
|
148
|
+
testCaseId: tcId,
|
|
149
|
+
status: result.results[i].passed ? "passed" : "failed",
|
|
150
|
+
output: result.results[i].actual ?? "",
|
|
151
|
+
latencyMs: result.results[i].durationMs,
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
// Match by inputHash (same canonicalization as platform)
|
|
157
|
+
const tcRes = await fetch(`${url}/api/evaluations/${evalId}/test-cases?limit=500`, {
|
|
158
|
+
headers: { Authorization: `Bearer ${evalaiKey}` },
|
|
159
|
+
});
|
|
160
|
+
if (!tcRes.ok) {
|
|
161
|
+
console.log("Could not fetch test cases. Check evaluationId and EVALAI_API_KEY.");
|
|
162
|
+
return evalResult;
|
|
163
|
+
}
|
|
164
|
+
const platformCases = (await tcRes.json());
|
|
165
|
+
const hashToIds = new Map();
|
|
166
|
+
for (const tc of platformCases) {
|
|
167
|
+
const input = tc.input ?? "";
|
|
168
|
+
if (!input.trim())
|
|
169
|
+
continue;
|
|
170
|
+
const hash = (0, input_hash_1.sha256Input)(input);
|
|
171
|
+
const existing = hashToIds.get(hash) ?? [];
|
|
172
|
+
existing.push(tc.id);
|
|
173
|
+
hashToIds.set(hash, existing);
|
|
174
|
+
}
|
|
175
|
+
for (const r of result.results) {
|
|
176
|
+
const hash = (0, input_hash_1.sha256Input)(r.input ?? "");
|
|
177
|
+
const ids = hashToIds.get(hash);
|
|
178
|
+
if (ids == null || ids.length === 0) {
|
|
179
|
+
console.log(`No platform test case matches input: "${(r.input ?? "").slice(0, 50)}…"`);
|
|
180
|
+
return evalResult;
|
|
181
|
+
}
|
|
182
|
+
if (ids.length > 1) {
|
|
183
|
+
console.log(`Multiple platform test cases share the same input (hash collision). Use testCaseId in cases.`);
|
|
184
|
+
return evalResult;
|
|
185
|
+
}
|
|
186
|
+
importResults.push({
|
|
187
|
+
testCaseId: ids[0],
|
|
188
|
+
status: r.passed ? "passed" : "failed",
|
|
189
|
+
output: r.actual ?? "",
|
|
190
|
+
latencyMs: r.durationMs,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
if (importResults.length !== result.results.length) {
|
|
195
|
+
console.log("Could not match all results to platform test cases.");
|
|
196
|
+
return evalResult;
|
|
197
|
+
}
|
|
198
|
+
const sdkVersion = "1.4.1";
|
|
199
|
+
const headers = {
|
|
200
|
+
"Content-Type": "application/json",
|
|
201
|
+
Authorization: `Bearer ${evalaiKey}`,
|
|
202
|
+
};
|
|
203
|
+
if (options.idempotencyKey) {
|
|
204
|
+
headers["Idempotency-Key"] = options.idempotencyKey;
|
|
205
|
+
}
|
|
206
|
+
const importRes = await fetch(`${url}/api/evaluations/${evalId}/runs/import`, {
|
|
207
|
+
method: "POST",
|
|
208
|
+
headers,
|
|
209
|
+
body: JSON.stringify({
|
|
210
|
+
environment: "dev",
|
|
211
|
+
results: importResults,
|
|
212
|
+
importClientVersion: sdkVersion,
|
|
213
|
+
}),
|
|
214
|
+
});
|
|
215
|
+
if (!importRes.ok) {
|
|
216
|
+
const body = await importRes.text();
|
|
217
|
+
console.log(`Upload failed: ${importRes.status} — ${body}`);
|
|
218
|
+
return evalResult;
|
|
219
|
+
}
|
|
220
|
+
const importData = (await importRes.json());
|
|
221
|
+
if (importData.dashboardUrl) {
|
|
222
|
+
console.log(`Dashboard: ${importData.dashboardUrl}`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
catch (err) {
|
|
226
|
+
console.log("Upload failed:", err instanceof Error ? err.message : String(err));
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
return evalResult;
|
|
230
|
+
}
|
|
@@ -41,7 +41,7 @@ const context_1 = require("../context");
|
|
|
41
41
|
* ```
|
|
42
42
|
*/
|
|
43
43
|
function traceOpenAI(openai, evalClient, options = {}) {
|
|
44
|
-
const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix =
|
|
44
|
+
const { captureInput = true, captureOutput = true, captureMetadata = true, organizationId, tracePrefix = "openai", } = options;
|
|
45
45
|
// Create proxy for chat.completions.create
|
|
46
46
|
const originalCreate = openai.chat.completions.create.bind(openai.chat.completions);
|
|
47
47
|
openai.chat.completions.create = async (params, requestOptions) => {
|
|
@@ -58,18 +58,20 @@ function traceOpenAI(openai, evalClient, options = {}) {
|
|
|
58
58
|
max_tokens: params.max_tokens,
|
|
59
59
|
...(captureInput ? { input: params.messages } : {}),
|
|
60
60
|
...(captureOutput ? { output: response.choices[0]?.message } : {}),
|
|
61
|
-
...(captureMetadata
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
...(captureMetadata
|
|
62
|
+
? {
|
|
63
|
+
usage: response.usage,
|
|
64
|
+
finish_reason: response.choices[0]?.finish_reason,
|
|
65
|
+
}
|
|
66
|
+
: {}),
|
|
65
67
|
});
|
|
66
68
|
await evalClient.traces.create({
|
|
67
69
|
name: `OpenAI: ${params.model}`,
|
|
68
70
|
traceId,
|
|
69
71
|
organizationId: organizationId || evalClient.getOrganizationId(),
|
|
70
|
-
status:
|
|
72
|
+
status: "success",
|
|
71
73
|
durationMs,
|
|
72
|
-
metadata: traceMetadata
|
|
74
|
+
metadata: traceMetadata,
|
|
73
75
|
});
|
|
74
76
|
return response;
|
|
75
77
|
}
|
|
@@ -82,16 +84,18 @@ function traceOpenAI(openai, evalClient, options = {}) {
|
|
|
82
84
|
max_tokens: params.max_tokens,
|
|
83
85
|
...(captureInput ? { input: params.messages } : {}),
|
|
84
86
|
...(captureMetadata ? { params } : {}),
|
|
85
|
-
error: error instanceof Error ? error.message : String(error)
|
|
87
|
+
error: error instanceof Error ? error.message : String(error),
|
|
86
88
|
});
|
|
87
|
-
await evalClient.traces
|
|
89
|
+
await evalClient.traces
|
|
90
|
+
.create({
|
|
88
91
|
name: `OpenAI: ${params.model}`,
|
|
89
92
|
traceId,
|
|
90
93
|
organizationId: organizationId || evalClient.getOrganizationId(),
|
|
91
|
-
status:
|
|
94
|
+
status: "error",
|
|
92
95
|
durationMs,
|
|
93
|
-
metadata: errorMetadata
|
|
94
|
-
})
|
|
96
|
+
metadata: errorMetadata,
|
|
97
|
+
})
|
|
98
|
+
.catch(() => {
|
|
95
99
|
// Ignore errors in trace creation to avoid masking the original error
|
|
96
100
|
});
|
|
97
101
|
throw error;
|
|
@@ -124,8 +128,8 @@ async function traceOpenAICall(evalClient, name, fn, options = {}) {
|
|
|
124
128
|
name,
|
|
125
129
|
traceId,
|
|
126
130
|
organizationId: options.organizationId || evalClient.getOrganizationId(),
|
|
127
|
-
status:
|
|
128
|
-
metadata: (0, context_1.mergeWithContext)({})
|
|
131
|
+
status: "pending",
|
|
132
|
+
metadata: (0, context_1.mergeWithContext)({}),
|
|
129
133
|
});
|
|
130
134
|
const result = await fn();
|
|
131
135
|
const durationMs = Date.now() - startTime;
|
|
@@ -133,9 +137,9 @@ async function traceOpenAICall(evalClient, name, fn, options = {}) {
|
|
|
133
137
|
name,
|
|
134
138
|
traceId,
|
|
135
139
|
organizationId: options.organizationId || evalClient.getOrganizationId(),
|
|
136
|
-
status:
|
|
140
|
+
status: "success",
|
|
137
141
|
durationMs,
|
|
138
|
-
metadata: (0, context_1.mergeWithContext)({})
|
|
142
|
+
metadata: (0, context_1.mergeWithContext)({}),
|
|
139
143
|
});
|
|
140
144
|
return result;
|
|
141
145
|
}
|
|
@@ -145,11 +149,11 @@ async function traceOpenAICall(evalClient, name, fn, options = {}) {
|
|
|
145
149
|
name,
|
|
146
150
|
traceId,
|
|
147
151
|
organizationId: options.organizationId || evalClient.getOrganizationId(),
|
|
148
|
-
status:
|
|
152
|
+
status: "error",
|
|
149
153
|
durationMs,
|
|
150
154
|
metadata: (0, context_1.mergeWithContext)({
|
|
151
|
-
error: error instanceof Error ? error.message : String(error)
|
|
152
|
-
})
|
|
155
|
+
error: error instanceof Error ? error.message : String(error),
|
|
156
|
+
}),
|
|
153
157
|
});
|
|
154
158
|
throw error;
|
|
155
159
|
}
|
package/dist/local.d.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*
|
|
5
5
|
* ⚠️ NOTE: This module requires Node.js and will not work in browsers.
|
|
6
6
|
*/
|
|
7
|
-
import type {
|
|
7
|
+
import type { Evaluation, Span, Trace } from "./types";
|
|
8
8
|
export interface LocalStorageOptions {
|
|
9
9
|
directory?: string;
|
|
10
10
|
autoSave?: boolean;
|
|
@@ -30,7 +30,7 @@ export declare class LocalStorage {
|
|
|
30
30
|
saveSpans(traceId: string, spans: Span[]): Promise<void>;
|
|
31
31
|
getSpans(traceId: string): Promise<Span[] | undefined>;
|
|
32
32
|
clear(): Promise<void>;
|
|
33
|
-
export(
|
|
33
|
+
export(_format: "json"): Promise<string>;
|
|
34
34
|
getStats(): {
|
|
35
35
|
traces: number;
|
|
36
36
|
evaluations: number;
|
package/dist/local.js
CHANGED
|
@@ -11,64 +11,64 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.LocalStorage = void 0;
|
|
13
13
|
// Environment check
|
|
14
|
-
const isNode = typeof process !==
|
|
14
|
+
const isNode = typeof process !== "undefined" && process.versions?.node;
|
|
15
15
|
if (!isNode) {
|
|
16
|
-
throw new Error(
|
|
17
|
-
|
|
16
|
+
throw new Error("Local storage mode requires Node.js and cannot run in browsers. " +
|
|
17
|
+
"This feature uses the filesystem for storing data.");
|
|
18
18
|
}
|
|
19
|
-
const promises_1 = __importDefault(require("fs/promises"));
|
|
20
|
-
const
|
|
19
|
+
const promises_1 = __importDefault(require("node:fs/promises"));
|
|
20
|
+
const node_path_1 = __importDefault(require("node:path"));
|
|
21
21
|
class LocalStorage {
|
|
22
22
|
constructor(options = {}) {
|
|
23
23
|
this.traces = new Map();
|
|
24
24
|
this.evaluations = new Map();
|
|
25
25
|
this.spans = new Map();
|
|
26
|
-
this.directory = options.directory ||
|
|
26
|
+
this.directory = options.directory || "./.evalai-data";
|
|
27
27
|
this.autoSave = options.autoSave !== false;
|
|
28
28
|
this.initialize();
|
|
29
29
|
}
|
|
30
30
|
async initialize() {
|
|
31
31
|
try {
|
|
32
32
|
await promises_1.default.mkdir(this.directory, { recursive: true });
|
|
33
|
-
await promises_1.default.mkdir(
|
|
34
|
-
await promises_1.default.mkdir(
|
|
35
|
-
await promises_1.default.mkdir(
|
|
33
|
+
await promises_1.default.mkdir(node_path_1.default.join(this.directory, "traces"), { recursive: true });
|
|
34
|
+
await promises_1.default.mkdir(node_path_1.default.join(this.directory, "evaluations"), { recursive: true });
|
|
35
|
+
await promises_1.default.mkdir(node_path_1.default.join(this.directory, "spans"), { recursive: true });
|
|
36
36
|
// Load existing data
|
|
37
37
|
await this.loadAllData();
|
|
38
38
|
}
|
|
39
39
|
catch (error) {
|
|
40
|
-
console.warn(
|
|
40
|
+
console.warn("Failed to initialize local storage:", error);
|
|
41
41
|
}
|
|
42
42
|
}
|
|
43
43
|
async loadAllData() {
|
|
44
44
|
try {
|
|
45
45
|
// Load traces
|
|
46
|
-
const tracesDir =
|
|
46
|
+
const tracesDir = node_path_1.default.join(this.directory, "traces");
|
|
47
47
|
const traceFiles = await promises_1.default.readdir(tracesDir);
|
|
48
48
|
for (const file of traceFiles) {
|
|
49
|
-
if (file.endsWith(
|
|
50
|
-
const content = await promises_1.default.readFile(
|
|
49
|
+
if (file.endsWith(".json")) {
|
|
50
|
+
const content = await promises_1.default.readFile(node_path_1.default.join(tracesDir, file), "utf-8");
|
|
51
51
|
const trace = JSON.parse(content);
|
|
52
52
|
this.traces.set(trace.id.toString(), trace);
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
// Load evaluations
|
|
56
|
-
const evalsDir =
|
|
56
|
+
const evalsDir = node_path_1.default.join(this.directory, "evaluations");
|
|
57
57
|
const evalFiles = await promises_1.default.readdir(evalsDir);
|
|
58
58
|
for (const file of evalFiles) {
|
|
59
|
-
if (file.endsWith(
|
|
60
|
-
const content = await promises_1.default.readFile(
|
|
59
|
+
if (file.endsWith(".json")) {
|
|
60
|
+
const content = await promises_1.default.readFile(node_path_1.default.join(evalsDir, file), "utf-8");
|
|
61
61
|
const evaluation = JSON.parse(content);
|
|
62
62
|
this.evaluations.set(evaluation.id.toString(), evaluation);
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
|
-
catch (
|
|
66
|
+
catch (_error) {
|
|
67
67
|
// Directories might not exist yet, that's fine
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
async saveTraceToDisk(trace) {
|
|
71
|
-
const filePath =
|
|
71
|
+
const filePath = node_path_1.default.join(this.directory, "traces", `${trace.id}.json`);
|
|
72
72
|
await promises_1.default.writeFile(filePath, JSON.stringify(trace, null, 2));
|
|
73
73
|
}
|
|
74
74
|
async saveTrace(trace) {
|
|
@@ -84,7 +84,7 @@ class LocalStorage {
|
|
|
84
84
|
return Array.from(this.traces.values());
|
|
85
85
|
}
|
|
86
86
|
async saveEvaluationToDisk(evaluation) {
|
|
87
|
-
const filePath =
|
|
87
|
+
const filePath = node_path_1.default.join(this.directory, "evaluations", `${evaluation.id}.json`);
|
|
88
88
|
await promises_1.default.writeFile(filePath, JSON.stringify(evaluation, null, 2));
|
|
89
89
|
}
|
|
90
90
|
async saveEvaluation(evaluation) {
|
|
@@ -100,7 +100,7 @@ class LocalStorage {
|
|
|
100
100
|
return Array.from(this.evaluations.values());
|
|
101
101
|
}
|
|
102
102
|
async saveSpansToDisk(traceId, spans) {
|
|
103
|
-
const filePath =
|
|
103
|
+
const filePath = node_path_1.default.join(this.directory, "spans", `${traceId}.json`);
|
|
104
104
|
await promises_1.default.writeFile(filePath, JSON.stringify(spans, null, 2));
|
|
105
105
|
}
|
|
106
106
|
async saveSpans(traceId, spans) {
|
|
@@ -122,16 +122,16 @@ class LocalStorage {
|
|
|
122
122
|
await this.initialize();
|
|
123
123
|
}
|
|
124
124
|
catch (error) {
|
|
125
|
-
console.warn(
|
|
125
|
+
console.warn("Failed to clear local storage:", error);
|
|
126
126
|
}
|
|
127
127
|
}
|
|
128
|
-
async export(
|
|
128
|
+
async export(_format) {
|
|
129
129
|
const data = {
|
|
130
130
|
traces: Array.from(this.traces.values()),
|
|
131
131
|
evaluations: Array.from(this.evaluations.values()),
|
|
132
|
-
spans: Object.fromEntries(this.spans)
|
|
132
|
+
spans: Object.fromEntries(this.spans),
|
|
133
133
|
};
|
|
134
|
-
const exportPath =
|
|
134
|
+
const exportPath = node_path_1.default.join(this.directory, `export-${Date.now()}.json`);
|
|
135
135
|
await promises_1.default.writeFile(exportPath, JSON.stringify(data, null, 2));
|
|
136
136
|
return exportPath;
|
|
137
137
|
}
|
|
@@ -139,7 +139,7 @@ class LocalStorage {
|
|
|
139
139
|
return {
|
|
140
140
|
traces: this.traces.size,
|
|
141
141
|
evaluations: this.evaluations.size,
|
|
142
|
-
totalSpans: Array.from(this.spans.values()).reduce((sum, spans) => sum + spans.length, 0)
|
|
142
|
+
totalSpans: Array.from(this.spans.values()).reduce((sum, spans) => sum + spans.length, 0),
|
|
143
143
|
};
|
|
144
144
|
}
|
|
145
145
|
}
|
package/dist/logger.d.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* logger.error('Request failed', { error: err });
|
|
13
13
|
* ```
|
|
14
14
|
*/
|
|
15
|
-
export type LogLevel =
|
|
15
|
+
export type LogLevel = "trace" | "debug" | "info" | "warn" | "error";
|
|
16
16
|
export interface LoggerOptions {
|
|
17
17
|
/** Log level (default: 'info') */
|
|
18
18
|
level?: LogLevel;
|
package/dist/logger.js
CHANGED
|
@@ -23,58 +23,58 @@ const LOG_LEVELS = {
|
|
|
23
23
|
debug: 1,
|
|
24
24
|
info: 2,
|
|
25
25
|
warn: 3,
|
|
26
|
-
error: 4
|
|
26
|
+
error: 4,
|
|
27
27
|
};
|
|
28
28
|
const LOG_COLORS = {
|
|
29
|
-
trace:
|
|
30
|
-
debug:
|
|
31
|
-
info:
|
|
32
|
-
warn:
|
|
33
|
-
error:
|
|
29
|
+
trace: "\x1b[90m", // gray
|
|
30
|
+
debug: "\x1b[36m", // cyan
|
|
31
|
+
info: "\x1b[32m", // green
|
|
32
|
+
warn: "\x1b[33m", // yellow
|
|
33
|
+
error: "\x1b[31m", // red
|
|
34
34
|
};
|
|
35
|
-
const COLOR_RESET =
|
|
35
|
+
const COLOR_RESET = "\x1b[0m";
|
|
36
36
|
/**
|
|
37
37
|
* Logger for SDK debugging and troubleshooting
|
|
38
38
|
*/
|
|
39
39
|
class Logger {
|
|
40
40
|
constructor(options = {}) {
|
|
41
41
|
this.options = {
|
|
42
|
-
level: options.level ||
|
|
42
|
+
level: options.level || "info",
|
|
43
43
|
pretty: options.pretty ?? false,
|
|
44
44
|
timestamps: options.timestamps ?? true,
|
|
45
45
|
handler: options.handler || this.defaultHandler.bind(this),
|
|
46
|
-
prefix: options.prefix ||
|
|
46
|
+
prefix: options.prefix || "EvalAI",
|
|
47
47
|
};
|
|
48
48
|
}
|
|
49
49
|
/**
|
|
50
50
|
* Log a trace message
|
|
51
51
|
*/
|
|
52
52
|
trace(message, data) {
|
|
53
|
-
this.log(
|
|
53
|
+
this.log("trace", message, data);
|
|
54
54
|
}
|
|
55
55
|
/**
|
|
56
56
|
* Log a debug message
|
|
57
57
|
*/
|
|
58
58
|
debug(message, data) {
|
|
59
|
-
this.log(
|
|
59
|
+
this.log("debug", message, data);
|
|
60
60
|
}
|
|
61
61
|
/**
|
|
62
62
|
* Log an info message
|
|
63
63
|
*/
|
|
64
64
|
info(message, data) {
|
|
65
|
-
this.log(
|
|
65
|
+
this.log("info", message, data);
|
|
66
66
|
}
|
|
67
67
|
/**
|
|
68
68
|
* Log a warning message
|
|
69
69
|
*/
|
|
70
70
|
warn(message, data) {
|
|
71
|
-
this.log(
|
|
71
|
+
this.log("warn", message, data);
|
|
72
72
|
}
|
|
73
73
|
/**
|
|
74
74
|
* Log an error message
|
|
75
75
|
*/
|
|
76
76
|
error(message, data) {
|
|
77
|
-
this.log(
|
|
77
|
+
this.log("error", message, data);
|
|
78
78
|
}
|
|
79
79
|
/**
|
|
80
80
|
* Log HTTP request
|
|
@@ -86,7 +86,7 @@ class Logger {
|
|
|
86
86
|
* Log HTTP response
|
|
87
87
|
*/
|
|
88
88
|
logResponse(method, url, status, duration, data) {
|
|
89
|
-
const level = status >= 400 ?
|
|
89
|
+
const level = status >= 400 ? "error" : status >= 300 ? "warn" : "debug";
|
|
90
90
|
this.log(level, `← ${method} ${url} ${status} (${duration}ms)`, data);
|
|
91
91
|
}
|
|
92
92
|
/**
|
|
@@ -95,7 +95,7 @@ class Logger {
|
|
|
95
95
|
child(prefix) {
|
|
96
96
|
return new Logger({
|
|
97
97
|
...this.options,
|
|
98
|
-
prefix: `${this.options.prefix}:${prefix}
|
|
98
|
+
prefix: `${this.options.prefix}:${prefix}`,
|
|
99
99
|
});
|
|
100
100
|
}
|
|
101
101
|
/**
|
|
@@ -118,7 +118,7 @@ class Logger {
|
|
|
118
118
|
message,
|
|
119
119
|
timestamp: new Date().toISOString(),
|
|
120
120
|
data,
|
|
121
|
-
prefix: this.options.prefix
|
|
121
|
+
prefix: this.options.prefix,
|
|
122
122
|
};
|
|
123
123
|
this.options.handler(entry);
|
|
124
124
|
}
|
|
@@ -133,23 +133,19 @@ class Logger {
|
|
|
133
133
|
}
|
|
134
134
|
// Level
|
|
135
135
|
const levelStr = entry.level.toUpperCase().padEnd(5);
|
|
136
|
-
parts.push(this.options.pretty
|
|
137
|
-
? `${LOG_COLORS[entry.level]}${levelStr}${COLOR_RESET}`
|
|
138
|
-
: levelStr);
|
|
136
|
+
parts.push(this.options.pretty ? `${LOG_COLORS[entry.level]}${levelStr}${COLOR_RESET}` : levelStr);
|
|
139
137
|
// Prefix
|
|
140
138
|
if (entry.prefix) {
|
|
141
|
-
parts.push(this.options.pretty
|
|
142
|
-
? `\x1b[35m[${entry.prefix}]${COLOR_RESET}`
|
|
143
|
-
: `[${entry.prefix}]`);
|
|
139
|
+
parts.push(this.options.pretty ? `\x1b[35m[${entry.prefix}]${COLOR_RESET}` : `[${entry.prefix}]`);
|
|
144
140
|
}
|
|
145
141
|
// Message
|
|
146
142
|
parts.push(entry.message);
|
|
147
143
|
// Log
|
|
148
|
-
const logLine = parts.join(
|
|
149
|
-
if (entry.level ===
|
|
144
|
+
const logLine = parts.join(" ");
|
|
145
|
+
if (entry.level === "error") {
|
|
150
146
|
console.error(logLine);
|
|
151
147
|
}
|
|
152
|
-
else if (entry.level ===
|
|
148
|
+
else if (entry.level === "warn") {
|
|
153
149
|
console.warn(logLine);
|
|
154
150
|
}
|
|
155
151
|
else {
|
|
@@ -211,7 +207,7 @@ class RequestLogger {
|
|
|
211
207
|
logRequest(request) {
|
|
212
208
|
this.logger.logRequest(request.method, request.url, {
|
|
213
209
|
headers: request.headers,
|
|
214
|
-
body: request.body
|
|
210
|
+
body: request.body,
|
|
215
211
|
});
|
|
216
212
|
}
|
|
217
213
|
/**
|
|
@@ -220,7 +216,7 @@ class RequestLogger {
|
|
|
220
216
|
logResponse(response) {
|
|
221
217
|
this.logger.logResponse(response.method, response.url, response.status, response.duration, {
|
|
222
218
|
headers: response.headers,
|
|
223
|
-
body: response.body
|
|
219
|
+
body: response.body,
|
|
224
220
|
});
|
|
225
221
|
}
|
|
226
222
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { extendExpectWithToPassGate, toPassGate } from "./to-pass-gate";
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.toPassGate = exports.extendExpectWithToPassGate = void 0;
|
|
4
|
+
var to_pass_gate_1 = require("./to-pass-gate");
|
|
5
|
+
Object.defineProperty(exports, "extendExpectWithToPassGate", { enumerable: true, get: function () { return to_pass_gate_1.extendExpectWithToPassGate; } });
|
|
6
|
+
Object.defineProperty(exports, "toPassGate", { enumerable: true, get: function () { return to_pass_gate_1.toPassGate; } });
|