@pauly4010/evalai-sdk 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +289 -0
- package/LICENSE +21 -0
- package/README.md +565 -0
- package/dist/assertions.d.ts +189 -0
- package/dist/assertions.js +596 -0
- package/dist/batch.d.ts +68 -0
- package/dist/batch.js +178 -0
- package/dist/cache.d.ts +65 -0
- package/dist/cache.js +135 -0
- package/dist/cli/index.d.ts +6 -0
- package/dist/cli/index.js +181 -0
- package/dist/client.d.ts +358 -0
- package/dist/client.js +802 -0
- package/dist/context.d.ts +134 -0
- package/dist/context.js +215 -0
- package/dist/errors.d.ts +80 -0
- package/dist/errors.js +285 -0
- package/dist/export.d.ts +195 -0
- package/dist/export.js +334 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +111 -0
- package/dist/integrations/anthropic.d.ts +72 -0
- package/dist/integrations/anthropic.js +159 -0
- package/dist/integrations/openai.d.ts +69 -0
- package/dist/integrations/openai.js +156 -0
- package/dist/local.d.ts +39 -0
- package/dist/local.js +146 -0
- package/dist/logger.d.ts +128 -0
- package/dist/logger.js +227 -0
- package/dist/pagination.d.ts +74 -0
- package/dist/pagination.js +135 -0
- package/dist/snapshot.d.ts +176 -0
- package/dist/snapshot.js +322 -0
- package/dist/streaming.d.ts +173 -0
- package/dist/streaming.js +268 -0
- package/dist/testing.d.ts +204 -0
- package/dist/testing.js +252 -0
- package/dist/types.d.ts +715 -0
- package/dist/types.js +54 -0
- package/dist/workflows.d.ts +378 -0
- package/dist/workflows.js +628 -0
- package/package.json +102 -0
package/dist/export.d.ts
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Data Export/Import System
|
|
3
|
+
* Tier 4.18: Platform migration and backup utilities
|
|
4
|
+
*
|
|
5
|
+
* @example
|
|
6
|
+
* ```typescript
|
|
7
|
+
* import { exportData, importData } from '@ai-eval-platform/sdk';
|
|
8
|
+
*
|
|
9
|
+
* // Export all data
|
|
10
|
+
* const data = await exportData(client, {
|
|
11
|
+
* format: 'json',
|
|
12
|
+
* includeTraces: true,
|
|
13
|
+
* includeEvaluations: true
|
|
14
|
+
* });
|
|
15
|
+
*
|
|
16
|
+
* // Save to file
|
|
17
|
+
* fs.writeFileSync('backup.json', JSON.stringify(data, null, 2));
|
|
18
|
+
*
|
|
19
|
+
* // Import from another platform
|
|
20
|
+
* await importFromLangSmith(client, langsmithData);
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
import type { AIEvalClient } from './client';
|
|
24
|
+
import type { Trace, Evaluation, TestCase, EvaluationRun } from './types';
|
|
25
|
+
export type ExportFormat = 'json' | 'csv' | 'jsonl';
|
|
26
|
+
export type { ExportFormat as ExportType };
|
|
27
|
+
export interface ExportOptions {
|
|
28
|
+
/** Export format */
|
|
29
|
+
format: 'json' | 'csv' | 'jsonl';
|
|
30
|
+
/** Include traces */
|
|
31
|
+
includeTraces?: boolean;
|
|
32
|
+
/** Include evaluations */
|
|
33
|
+
includeEvaluations?: boolean;
|
|
34
|
+
/** Include test cases */
|
|
35
|
+
includeTestCases?: boolean;
|
|
36
|
+
/** Include evaluation runs */
|
|
37
|
+
includeRuns?: boolean;
|
|
38
|
+
/** Date range filter */
|
|
39
|
+
dateRange?: {
|
|
40
|
+
from: string;
|
|
41
|
+
to: string;
|
|
42
|
+
};
|
|
43
|
+
/** Organization ID filter */
|
|
44
|
+
organizationId?: number;
|
|
45
|
+
/** Maximum items to export (default: no limit) */
|
|
46
|
+
limit?: number;
|
|
47
|
+
}
|
|
48
|
+
export interface ExportData {
|
|
49
|
+
/** Export metadata */
|
|
50
|
+
metadata: {
|
|
51
|
+
exportedAt: string;
|
|
52
|
+
version: string;
|
|
53
|
+
format: string;
|
|
54
|
+
organizationId?: number;
|
|
55
|
+
};
|
|
56
|
+
/** Exported traces */
|
|
57
|
+
traces?: Trace[];
|
|
58
|
+
/** Exported evaluations */
|
|
59
|
+
evaluations?: Evaluation[];
|
|
60
|
+
/** Exported test cases */
|
|
61
|
+
testCases?: TestCase[];
|
|
62
|
+
/** Exported runs */
|
|
63
|
+
runs?: EvaluationRun[];
|
|
64
|
+
}
|
|
65
|
+
export interface ImportOptions {
|
|
66
|
+
/** Organization ID to import into */
|
|
67
|
+
organizationId?: number;
|
|
68
|
+
/** User ID for created resources */
|
|
69
|
+
createdBy?: number;
|
|
70
|
+
/** Skip duplicates (based on name) */
|
|
71
|
+
skipDuplicates?: boolean;
|
|
72
|
+
/** Dry run (don't actually import) */
|
|
73
|
+
dryRun?: boolean;
|
|
74
|
+
}
|
|
75
|
+
export interface ImportResult {
|
|
76
|
+
/** Import summary */
|
|
77
|
+
summary: {
|
|
78
|
+
total: number;
|
|
79
|
+
imported: number;
|
|
80
|
+
skipped: number;
|
|
81
|
+
failed: number;
|
|
82
|
+
};
|
|
83
|
+
/** Detailed results */
|
|
84
|
+
details: {
|
|
85
|
+
traces?: {
|
|
86
|
+
imported: number;
|
|
87
|
+
skipped: number;
|
|
88
|
+
failed: number;
|
|
89
|
+
};
|
|
90
|
+
evaluations?: {
|
|
91
|
+
imported: number;
|
|
92
|
+
skipped: number;
|
|
93
|
+
failed: number;
|
|
94
|
+
};
|
|
95
|
+
testCases?: {
|
|
96
|
+
imported: number;
|
|
97
|
+
skipped: number;
|
|
98
|
+
failed: number;
|
|
99
|
+
};
|
|
100
|
+
runs?: {
|
|
101
|
+
imported: number;
|
|
102
|
+
skipped: number;
|
|
103
|
+
failed: number;
|
|
104
|
+
};
|
|
105
|
+
};
|
|
106
|
+
/** Errors encountered */
|
|
107
|
+
errors?: Array<{
|
|
108
|
+
item: string;
|
|
109
|
+
error: string;
|
|
110
|
+
}>;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Export data from the platform
|
|
114
|
+
*
|
|
115
|
+
* @example
|
|
116
|
+
* ```typescript
|
|
117
|
+
* const data = await exportData(client, {
|
|
118
|
+
* format: 'json',
|
|
119
|
+
* includeTraces: true,
|
|
120
|
+
* includeEvaluations: true,
|
|
121
|
+
* dateRange: { from: '2024-01-01', to: '2024-12-31' }
|
|
122
|
+
* });
|
|
123
|
+
*
|
|
124
|
+
* // Save to file
|
|
125
|
+
* fs.writeFileSync('backup.json', JSON.stringify(data, null, 2));
|
|
126
|
+
* ```
|
|
127
|
+
*/
|
|
128
|
+
export declare function exportData(client: AIEvalClient, options: ExportOptions): Promise<ExportData>;
|
|
129
|
+
/**
|
|
130
|
+
* Import data into the platform
|
|
131
|
+
*
|
|
132
|
+
* @example
|
|
133
|
+
* ```typescript
|
|
134
|
+
* const data = JSON.parse(fs.readFileSync('backup.json', 'utf-8'));
|
|
135
|
+
* const result = await importData(client, data, {
|
|
136
|
+
* organizationId: 123,
|
|
137
|
+
* skipDuplicates: true
|
|
138
|
+
* });
|
|
139
|
+
*
|
|
140
|
+
* console.log(`Imported ${result.summary.imported} items`);
|
|
141
|
+
* ```
|
|
142
|
+
*/
|
|
143
|
+
export declare function importData(client: AIEvalClient, data: ExportData, options: ImportOptions): Promise<ImportResult>;
|
|
144
|
+
/**
|
|
145
|
+
* Export data to JSON file
|
|
146
|
+
*
|
|
147
|
+
* @example
|
|
148
|
+
* ```typescript
|
|
149
|
+
* await exportToFile(client, './backup.json', {
|
|
150
|
+
* includeTraces: true,
|
|
151
|
+
* includeEvaluations: true
|
|
152
|
+
* });
|
|
153
|
+
* ```
|
|
154
|
+
*/
|
|
155
|
+
export declare function exportToFile(client: AIEvalClient, filePath: string, options: Omit<ExportOptions, 'format'>): Promise<void>;
|
|
156
|
+
/**
|
|
157
|
+
* Import data from JSON file
|
|
158
|
+
*
|
|
159
|
+
* @example
|
|
160
|
+
* ```typescript
|
|
161
|
+
* const result = await importFromFile(client, './backup.json', {
|
|
162
|
+
* organizationId: 123,
|
|
163
|
+
* createdBy: 1
|
|
164
|
+
* });
|
|
165
|
+
* ```
|
|
166
|
+
*/
|
|
167
|
+
export declare function importFromFile(client: AIEvalClient, filePath: string, options: ImportOptions): Promise<ImportResult>;
|
|
168
|
+
/**
|
|
169
|
+
* Import from LangSmith format
|
|
170
|
+
*
|
|
171
|
+
* @example
|
|
172
|
+
* ```typescript
|
|
173
|
+
* const langsmithData = {
|
|
174
|
+
* runs: [
|
|
175
|
+
* { name: 'test-1', inputs: { ... }, outputs: { ... } }
|
|
176
|
+
* ]
|
|
177
|
+
* };
|
|
178
|
+
*
|
|
179
|
+
* await importFromLangSmith(client, langsmithData, {
|
|
180
|
+
* organizationId: 123
|
|
181
|
+
* });
|
|
182
|
+
* ```
|
|
183
|
+
*/
|
|
184
|
+
export declare function importFromLangSmith(client: AIEvalClient, langsmithData: any, options: ImportOptions): Promise<ImportResult>;
|
|
185
|
+
/**
|
|
186
|
+
* Convert export data to CSV format
|
|
187
|
+
*
|
|
188
|
+
* @example
|
|
189
|
+
* ```typescript
|
|
190
|
+
* const data = await exportData(client, { format: 'json', includeTraces: true });
|
|
191
|
+
* const csv = convertToCSV(data, 'traces');
|
|
192
|
+
* fs.writeFileSync('traces.csv', csv);
|
|
193
|
+
* ```
|
|
194
|
+
*/
|
|
195
|
+
export declare function convertToCSV(data: ExportData, type: 'traces' | 'evaluations'): string;
|
package/dist/export.js
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Data Export/Import System
|
|
4
|
+
* Tier 4.18: Platform migration and backup utilities
|
|
5
|
+
*
|
|
6
|
+
* @example
|
|
7
|
+
* ```typescript
|
|
8
|
+
* import { exportData, importData } from '@ai-eval-platform/sdk';
|
|
9
|
+
*
|
|
10
|
+
* // Export all data
|
|
11
|
+
* const data = await exportData(client, {
|
|
12
|
+
* format: 'json',
|
|
13
|
+
* includeTraces: true,
|
|
14
|
+
* includeEvaluations: true
|
|
15
|
+
* });
|
|
16
|
+
*
|
|
17
|
+
* // Save to file
|
|
18
|
+
* fs.writeFileSync('backup.json', JSON.stringify(data, null, 2));
|
|
19
|
+
*
|
|
20
|
+
* // Import from another platform
|
|
21
|
+
* await importFromLangSmith(client, langsmithData);
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
25
|
+
if (k2 === undefined) k2 = k;
|
|
26
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
27
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
28
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
29
|
+
}
|
|
30
|
+
Object.defineProperty(o, k2, desc);
|
|
31
|
+
}) : (function(o, m, k, k2) {
|
|
32
|
+
if (k2 === undefined) k2 = k;
|
|
33
|
+
o[k2] = m[k];
|
|
34
|
+
}));
|
|
35
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
36
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
37
|
+
}) : function(o, v) {
|
|
38
|
+
o["default"] = v;
|
|
39
|
+
});
|
|
40
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
41
|
+
var ownKeys = function(o) {
|
|
42
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
43
|
+
var ar = [];
|
|
44
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
45
|
+
return ar;
|
|
46
|
+
};
|
|
47
|
+
return ownKeys(o);
|
|
48
|
+
};
|
|
49
|
+
return function (mod) {
|
|
50
|
+
if (mod && mod.__esModule) return mod;
|
|
51
|
+
var result = {};
|
|
52
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
53
|
+
__setModuleDefault(result, mod);
|
|
54
|
+
return result;
|
|
55
|
+
};
|
|
56
|
+
})();
|
|
57
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
58
|
+
exports.exportData = exportData;
|
|
59
|
+
exports.importData = importData;
|
|
60
|
+
exports.exportToFile = exportToFile;
|
|
61
|
+
exports.importFromFile = importFromFile;
|
|
62
|
+
exports.importFromLangSmith = importFromLangSmith;
|
|
63
|
+
exports.convertToCSV = convertToCSV;
|
|
64
|
+
/**
|
|
65
|
+
* Export data from the platform
|
|
66
|
+
*
|
|
67
|
+
* @example
|
|
68
|
+
* ```typescript
|
|
69
|
+
* const data = await exportData(client, {
|
|
70
|
+
* format: 'json',
|
|
71
|
+
* includeTraces: true,
|
|
72
|
+
* includeEvaluations: true,
|
|
73
|
+
* dateRange: { from: '2024-01-01', to: '2024-12-31' }
|
|
74
|
+
* });
|
|
75
|
+
*
|
|
76
|
+
* // Save to file
|
|
77
|
+
* fs.writeFileSync('backup.json', JSON.stringify(data, null, 2));
|
|
78
|
+
* ```
|
|
79
|
+
*/
|
|
80
|
+
async function exportData(client, options) {
|
|
81
|
+
const exportData = {
|
|
82
|
+
metadata: {
|
|
83
|
+
exportedAt: new Date().toISOString(),
|
|
84
|
+
version: '1.0.0',
|
|
85
|
+
format: options.format,
|
|
86
|
+
organizationId: options.organizationId
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
// Export traces
|
|
90
|
+
if (options.includeTraces) {
|
|
91
|
+
const traces = await client.traces.list({
|
|
92
|
+
organizationId: options.organizationId,
|
|
93
|
+
limit: options.limit
|
|
94
|
+
});
|
|
95
|
+
exportData.traces = traces;
|
|
96
|
+
}
|
|
97
|
+
// Export evaluations
|
|
98
|
+
if (options.includeEvaluations) {
|
|
99
|
+
const evaluations = await client.evaluations.list({
|
|
100
|
+
organizationId: options.organizationId,
|
|
101
|
+
limit: options.limit
|
|
102
|
+
});
|
|
103
|
+
exportData.evaluations = evaluations;
|
|
104
|
+
// Export test cases for each evaluation
|
|
105
|
+
if (options.includeTestCases) {
|
|
106
|
+
const allTestCases = [];
|
|
107
|
+
for (const evaluation of evaluations) {
|
|
108
|
+
const testCases = await client.evaluations.listTestCases(evaluation.id);
|
|
109
|
+
allTestCases.push(...testCases);
|
|
110
|
+
}
|
|
111
|
+
exportData.testCases = allTestCases;
|
|
112
|
+
}
|
|
113
|
+
// Export runs for each evaluation
|
|
114
|
+
if (options.includeRuns) {
|
|
115
|
+
const allRuns = [];
|
|
116
|
+
for (const evaluation of evaluations) {
|
|
117
|
+
const runs = await client.evaluations.listRuns(evaluation.id);
|
|
118
|
+
allRuns.push(...runs);
|
|
119
|
+
}
|
|
120
|
+
exportData.runs = allRuns;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return exportData;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Import data into the platform
|
|
127
|
+
*
|
|
128
|
+
* @example
|
|
129
|
+
* ```typescript
|
|
130
|
+
* const data = JSON.parse(fs.readFileSync('backup.json', 'utf-8'));
|
|
131
|
+
* const result = await importData(client, data, {
|
|
132
|
+
* organizationId: 123,
|
|
133
|
+
* skipDuplicates: true
|
|
134
|
+
* });
|
|
135
|
+
*
|
|
136
|
+
* console.log(`Imported ${result.summary.imported} items`);
|
|
137
|
+
* ```
|
|
138
|
+
*/
|
|
139
|
+
async function importData(client, data, options) {
|
|
140
|
+
const result = {
|
|
141
|
+
summary: { total: 0, imported: 0, skipped: 0, failed: 0 },
|
|
142
|
+
details: {},
|
|
143
|
+
errors: []
|
|
144
|
+
};
|
|
145
|
+
if (options.dryRun) {
|
|
146
|
+
// Count what would be imported
|
|
147
|
+
if (data.traces)
|
|
148
|
+
result.summary.total += data.traces.length;
|
|
149
|
+
if (data.evaluations)
|
|
150
|
+
result.summary.total += data.evaluations.length;
|
|
151
|
+
if (data.testCases)
|
|
152
|
+
result.summary.total += data.testCases.length;
|
|
153
|
+
if (data.runs)
|
|
154
|
+
result.summary.total += data.runs.length;
|
|
155
|
+
return result;
|
|
156
|
+
}
|
|
157
|
+
// Import traces
|
|
158
|
+
if (data.traces) {
|
|
159
|
+
const traceResults = { imported: 0, skipped: 0, failed: 0 };
|
|
160
|
+
for (const trace of data.traces) {
|
|
161
|
+
try {
|
|
162
|
+
await client.traces.create({
|
|
163
|
+
name: trace.name,
|
|
164
|
+
traceId: trace.traceId,
|
|
165
|
+
organizationId: options.organizationId || trace.organizationId,
|
|
166
|
+
status: trace.status,
|
|
167
|
+
durationMs: trace.durationMs || undefined,
|
|
168
|
+
metadata: trace.metadata || undefined
|
|
169
|
+
});
|
|
170
|
+
traceResults.imported++;
|
|
171
|
+
result.summary.imported++;
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
if (options.skipDuplicates && error instanceof Error && error.message.includes('already exists')) {
|
|
175
|
+
traceResults.skipped++;
|
|
176
|
+
result.summary.skipped++;
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
traceResults.failed++;
|
|
180
|
+
result.summary.failed++;
|
|
181
|
+
result.errors?.push({
|
|
182
|
+
item: `trace:${trace.traceId}`,
|
|
183
|
+
error: error instanceof Error ? error.message : String(error)
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
result.details.traces = traceResults;
|
|
189
|
+
result.summary.total += data.traces.length;
|
|
190
|
+
}
|
|
191
|
+
// Import evaluations
|
|
192
|
+
if (data.evaluations) {
|
|
193
|
+
const evalResults = { imported: 0, skipped: 0, failed: 0 };
|
|
194
|
+
for (const evaluation of data.evaluations) {
|
|
195
|
+
try {
|
|
196
|
+
if (!options.createdBy) {
|
|
197
|
+
throw new Error('createdBy is required for importing evaluations');
|
|
198
|
+
}
|
|
199
|
+
await client.evaluations.create({
|
|
200
|
+
name: evaluation.name,
|
|
201
|
+
description: evaluation.description || undefined,
|
|
202
|
+
type: evaluation.type,
|
|
203
|
+
organizationId: options.organizationId || evaluation.organizationId,
|
|
204
|
+
createdBy: options.createdBy,
|
|
205
|
+
status: evaluation.status
|
|
206
|
+
});
|
|
207
|
+
evalResults.imported++;
|
|
208
|
+
result.summary.imported++;
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
if (options.skipDuplicates && error instanceof Error && error.message.includes('already exists')) {
|
|
212
|
+
evalResults.skipped++;
|
|
213
|
+
result.summary.skipped++;
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
evalResults.failed++;
|
|
217
|
+
result.summary.failed++;
|
|
218
|
+
result.errors?.push({
|
|
219
|
+
item: `evaluation:${evaluation.name}`,
|
|
220
|
+
error: error instanceof Error ? error.message : String(error)
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
result.details.evaluations = evalResults;
|
|
226
|
+
result.summary.total += data.evaluations.length;
|
|
227
|
+
}
|
|
228
|
+
return result;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Export data to JSON file
|
|
232
|
+
*
|
|
233
|
+
* @example
|
|
234
|
+
* ```typescript
|
|
235
|
+
* await exportToFile(client, './backup.json', {
|
|
236
|
+
* includeTraces: true,
|
|
237
|
+
* includeEvaluations: true
|
|
238
|
+
* });
|
|
239
|
+
* ```
|
|
240
|
+
*/
|
|
241
|
+
async function exportToFile(client, filePath, options) {
|
|
242
|
+
const data = await exportData(client, { ...options, format: 'json' });
|
|
243
|
+
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
|
|
244
|
+
fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Import data from JSON file
|
|
248
|
+
*
|
|
249
|
+
* @example
|
|
250
|
+
* ```typescript
|
|
251
|
+
* const result = await importFromFile(client, './backup.json', {
|
|
252
|
+
* organizationId: 123,
|
|
253
|
+
* createdBy: 1
|
|
254
|
+
* });
|
|
255
|
+
* ```
|
|
256
|
+
*/
|
|
257
|
+
async function importFromFile(client, filePath, options) {
|
|
258
|
+
const fs = await Promise.resolve().then(() => __importStar(require('fs')));
|
|
259
|
+
const content = fs.readFileSync(filePath, 'utf-8');
|
|
260
|
+
const data = JSON.parse(content);
|
|
261
|
+
return importData(client, data, options);
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Import from LangSmith format
|
|
265
|
+
*
|
|
266
|
+
* @example
|
|
267
|
+
* ```typescript
|
|
268
|
+
* const langsmithData = {
|
|
269
|
+
* runs: [
|
|
270
|
+
* { name: 'test-1', inputs: { ... }, outputs: { ... } }
|
|
271
|
+
* ]
|
|
272
|
+
* };
|
|
273
|
+
*
|
|
274
|
+
* await importFromLangSmith(client, langsmithData, {
|
|
275
|
+
* organizationId: 123
|
|
276
|
+
* });
|
|
277
|
+
* ```
|
|
278
|
+
*/
|
|
279
|
+
async function importFromLangSmith(client, langsmithData, options) {
|
|
280
|
+
// Transform LangSmith format to our format
|
|
281
|
+
const transformedData = {
|
|
282
|
+
metadata: {
|
|
283
|
+
exportedAt: new Date().toISOString(),
|
|
284
|
+
version: '1.0.0',
|
|
285
|
+
format: 'json',
|
|
286
|
+
organizationId: options.organizationId
|
|
287
|
+
},
|
|
288
|
+
traces: []
|
|
289
|
+
};
|
|
290
|
+
// Transform runs to traces
|
|
291
|
+
if (langsmithData.runs && Array.isArray(langsmithData.runs)) {
|
|
292
|
+
transformedData.traces = langsmithData.runs.map((run) => ({
|
|
293
|
+
name: run.name || 'Imported Trace',
|
|
294
|
+
traceId: run.id || `langsmith-${Date.now()}-${Math.random()}`,
|
|
295
|
+
organizationId: options.organizationId,
|
|
296
|
+
status: run.error ? 'error' : 'success',
|
|
297
|
+
durationMs: run.execution_time ? Math.round(run.execution_time * 1000) : null,
|
|
298
|
+
metadata: {
|
|
299
|
+
source: 'langsmith',
|
|
300
|
+
original_id: run.id,
|
|
301
|
+
inputs: run.inputs,
|
|
302
|
+
outputs: run.outputs
|
|
303
|
+
},
|
|
304
|
+
createdAt: run.start_time || new Date().toISOString()
|
|
305
|
+
}));
|
|
306
|
+
}
|
|
307
|
+
return importData(client, transformedData, options);
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Convert export data to CSV format
|
|
311
|
+
*
|
|
312
|
+
* @example
|
|
313
|
+
* ```typescript
|
|
314
|
+
* const data = await exportData(client, { format: 'json', includeTraces: true });
|
|
315
|
+
* const csv = convertToCSV(data, 'traces');
|
|
316
|
+
* fs.writeFileSync('traces.csv', csv);
|
|
317
|
+
* ```
|
|
318
|
+
*/
|
|
319
|
+
function convertToCSV(data, type) {
|
|
320
|
+
const items = type === 'traces' ? data.traces : data.evaluations;
|
|
321
|
+
if (!items || items.length === 0)
|
|
322
|
+
return '';
|
|
323
|
+
// Get headers from first item
|
|
324
|
+
const headers = Object.keys(items[0]);
|
|
325
|
+
const rows = items.map(item => headers.map(h => {
|
|
326
|
+
const value = item[h];
|
|
327
|
+
if (value === null || value === undefined)
|
|
328
|
+
return '';
|
|
329
|
+
if (typeof value === 'object')
|
|
330
|
+
return JSON.stringify(value);
|
|
331
|
+
return String(value);
|
|
332
|
+
}).join(','));
|
|
333
|
+
return [headers.join(','), ...rows].join('\n');
|
|
334
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AI Evaluation Platform SDK
|
|
3
|
+
*
|
|
4
|
+
* Official TypeScript/JavaScript SDK for the AI Evaluation Platform.
|
|
5
|
+
* Build confidence in your AI systems with comprehensive evaluation tools.
|
|
6
|
+
*
|
|
7
|
+
* @packageDocumentation
|
|
8
|
+
*/
|
|
9
|
+
export { AIEvalClient } from './client';
|
|
10
|
+
import { EvalAIError, RateLimitError, AuthenticationError, NetworkError, SDKError } from './errors';
|
|
11
|
+
export { EvalAIError, RateLimitError, AuthenticationError, SDKError as ValidationError, // Using SDKError as ValidationError for backward compatibility
|
|
12
|
+
NetworkError };
|
|
13
|
+
export { expect, containsKeywords, matchesPattern, hasLength, containsJSON, notContainsPII, hasSentiment, similarTo, withinRange, isValidEmail, isValidURL, hasNoHallucinations, matchesSchema, hasReadabilityScore, containsLanguage, hasFactualAccuracy, respondedWithinTime, hasNoToxicity, followsInstructions, containsAllRequiredFields, hasValidCodeSyntax } from './assertions';
|
|
14
|
+
import { createContext, getCurrentContext, withContext, EvalContext } from './context';
|
|
15
|
+
export { createContext, getCurrentContext as getContext, withContext, EvalContext as ContextManager };
|
|
16
|
+
export { createTestSuite, TestSuite, TestSuiteCase, TestSuiteCaseResult, TestSuiteResult, TestSuiteConfig, type TestCaseResult } from './testing';
|
|
17
|
+
import { snapshot, compareWithSnapshot } from './snapshot';
|
|
18
|
+
export { snapshot, compareWithSnapshot, snapshot as saveSnapshot, compareWithSnapshot as compareSnapshots };
|
|
19
|
+
import { exportData, importData } from './export';
|
|
20
|
+
import type { ExportFormat } from './export';
|
|
21
|
+
export { exportData, importData };
|
|
22
|
+
export type { ExportFormat, ExportFormat as ExportType };
|
|
23
|
+
export { batchProcess, streamEvaluation, batchRead, RateLimiter } from './streaming';
|
|
24
|
+
export { RequestCache, CacheTTL } from './cache';
|
|
25
|
+
export { PaginatedIterator, createPaginatedIterator, autoPaginate, encodeCursor, decodeCursor, type PaginatedResponse, type PaginationParams } from './pagination';
|
|
26
|
+
export { RequestBatcher } from './batch';
|
|
27
|
+
export { Logger } from './logger';
|
|
28
|
+
export { traceOpenAI } from './integrations/openai';
|
|
29
|
+
export { traceAnthropic } from './integrations/anthropic';
|
|
30
|
+
export { WorkflowTracer, createWorkflowTracer, traceWorkflowStep, traceLangChainAgent, traceCrewAI, traceAutoGen, type WorkflowNode, type WorkflowEdge, type WorkflowDefinition, type WorkflowContext, type WorkflowStatus, type HandoffType, type AgentHandoff, type DecisionAlternative, type DecisionType, type RecordDecisionParams, type LLMProvider, type CostCategory, type RecordCostParams, type CostRecord, type WorkflowTracerOptions, type AgentSpanContext, } from './workflows';
|
|
31
|
+
export type { ClientConfig as AIEvalConfig, Trace as TraceData, Span as SpanData, Evaluation as EvaluationData, LLMJudgeResult as LLMJudgeData, RetryConfig, GenericMetadata as AnnotationData, TracedResponse, TestCase, TestResult, SnapshotData, ExportOptions, ImportOptions, StreamOptions, BatchOptions } from './types';
|
|
32
|
+
export { EvaluationTemplates, type EvaluationTemplateType, type FeatureUsage, type OrganizationLimits } from './types';
|
|
33
|
+
export type { Annotation, CreateAnnotationParams, ListAnnotationsParams, AnnotationTask, CreateAnnotationTaskParams, ListAnnotationTasksParams, AnnotationItem, CreateAnnotationItemParams, ListAnnotationItemsParams, APIKey, APIKeyWithSecret, CreateAPIKeyParams, UpdateAPIKeyParams, ListAPIKeysParams, APIKeyUsage, Webhook, CreateWebhookParams, UpdateWebhookParams, ListWebhooksParams, WebhookDelivery, ListWebhookDeliveriesParams, UsageStats, GetUsageParams, UsageSummary, LLMJudgeConfig, CreateLLMJudgeConfigParams, ListLLMJudgeConfigsParams, ListLLMJudgeResultsParams, LLMJudgeAlignment, GetLLMJudgeAlignmentParams, Organization, } from './types';
|
|
34
|
+
import { AIEvalClient } from './client';
|
|
35
|
+
export default AIEvalClient;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* AI Evaluation Platform SDK
|
|
4
|
+
*
|
|
5
|
+
* Official TypeScript/JavaScript SDK for the AI Evaluation Platform.
|
|
6
|
+
* Build confidence in your AI systems with comprehensive evaluation tools.
|
|
7
|
+
*
|
|
8
|
+
* @packageDocumentation
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.decodeCursor = exports.encodeCursor = exports.autoPaginate = exports.createPaginatedIterator = exports.PaginatedIterator = exports.CacheTTL = exports.RequestCache = exports.RateLimiter = exports.batchRead = exports.streamEvaluation = exports.batchProcess = exports.importData = exports.exportData = exports.compareSnapshots = exports.saveSnapshot = exports.compareWithSnapshot = exports.snapshot = exports.TestSuite = exports.createTestSuite = exports.ContextManager = exports.withContext = exports.getContext = exports.createContext = exports.hasValidCodeSyntax = exports.containsAllRequiredFields = exports.followsInstructions = exports.hasNoToxicity = exports.respondedWithinTime = exports.hasFactualAccuracy = exports.containsLanguage = exports.hasReadabilityScore = exports.matchesSchema = exports.hasNoHallucinations = exports.isValidURL = exports.isValidEmail = exports.withinRange = exports.similarTo = exports.hasSentiment = exports.notContainsPII = exports.containsJSON = exports.hasLength = exports.matchesPattern = exports.containsKeywords = exports.expect = exports.NetworkError = exports.ValidationError = exports.AuthenticationError = exports.RateLimitError = exports.EvalAIError = exports.AIEvalClient = void 0;
|
|
12
|
+
exports.EvaluationTemplates = exports.traceAutoGen = exports.traceCrewAI = exports.traceLangChainAgent = exports.traceWorkflowStep = exports.createWorkflowTracer = exports.WorkflowTracer = exports.traceAnthropic = exports.traceOpenAI = exports.Logger = exports.RequestBatcher = void 0;
|
|
13
|
+
// Main SDK exports
|
|
14
|
+
var client_1 = require("./client");
|
|
15
|
+
Object.defineProperty(exports, "AIEvalClient", { enumerable: true, get: function () { return client_1.AIEvalClient; } });
|
|
16
|
+
// Enhanced error handling (Tier 1.5)
|
|
17
|
+
const errors_1 = require("./errors");
|
|
18
|
+
Object.defineProperty(exports, "EvalAIError", { enumerable: true, get: function () { return errors_1.EvalAIError; } });
|
|
19
|
+
Object.defineProperty(exports, "RateLimitError", { enumerable: true, get: function () { return errors_1.RateLimitError; } });
|
|
20
|
+
Object.defineProperty(exports, "AuthenticationError", { enumerable: true, get: function () { return errors_1.AuthenticationError; } });
|
|
21
|
+
Object.defineProperty(exports, "NetworkError", { enumerable: true, get: function () { return errors_1.NetworkError; } });
|
|
22
|
+
Object.defineProperty(exports, "ValidationError", { enumerable: true, get: function () { return errors_1.SDKError; } });
|
|
23
|
+
// Enhanced assertions (Tier 1.3)
|
|
24
|
+
var assertions_1 = require("./assertions");
|
|
25
|
+
Object.defineProperty(exports, "expect", { enumerable: true, get: function () { return assertions_1.expect; } });
|
|
26
|
+
Object.defineProperty(exports, "containsKeywords", { enumerable: true, get: function () { return assertions_1.containsKeywords; } });
|
|
27
|
+
Object.defineProperty(exports, "matchesPattern", { enumerable: true, get: function () { return assertions_1.matchesPattern; } });
|
|
28
|
+
Object.defineProperty(exports, "hasLength", { enumerable: true, get: function () { return assertions_1.hasLength; } });
|
|
29
|
+
Object.defineProperty(exports, "containsJSON", { enumerable: true, get: function () { return assertions_1.containsJSON; } });
|
|
30
|
+
Object.defineProperty(exports, "notContainsPII", { enumerable: true, get: function () { return assertions_1.notContainsPII; } });
|
|
31
|
+
Object.defineProperty(exports, "hasSentiment", { enumerable: true, get: function () { return assertions_1.hasSentiment; } });
|
|
32
|
+
Object.defineProperty(exports, "similarTo", { enumerable: true, get: function () { return assertions_1.similarTo; } });
|
|
33
|
+
Object.defineProperty(exports, "withinRange", { enumerable: true, get: function () { return assertions_1.withinRange; } });
|
|
34
|
+
Object.defineProperty(exports, "isValidEmail", { enumerable: true, get: function () { return assertions_1.isValidEmail; } });
|
|
35
|
+
Object.defineProperty(exports, "isValidURL", { enumerable: true, get: function () { return assertions_1.isValidURL; } });
|
|
36
|
+
Object.defineProperty(exports, "hasNoHallucinations", { enumerable: true, get: function () { return assertions_1.hasNoHallucinations; } });
|
|
37
|
+
Object.defineProperty(exports, "matchesSchema", { enumerable: true, get: function () { return assertions_1.matchesSchema; } });
|
|
38
|
+
Object.defineProperty(exports, "hasReadabilityScore", { enumerable: true, get: function () { return assertions_1.hasReadabilityScore; } });
|
|
39
|
+
Object.defineProperty(exports, "containsLanguage", { enumerable: true, get: function () { return assertions_1.containsLanguage; } });
|
|
40
|
+
Object.defineProperty(exports, "hasFactualAccuracy", { enumerable: true, get: function () { return assertions_1.hasFactualAccuracy; } });
|
|
41
|
+
Object.defineProperty(exports, "respondedWithinTime", { enumerable: true, get: function () { return assertions_1.respondedWithinTime; } });
|
|
42
|
+
Object.defineProperty(exports, "hasNoToxicity", { enumerable: true, get: function () { return assertions_1.hasNoToxicity; } });
|
|
43
|
+
Object.defineProperty(exports, "followsInstructions", { enumerable: true, get: function () { return assertions_1.followsInstructions; } });
|
|
44
|
+
Object.defineProperty(exports, "containsAllRequiredFields", { enumerable: true, get: function () { return assertions_1.containsAllRequiredFields; } });
|
|
45
|
+
Object.defineProperty(exports, "hasValidCodeSyntax", { enumerable: true, get: function () { return assertions_1.hasValidCodeSyntax; } });
|
|
46
|
+
// Context propagation (Tier 2.9)
|
|
47
|
+
const context_1 = require("./context");
|
|
48
|
+
Object.defineProperty(exports, "createContext", { enumerable: true, get: function () { return context_1.createContext; } });
|
|
49
|
+
Object.defineProperty(exports, "getContext", { enumerable: true, get: function () { return context_1.getCurrentContext; } });
|
|
50
|
+
Object.defineProperty(exports, "withContext", { enumerable: true, get: function () { return context_1.withContext; } });
|
|
51
|
+
Object.defineProperty(exports, "ContextManager", { enumerable: true, get: function () { return context_1.EvalContext; } });
|
|
52
|
+
// Test suite builder (Tier 2.7)
|
|
53
|
+
var testing_1 = require("./testing");
|
|
54
|
+
Object.defineProperty(exports, "createTestSuite", { enumerable: true, get: function () { return testing_1.createTestSuite; } });
|
|
55
|
+
Object.defineProperty(exports, "TestSuite", { enumerable: true, get: function () { return testing_1.TestSuite; } });
|
|
56
|
+
// Snapshot testing (Tier 2.8)
|
|
57
|
+
const snapshot_1 = require("./snapshot");
|
|
58
|
+
Object.defineProperty(exports, "snapshot", { enumerable: true, get: function () { return snapshot_1.snapshot; } });
|
|
59
|
+
Object.defineProperty(exports, "saveSnapshot", { enumerable: true, get: function () { return snapshot_1.snapshot; } });
|
|
60
|
+
Object.defineProperty(exports, "compareWithSnapshot", { enumerable: true, get: function () { return snapshot_1.compareWithSnapshot; } });
|
|
61
|
+
Object.defineProperty(exports, "compareSnapshots", { enumerable: true, get: function () { return snapshot_1.compareWithSnapshot; } });
|
|
62
|
+
// Export/Import utilities (Tier 4.18)
|
|
63
|
+
const export_1 = require("./export");
|
|
64
|
+
Object.defineProperty(exports, "exportData", { enumerable: true, get: function () { return export_1.exportData; } });
|
|
65
|
+
Object.defineProperty(exports, "importData", { enumerable: true, get: function () { return export_1.importData; } });
|
|
66
|
+
// Streaming and batch processing (Tier 3.3)
|
|
67
|
+
// Use functions from ./streaming module instead of these deprecated exports
|
|
68
|
+
var streaming_1 = require("./streaming");
|
|
69
|
+
Object.defineProperty(exports, "batchProcess", { enumerable: true, get: function () { return streaming_1.batchProcess; } });
|
|
70
|
+
Object.defineProperty(exports, "streamEvaluation", { enumerable: true, get: function () { return streaming_1.streamEvaluation; } });
|
|
71
|
+
Object.defineProperty(exports, "batchRead", { enumerable: true, get: function () { return streaming_1.batchRead; } });
|
|
72
|
+
Object.defineProperty(exports, "RateLimiter", { enumerable: true, get: function () { return streaming_1.RateLimiter; } });
|
|
73
|
+
// Performance optimization utilities (v1.3.0)
|
|
74
|
+
// Note: RequestCache and CacheTTL are for advanced users only
|
|
75
|
+
// Most users don't need these - caching is automatic
|
|
76
|
+
var cache_1 = require("./cache");
|
|
77
|
+
Object.defineProperty(exports, "RequestCache", { enumerable: true, get: function () { return cache_1.RequestCache; } });
|
|
78
|
+
Object.defineProperty(exports, "CacheTTL", { enumerable: true, get: function () { return cache_1.CacheTTL; } });
|
|
79
|
+
var pagination_1 = require("./pagination");
|
|
80
|
+
Object.defineProperty(exports, "PaginatedIterator", { enumerable: true, get: function () { return pagination_1.PaginatedIterator; } });
|
|
81
|
+
Object.defineProperty(exports, "createPaginatedIterator", { enumerable: true, get: function () { return pagination_1.createPaginatedIterator; } });
|
|
82
|
+
Object.defineProperty(exports, "autoPaginate", { enumerable: true, get: function () { return pagination_1.autoPaginate; } });
|
|
83
|
+
Object.defineProperty(exports, "encodeCursor", { enumerable: true, get: function () { return pagination_1.encodeCursor; } });
|
|
84
|
+
Object.defineProperty(exports, "decodeCursor", { enumerable: true, get: function () { return pagination_1.decodeCursor; } });
|
|
85
|
+
// Note: RequestBatcher is for advanced users only
|
|
86
|
+
// Most users don't need this - batching is automatic
|
|
87
|
+
var batch_1 = require("./batch");
|
|
88
|
+
Object.defineProperty(exports, "RequestBatcher", { enumerable: true, get: function () { return batch_1.RequestBatcher; } });
|
|
89
|
+
// Debug logger (Tier 4.17)
|
|
90
|
+
var logger_1 = require("./logger");
|
|
91
|
+
Object.defineProperty(exports, "Logger", { enumerable: true, get: function () { return logger_1.Logger; } });
|
|
92
|
+
// Framework integrations (Tier 1.2)
|
|
93
|
+
var openai_1 = require("./integrations/openai");
|
|
94
|
+
Object.defineProperty(exports, "traceOpenAI", { enumerable: true, get: function () { return openai_1.traceOpenAI; } });
|
|
95
|
+
var anthropic_1 = require("./integrations/anthropic");
|
|
96
|
+
Object.defineProperty(exports, "traceAnthropic", { enumerable: true, get: function () { return anthropic_1.traceAnthropic; } });
|
|
97
|
+
// Workflow tracing (Orchestration Layer)
|
|
98
|
+
var workflows_1 = require("./workflows");
|
|
99
|
+
Object.defineProperty(exports, "WorkflowTracer", { enumerable: true, get: function () { return workflows_1.WorkflowTracer; } });
|
|
100
|
+
Object.defineProperty(exports, "createWorkflowTracer", { enumerable: true, get: function () { return workflows_1.createWorkflowTracer; } });
|
|
101
|
+
Object.defineProperty(exports, "traceWorkflowStep", { enumerable: true, get: function () { return workflows_1.traceWorkflowStep; } });
|
|
102
|
+
// Framework integrations
|
|
103
|
+
Object.defineProperty(exports, "traceLangChainAgent", { enumerable: true, get: function () { return workflows_1.traceLangChainAgent; } });
|
|
104
|
+
Object.defineProperty(exports, "traceCrewAI", { enumerable: true, get: function () { return workflows_1.traceCrewAI; } });
|
|
105
|
+
Object.defineProperty(exports, "traceAutoGen", { enumerable: true, get: function () { return workflows_1.traceAutoGen; } });
|
|
106
|
+
// New exports for v1.1.0
|
|
107
|
+
var types_1 = require("./types");
|
|
108
|
+
Object.defineProperty(exports, "EvaluationTemplates", { enumerable: true, get: function () { return types_1.EvaluationTemplates; } });
|
|
109
|
+
// Default export for convenience
|
|
110
|
+
const client_2 = require("./client");
|
|
111
|
+
exports.default = client_2.AIEvalClient;
|