@artemiskit/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +229 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +352 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +354 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +53 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +44918 -28644
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +13 -59
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/jest.js +0 -46
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/matchers/vitest.js +0 -46
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +379 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +58 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +450 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +310 -0
- package/src/artemiskit.ts +479 -5
- package/src/builders/index.ts +700 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +186 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
- package/adapters/openai/dist/index.js +0 -5625
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Type definitions for the programmatic SDK
|
|
4
|
+
*/
|
|
5
|
+
import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
|
|
6
|
+
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
|
+
import type { Scenario } from '@artemiskit/core';
|
|
8
|
+
/**
|
|
9
|
+
* Event emitted when a test case starts
|
|
10
|
+
*/
|
|
11
|
+
export interface CaseStartEvent {
|
|
12
|
+
caseId: string;
|
|
13
|
+
caseName?: string;
|
|
14
|
+
index: number;
|
|
15
|
+
total: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Event emitted when a test case completes
|
|
19
|
+
*/
|
|
20
|
+
export interface CaseCompleteEvent {
|
|
21
|
+
result: CaseResult;
|
|
22
|
+
index: number;
|
|
23
|
+
total: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Event emitted for progress updates
|
|
27
|
+
*/
|
|
28
|
+
export interface ProgressEvent {
|
|
29
|
+
message: string;
|
|
30
|
+
phase: 'setup' | 'running' | 'teardown';
|
|
31
|
+
progress?: number;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Event emitted when a red team mutation starts
|
|
35
|
+
*/
|
|
36
|
+
export interface RedTeamMutationStartEvent {
|
|
37
|
+
mutation: string;
|
|
38
|
+
caseId: string;
|
|
39
|
+
index: number;
|
|
40
|
+
total: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Event emitted when a red team mutation completes
|
|
44
|
+
*/
|
|
45
|
+
export interface RedTeamMutationCompleteEvent {
|
|
46
|
+
mutation: string;
|
|
47
|
+
caseId: string;
|
|
48
|
+
status: 'safe' | 'unsafe' | 'blocked' | 'error';
|
|
49
|
+
severity: 'none' | 'low' | 'medium' | 'high' | 'critical';
|
|
50
|
+
index: number;
|
|
51
|
+
total: number;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Event emitted for stress test request completion
|
|
55
|
+
*/
|
|
56
|
+
export interface StressRequestCompleteEvent {
|
|
57
|
+
result: StressRequestResult;
|
|
58
|
+
index: number;
|
|
59
|
+
total: number;
|
|
60
|
+
currentRPS: number;
|
|
61
|
+
}
|
|
62
|
+
export type CaseStartHandler = (event: CaseStartEvent) => void;
|
|
63
|
+
export type CaseCompleteHandler = (event: CaseCompleteEvent) => void;
|
|
64
|
+
export type ProgressHandler = (event: ProgressEvent) => void;
|
|
65
|
+
export type RedTeamMutationStartHandler = (event: RedTeamMutationStartEvent) => void;
|
|
66
|
+
export type RedTeamMutationCompleteHandler = (event: RedTeamMutationCompleteEvent) => void;
|
|
67
|
+
export type StressRequestCompleteHandler = (event: StressRequestCompleteEvent) => void;
|
|
68
|
+
/**
|
|
69
|
+
* SDK-level configuration options
|
|
70
|
+
*/
|
|
71
|
+
export interface ArtemisKitConfig {
|
|
72
|
+
/** Project name for manifest grouping */
|
|
73
|
+
project?: string;
|
|
74
|
+
/** Default provider to use */
|
|
75
|
+
provider?: AdapterConfig['provider'];
|
|
76
|
+
/** Default model to use */
|
|
77
|
+
model?: string;
|
|
78
|
+
/** Provider-specific configuration */
|
|
79
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
80
|
+
/** Default redaction settings */
|
|
81
|
+
redaction?: RedactionConfig;
|
|
82
|
+
/** Default timeout per case in milliseconds */
|
|
83
|
+
timeout?: number;
|
|
84
|
+
/** Default number of retries per case */
|
|
85
|
+
retries?: number;
|
|
86
|
+
/** Default concurrency for parallel execution */
|
|
87
|
+
concurrency?: number;
|
|
88
|
+
/**
|
|
89
|
+
* Storage configuration for persisting and loading run manifests
|
|
90
|
+
* Required for compare() method to load historical runs
|
|
91
|
+
* @since 0.3.2
|
|
92
|
+
*/
|
|
93
|
+
storage?: StorageConfig;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Options for running test scenarios
|
|
97
|
+
*/
|
|
98
|
+
export interface RunOptions {
|
|
99
|
+
/** Path to scenario file or inline Scenario object */
|
|
100
|
+
scenario: string | Scenario;
|
|
101
|
+
/** Override provider */
|
|
102
|
+
provider?: AdapterConfig['provider'];
|
|
103
|
+
/** Override model */
|
|
104
|
+
model?: string;
|
|
105
|
+
/** Provider-specific configuration override */
|
|
106
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
107
|
+
/** Pre-configured model client (skips adapter creation) */
|
|
108
|
+
client?: ModelClient;
|
|
109
|
+
/** Filter test cases by tags */
|
|
110
|
+
tags?: string[];
|
|
111
|
+
/** Number of concurrent requests */
|
|
112
|
+
concurrency?: number;
|
|
113
|
+
/** Timeout per case in milliseconds */
|
|
114
|
+
timeout?: number;
|
|
115
|
+
/** Number of retries per case */
|
|
116
|
+
retries?: number;
|
|
117
|
+
/** Redaction configuration */
|
|
118
|
+
redaction?: RedactionConfig;
|
|
119
|
+
/** Explicit executor for real tools; CLI scenarios remain fixture-only. */
|
|
120
|
+
toolExecutor?: ToolExecutor;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Options for red team testing
|
|
124
|
+
*/
|
|
125
|
+
export interface RedTeamOptions {
|
|
126
|
+
/** Path to scenario file or inline Scenario object */
|
|
127
|
+
scenario: string | Scenario;
|
|
128
|
+
/** Override provider */
|
|
129
|
+
provider?: AdapterConfig['provider'];
|
|
130
|
+
/** Override model */
|
|
131
|
+
model?: string;
|
|
132
|
+
/** Provider-specific configuration override */
|
|
133
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
134
|
+
/** Pre-configured model client (skips adapter creation) */
|
|
135
|
+
client?: ModelClient;
|
|
136
|
+
/** Mutations to apply (default: all) */
|
|
137
|
+
mutations?: string[];
|
|
138
|
+
/** Number of mutations per case */
|
|
139
|
+
countPerCase?: number;
|
|
140
|
+
/** Filter test cases by tags */
|
|
141
|
+
tags?: string[];
|
|
142
|
+
/** Timeout per case in milliseconds */
|
|
143
|
+
timeout?: number;
|
|
144
|
+
/** Redaction configuration */
|
|
145
|
+
redaction?: RedactionConfig;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Options for stress testing
|
|
149
|
+
*/
|
|
150
|
+
export interface StressOptions {
|
|
151
|
+
/** Path to scenario file or inline Scenario object */
|
|
152
|
+
scenario: string | Scenario;
|
|
153
|
+
/** Override provider */
|
|
154
|
+
provider?: AdapterConfig['provider'];
|
|
155
|
+
/** Override model */
|
|
156
|
+
model?: string;
|
|
157
|
+
/** Provider-specific configuration override */
|
|
158
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
159
|
+
/** Pre-configured model client (skips adapter creation) */
|
|
160
|
+
client?: ModelClient;
|
|
161
|
+
/** Number of concurrent requests */
|
|
162
|
+
concurrency?: number;
|
|
163
|
+
/** Test duration in seconds */
|
|
164
|
+
duration?: number;
|
|
165
|
+
/** Ramp-up period in seconds */
|
|
166
|
+
rampUp?: number;
|
|
167
|
+
/** Maximum number of requests (optional limit) */
|
|
168
|
+
maxRequests?: number;
|
|
169
|
+
/** Redaction configuration */
|
|
170
|
+
redaction?: RedactionConfig;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Result from a test scenario run
|
|
174
|
+
*/
|
|
175
|
+
export interface RunResult {
|
|
176
|
+
/** The generated manifest */
|
|
177
|
+
manifest: RunManifest;
|
|
178
|
+
/** Individual case results */
|
|
179
|
+
cases: CaseResult[];
|
|
180
|
+
/** Whether all cases passed */
|
|
181
|
+
success: boolean;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Result from a red team test run
|
|
185
|
+
*/
|
|
186
|
+
export interface RedTeamResult {
|
|
187
|
+
/** The generated manifest */
|
|
188
|
+
manifest: RedTeamManifest;
|
|
189
|
+
/** Whether the defense rate is acceptable (>= threshold) */
|
|
190
|
+
success: boolean;
|
|
191
|
+
/** Defense rate (0-1) */
|
|
192
|
+
defenseRate: number;
|
|
193
|
+
/** Count of unsafe responses */
|
|
194
|
+
unsafeCount: number;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Result from a stress test run
|
|
198
|
+
*/
|
|
199
|
+
export interface StressResult {
|
|
200
|
+
/** The generated manifest */
|
|
201
|
+
manifest: StressManifest;
|
|
202
|
+
/** Whether the test passed success rate threshold */
|
|
203
|
+
success: boolean;
|
|
204
|
+
/** Success rate (0-1) */
|
|
205
|
+
successRate: number;
|
|
206
|
+
/** Requests per second achieved */
|
|
207
|
+
rps: number;
|
|
208
|
+
/** P95 latency in milliseconds */
|
|
209
|
+
p95LatencyMs: number;
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Event types supported by ArtemisKit
|
|
213
|
+
*/
|
|
214
|
+
export interface ArtemisKitEvents {
|
|
215
|
+
caseStart: CaseStartEvent;
|
|
216
|
+
caseComplete: CaseCompleteEvent;
|
|
217
|
+
progress: ProgressEvent;
|
|
218
|
+
redteamMutationStart: RedTeamMutationStartEvent;
|
|
219
|
+
redteamMutationComplete: RedTeamMutationCompleteEvent;
|
|
220
|
+
stressRequestComplete: StressRequestCompleteEvent;
|
|
221
|
+
}
|
|
222
|
+
export type ArtemisKitEventName = keyof ArtemisKitEvents;
|
|
223
|
+
/**
|
|
224
|
+
* Options for validating scenario files
|
|
225
|
+
*/
|
|
226
|
+
export interface ValidateOptions {
|
|
227
|
+
/**
|
|
228
|
+
* Scenario file path(s) or glob pattern
|
|
229
|
+
* Can be a single path, array of paths, or glob pattern
|
|
230
|
+
*/
|
|
231
|
+
scenario: string | string[];
|
|
232
|
+
/**
|
|
233
|
+
* Strict mode - fail on warnings as well as errors
|
|
234
|
+
* @default false
|
|
235
|
+
*/
|
|
236
|
+
strict?: boolean;
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Validation error details
|
|
240
|
+
*/
|
|
241
|
+
export interface ValidationError {
|
|
242
|
+
/** File path where error occurred */
|
|
243
|
+
file: string;
|
|
244
|
+
/** Error message */
|
|
245
|
+
message: string;
|
|
246
|
+
/** Line number if available */
|
|
247
|
+
line?: number;
|
|
248
|
+
/** Column number if available */
|
|
249
|
+
column?: number;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Validation warning details
|
|
253
|
+
*/
|
|
254
|
+
export interface ValidationWarning {
|
|
255
|
+
/** File path where warning occurred */
|
|
256
|
+
file: string;
|
|
257
|
+
/** Warning message */
|
|
258
|
+
message: string;
|
|
259
|
+
/** Line number if available */
|
|
260
|
+
line?: number;
|
|
261
|
+
/** Column number if available */
|
|
262
|
+
column?: number;
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Per-scenario validation result
|
|
266
|
+
*/
|
|
267
|
+
export interface ScenarioValidation {
|
|
268
|
+
/** File path */
|
|
269
|
+
file: string;
|
|
270
|
+
/** Scenario name */
|
|
271
|
+
name: string;
|
|
272
|
+
/** Number of test cases */
|
|
273
|
+
caseCount: number;
|
|
274
|
+
/** Whether this scenario is valid */
|
|
275
|
+
valid: boolean;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Result from validate() method
|
|
279
|
+
*/
|
|
280
|
+
export interface ValidationResult {
|
|
281
|
+
/** Overall validation passed */
|
|
282
|
+
valid: boolean;
|
|
283
|
+
/** Individual scenario validation results */
|
|
284
|
+
scenarios: ScenarioValidation[];
|
|
285
|
+
/** Validation errors */
|
|
286
|
+
errors: ValidationError[];
|
|
287
|
+
/** Validation warnings (informational) */
|
|
288
|
+
warnings: ValidationWarning[];
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Options for comparing test runs
|
|
292
|
+
*/
|
|
293
|
+
export interface CompareOptions {
|
|
294
|
+
/**
|
|
295
|
+
* Baseline run identifier
|
|
296
|
+
* Can be a run ID, 'latest', or a named baseline
|
|
297
|
+
*/
|
|
298
|
+
baseline: string;
|
|
299
|
+
/**
|
|
300
|
+
* Current run identifier
|
|
301
|
+
* Can be a run ID
|
|
302
|
+
*/
|
|
303
|
+
current: string;
|
|
304
|
+
/**
|
|
305
|
+
* Regression threshold (0-1)
|
|
306
|
+
* A regression is detected if success rate drops by more than this amount
|
|
307
|
+
* @default 0.05 (5%)
|
|
308
|
+
*/
|
|
309
|
+
threshold?: number;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Summary of a run for comparison
|
|
313
|
+
*/
|
|
314
|
+
export interface RunSummary {
|
|
315
|
+
/** Run identifier */
|
|
316
|
+
runId: string;
|
|
317
|
+
/** Success rate (0-1) */
|
|
318
|
+
successRate: number;
|
|
319
|
+
/** Total number of test cases */
|
|
320
|
+
totalCases: number;
|
|
321
|
+
/** Number of passed cases */
|
|
322
|
+
passedCases: number;
|
|
323
|
+
/** Number of failed cases */
|
|
324
|
+
failedCases: number;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Comparison details between two runs
|
|
328
|
+
*/
|
|
329
|
+
export interface ComparisonDetails {
|
|
330
|
+
/** Change in success rate (current - baseline) */
|
|
331
|
+
successRateDelta: number;
|
|
332
|
+
/** Cases that passed in baseline but failed in current */
|
|
333
|
+
newFailures: Array<{
|
|
334
|
+
caseId: string;
|
|
335
|
+
caseName?: string;
|
|
336
|
+
baselineStatus: string;
|
|
337
|
+
currentStatus: string;
|
|
338
|
+
}>;
|
|
339
|
+
/** Cases that failed in baseline but passed in current */
|
|
340
|
+
newPasses: Array<{
|
|
341
|
+
caseId: string;
|
|
342
|
+
caseName?: string;
|
|
343
|
+
baselineStatus: string;
|
|
344
|
+
currentStatus: string;
|
|
345
|
+
}>;
|
|
346
|
+
/** Cases with unchanged status */
|
|
347
|
+
unchanged: Array<{
|
|
348
|
+
caseId: string;
|
|
349
|
+
status: string;
|
|
350
|
+
}>;
|
|
351
|
+
/** Cases that exist in current run but not in baseline (new test cases) */
|
|
352
|
+
addedCases: Array<{
|
|
353
|
+
caseId: string;
|
|
354
|
+
caseName?: string;
|
|
355
|
+
status: string;
|
|
356
|
+
}>;
|
|
357
|
+
/** Cases that exist in baseline but not in current run (removed test cases) */
|
|
358
|
+
removedCases: Array<{
|
|
359
|
+
caseId: string;
|
|
360
|
+
caseName?: string;
|
|
361
|
+
status: string;
|
|
362
|
+
}>;
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Result from compare() method
|
|
366
|
+
*/
|
|
367
|
+
export interface CompareResult {
|
|
368
|
+
/** Baseline run summary */
|
|
369
|
+
baseline: RunSummary;
|
|
370
|
+
/** Current run summary */
|
|
371
|
+
current: RunSummary;
|
|
372
|
+
/** Detailed comparison */
|
|
373
|
+
comparison: ComparisonDetails;
|
|
374
|
+
/** Whether a regression was detected */
|
|
375
|
+
hasRegression: boolean;
|
|
376
|
+
/** Threshold used for regression detection */
|
|
377
|
+
threshold: number;
|
|
378
|
+
}
|
|
379
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,wCAAwC;IACxC,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Utility types and helper functions
|
|
4
|
+
*/
|
|
5
|
+
import type { AdapterConfig, AnyManifest, CaseResult, Expected, RedTeamManifest, RunManifest, Scenario, StressManifest } from '@artemiskit/core';
|
|
6
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
7
|
+
/**
|
|
8
|
+
* All supported provider names as a union type
|
|
9
|
+
*/
|
|
10
|
+
export type ProviderName = AdapterConfig['provider'];
|
|
11
|
+
/**
|
|
12
|
+
* All supported expectation types
|
|
13
|
+
*/
|
|
14
|
+
export type ExpectationType = Expected['type'];
|
|
15
|
+
/**
|
|
16
|
+
* Union of all SDK result types
|
|
17
|
+
*/
|
|
18
|
+
export type AnyResult = RunResult | RedTeamResult | StressResult;
|
|
19
|
+
/**
|
|
20
|
+
* Union of all manifest types (re-export from core for convenience)
|
|
21
|
+
*/
|
|
22
|
+
export type { AnyManifest };
|
|
23
|
+
/**
|
|
24
|
+
* Extract case results from a RunResult
|
|
25
|
+
*/
|
|
26
|
+
export type ExtractRunCases = RunResult['cases'];
|
|
27
|
+
/**
|
|
28
|
+
* Extract red team cases from a RedTeamResult
|
|
29
|
+
*/
|
|
30
|
+
export type ExtractRedTeamCases = RedTeamResult['manifest']['results'];
|
|
31
|
+
/**
|
|
32
|
+
* Extract stress results from a StressResult
|
|
33
|
+
*/
|
|
34
|
+
export type ExtractStressResults = StressResult['manifest']['sample_results'];
|
|
35
|
+
/**
|
|
36
|
+
* Extract the manifest type from any result type
|
|
37
|
+
*/
|
|
38
|
+
export type ExtractManifest<T extends AnyResult> = T extends RunResult ? RunManifest : T extends RedTeamResult ? RedTeamManifest : T extends StressResult ? StressManifest : never;
|
|
39
|
+
/**
|
|
40
|
+
* Deep partial utility type - makes all nested properties optional
|
|
41
|
+
*/
|
|
42
|
+
export type DeepPartial<T> = T extends object ? {
|
|
43
|
+
[P in keyof T]?: DeepPartial<T[P]>;
|
|
44
|
+
} : T;
|
|
45
|
+
/**
|
|
46
|
+
* Partial scenario for incremental construction
|
|
47
|
+
*/
|
|
48
|
+
export type PartialScenario = DeepPartial<Scenario>;
|
|
49
|
+
/**
|
|
50
|
+
* Make specific fields required
|
|
51
|
+
*/
|
|
52
|
+
export type RequireFields<T, K extends keyof T> = T & Required<Pick<T, K>>;
|
|
53
|
+
/**
|
|
54
|
+
* Make specific fields optional
|
|
55
|
+
*/
|
|
56
|
+
export type OptionalFields<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
|
|
57
|
+
/**
|
|
58
|
+
* Strict version of AdapterConfig with all common fields required
|
|
59
|
+
*/
|
|
60
|
+
export type StrictAdapterConfig = RequireFields<AdapterConfig, 'provider' | 'apiKey'>;
|
|
61
|
+
/**
|
|
62
|
+
* Type guard for RunResult
|
|
63
|
+
*/
|
|
64
|
+
export declare function isRunResult(result: AnyResult): result is RunResult;
|
|
65
|
+
/**
|
|
66
|
+
* Type guard for RedTeamResult
|
|
67
|
+
*/
|
|
68
|
+
export declare function isRedTeamResult(result: AnyResult): result is RedTeamResult;
|
|
69
|
+
/**
|
|
70
|
+
* Type guard for StressResult
|
|
71
|
+
*/
|
|
72
|
+
export declare function isStressResult(result: AnyResult): result is StressResult;
|
|
73
|
+
/**
|
|
74
|
+
* Type guard for RunManifest
|
|
75
|
+
*
|
|
76
|
+
* Uses structural field checks rather than absence-of-field logic for robustness.
|
|
77
|
+
* Checks for RunManifest-specific fields: 'metrics.success_rate' and 'cases' array.
|
|
78
|
+
*/
|
|
79
|
+
export declare function isRunManifestType(manifest: AnyManifest): manifest is RunManifest;
|
|
80
|
+
/**
|
|
81
|
+
* Type guard for RedTeamManifest
|
|
82
|
+
*/
|
|
83
|
+
export declare function isRedTeamManifestType(manifest: AnyManifest): manifest is RedTeamManifest;
|
|
84
|
+
/**
|
|
85
|
+
* Type guard for StressManifest
|
|
86
|
+
*/
|
|
87
|
+
export declare function isStressManifestType(manifest: AnyManifest): manifest is StressManifest;
|
|
88
|
+
/**
|
|
89
|
+
* Assert that a value is defined (non-null, non-undefined)
|
|
90
|
+
*/
|
|
91
|
+
export declare function assertDefined<T>(value: T | null | undefined, message?: string): asserts value is T;
|
|
92
|
+
/**
|
|
93
|
+
* Assert that a condition is true
|
|
94
|
+
*/
|
|
95
|
+
export declare function assert(condition: boolean, message?: string): asserts condition;
|
|
96
|
+
/**
|
|
97
|
+
* Get failed cases from a run result
|
|
98
|
+
*/
|
|
99
|
+
export declare function getFailedCases(result: RunResult): CaseResult[];
|
|
100
|
+
/**
|
|
101
|
+
* Get passed cases from a run result
|
|
102
|
+
*/
|
|
103
|
+
export declare function getPassedCases(result: RunResult): CaseResult[];
|
|
104
|
+
/**
|
|
105
|
+
* Get cases by tag from a run result
|
|
106
|
+
*/
|
|
107
|
+
export declare function getCasesByTag(result: RunResult, tag: string): CaseResult[];
|
|
108
|
+
/**
|
|
109
|
+
* Calculate success rate from a run result
|
|
110
|
+
*/
|
|
111
|
+
export declare function calculateSuccessRate(result: RunResult): number;
|
|
112
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,UAAU,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,QAAQ,EACR,cAAc,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAMvE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAErD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAM/C;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,aAAa,GAAG,YAAY,CAAC;AAEjE;;GAEG;AACH,YAAY,EAAE,WAAW,EAAE,CAAC;AAM5B;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;AAE9E;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,SAAS,IAAI,CAAC,SAAS,SAAS,GAClE,WAAW,GACX,CAAC,SAAS,aAAa,GACrB,eAAe,GACf,CAAC,SAAS,YAAY,GACpB,cAAc,GACd,KAAK,CAAC;AAMd;;GAEG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,GACzC;KACG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CACnC,GACD,CAAC,CAAC;AAEN;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAEpD;;GAEG;AACH,MAAM,MAAM,aAAa,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAE3E;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAMpF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,aAAa,EAAE,UAAU,GAAG,QAAQ,CAAC,CAAC;AAMtF;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,SAAS,CAElE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,aAAa,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,YAAY,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAShF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAExF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAEtF;AAMD;;GAEG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAC7B,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,SAAS,EAC3B,OAAO,SAA+B,GACrC,OAAO,CAAC,KAAK,IAAI,CAAC,CAIpB;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,SAAqB,GAAG,OAAO,CAAC,SAAS,CAI1F;AAMD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,EAAE,CAE9D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,EAAE,CAE9D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,EAAE,CAE1E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAG9D"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
// src/utils/index.ts
|
|
3
|
+
function isRunResult(result) {
|
|
4
|
+
return "cases" in result && Array.isArray(result.cases) && !("defenseRate" in result);
|
|
5
|
+
}
|
|
6
|
+
function isRedTeamResult(result) {
|
|
7
|
+
return "defenseRate" in result && "unsafeCount" in result;
|
|
8
|
+
}
|
|
9
|
+
function isStressResult(result) {
|
|
10
|
+
return "rps" in result && "p95LatencyMs" in result;
|
|
11
|
+
}
|
|
12
|
+
function isRunManifestType(manifest) {
|
|
13
|
+
return "metrics" in manifest && manifest.metrics !== null && typeof manifest.metrics === "object" && "success_rate" in manifest.metrics && "cases" in manifest && Array.isArray(manifest.cases);
|
|
14
|
+
}
|
|
15
|
+
function isRedTeamManifestType(manifest) {
|
|
16
|
+
return "type" in manifest && manifest.type === "redteam";
|
|
17
|
+
}
|
|
18
|
+
function isStressManifestType(manifest) {
|
|
19
|
+
return "type" in manifest && manifest.type === "stress";
|
|
20
|
+
}
|
|
21
|
+
function assertDefined(value, message = "Value is null or undefined") {
|
|
22
|
+
if (value === null || value === undefined) {
|
|
23
|
+
throw new Error(message);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function assert(condition, message = "Assertion failed") {
|
|
27
|
+
if (!condition) {
|
|
28
|
+
throw new Error(message);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function getFailedCases(result) {
|
|
32
|
+
return result.cases.filter((c) => !c.ok);
|
|
33
|
+
}
|
|
34
|
+
function getPassedCases(result) {
|
|
35
|
+
return result.cases.filter((c) => c.ok);
|
|
36
|
+
}
|
|
37
|
+
function getCasesByTag(result, tag) {
|
|
38
|
+
return result.cases.filter((c) => c.tags.includes(tag));
|
|
39
|
+
}
|
|
40
|
+
function calculateSuccessRate(result) {
|
|
41
|
+
if (result.cases.length === 0)
|
|
42
|
+
return 0;
|
|
43
|
+
return result.cases.filter((c) => c.ok).length / result.cases.length;
|
|
44
|
+
}
|
|
45
|
+
export {
|
|
46
|
+
assert,
|
|
47
|
+
assertDefined,
|
|
48
|
+
calculateSuccessRate,
|
|
49
|
+
getCasesByTag,
|
|
50
|
+
getFailedCases,
|
|
51
|
+
getPassedCases,
|
|
52
|
+
isRedTeamManifestType,
|
|
53
|
+
isRedTeamResult,
|
|
54
|
+
isRunManifestType,
|
|
55
|
+
isRunResult,
|
|
56
|
+
isStressManifestType,
|
|
57
|
+
isStressResult
|
|
58
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@artemiskit/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -46,18 +46,33 @@
|
|
|
46
46
|
"./jest": {
|
|
47
47
|
"import": "./dist/matchers/jest.js",
|
|
48
48
|
"types": "./dist/matchers/jest.d.ts"
|
|
49
|
+
},
|
|
50
|
+
"./builders": {
|
|
51
|
+
"import": "./dist/builders/index.js",
|
|
52
|
+
"types": "./dist/builders/index.d.ts"
|
|
53
|
+
},
|
|
54
|
+
"./contracts": {
|
|
55
|
+
"import": "./dist/contracts/index.js",
|
|
56
|
+
"types": "./dist/contracts/index.d.ts"
|
|
57
|
+
},
|
|
58
|
+
"./utils": {
|
|
59
|
+
"import": "./dist/utils/index.js",
|
|
60
|
+
"types": "./dist/utils/index.d.ts"
|
|
61
|
+
},
|
|
62
|
+
"./types": {
|
|
63
|
+
"types": "./dist/types-only.d.ts"
|
|
49
64
|
}
|
|
50
65
|
},
|
|
51
66
|
"scripts": {
|
|
52
|
-
"build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts --outdir ./dist --target bun",
|
|
67
|
+
"build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts ./src/builders/index.ts ./src/contracts/index.ts ./src/utils/index.ts ./src/types-only.ts --outdir ./dist --target bun",
|
|
53
68
|
"typecheck": "tsc --noEmit",
|
|
54
69
|
"clean": "rm -rf dist",
|
|
55
70
|
"test": "bun test",
|
|
56
71
|
"test:coverage": "bun test --coverage"
|
|
57
72
|
},
|
|
58
73
|
"dependencies": {
|
|
59
|
-
"@artemiskit/core": "0.
|
|
60
|
-
"@artemiskit/redteam": "0.3.
|
|
74
|
+
"@artemiskit/core": "0.4.0",
|
|
75
|
+
"@artemiskit/redteam": "0.3.1"
|
|
61
76
|
},
|
|
62
77
|
"devDependencies": {
|
|
63
78
|
"@types/bun": "^1.1.0",
|