@artemiskit/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +229 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +352 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +354 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +53 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +44918 -28644
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +13 -59
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/jest.js +0 -46
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/matchers/vitest.js +0 -46
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +379 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +58 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +450 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +310 -0
- package/src/artemiskit.ts +479 -5
- package/src/builders/index.ts +700 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +186 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
- package/adapters/openai/dist/index.js +0 -5625
package/src/types.ts
CHANGED
|
@@ -9,8 +9,10 @@ import type {
|
|
|
9
9
|
ModelClient,
|
|
10
10
|
RedTeamManifest,
|
|
11
11
|
RunManifest,
|
|
12
|
+
StorageConfig,
|
|
12
13
|
StressManifest,
|
|
13
14
|
StressRequestResult,
|
|
15
|
+
ToolExecutor,
|
|
14
16
|
} from '@artemiskit/core';
|
|
15
17
|
import type { RedactionConfig } from '@artemiskit/core';
|
|
16
18
|
import type { Scenario } from '@artemiskit/core';
|
|
@@ -114,6 +116,12 @@ export interface ArtemisKitConfig {
|
|
|
114
116
|
retries?: number;
|
|
115
117
|
/** Default concurrency for parallel execution */
|
|
116
118
|
concurrency?: number;
|
|
119
|
+
/**
|
|
120
|
+
* Storage configuration for persisting and loading run manifests
|
|
121
|
+
* Required for compare() method to load historical runs
|
|
122
|
+
* @since 0.3.2
|
|
123
|
+
*/
|
|
124
|
+
storage?: StorageConfig;
|
|
117
125
|
}
|
|
118
126
|
|
|
119
127
|
/**
|
|
@@ -140,6 +148,8 @@ export interface RunOptions {
|
|
|
140
148
|
retries?: number;
|
|
141
149
|
/** Redaction configuration */
|
|
142
150
|
redaction?: RedactionConfig;
|
|
151
|
+
/** Explicit executor for real tools; CLI scenarios remain fixture-only. */
|
|
152
|
+
toolExecutor?: ToolExecutor;
|
|
143
153
|
}
|
|
144
154
|
|
|
145
155
|
/**
|
|
@@ -257,3 +267,179 @@ export interface ArtemisKitEvents {
|
|
|
257
267
|
}
|
|
258
268
|
|
|
259
269
|
export type ArtemisKitEventName = keyof ArtemisKitEvents;
|
|
270
|
+
|
|
271
|
+
// ============================================================================
|
|
272
|
+
// Validation Types (v0.3.2+)
|
|
273
|
+
// ============================================================================
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Options for validating scenario files
|
|
277
|
+
*/
|
|
278
|
+
export interface ValidateOptions {
|
|
279
|
+
/**
|
|
280
|
+
* Scenario file path(s) or glob pattern
|
|
281
|
+
* Can be a single path, array of paths, or glob pattern
|
|
282
|
+
*/
|
|
283
|
+
scenario: string | string[];
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Strict mode - fail on warnings as well as errors
|
|
287
|
+
* @default false
|
|
288
|
+
*/
|
|
289
|
+
strict?: boolean;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Validation error details
|
|
294
|
+
*/
|
|
295
|
+
export interface ValidationError {
|
|
296
|
+
/** File path where error occurred */
|
|
297
|
+
file: string;
|
|
298
|
+
/** Error message */
|
|
299
|
+
message: string;
|
|
300
|
+
/** Line number if available */
|
|
301
|
+
line?: number;
|
|
302
|
+
/** Column number if available */
|
|
303
|
+
column?: number;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Validation warning details
|
|
308
|
+
*/
|
|
309
|
+
export interface ValidationWarning {
|
|
310
|
+
/** File path where warning occurred */
|
|
311
|
+
file: string;
|
|
312
|
+
/** Warning message */
|
|
313
|
+
message: string;
|
|
314
|
+
/** Line number if available */
|
|
315
|
+
line?: number;
|
|
316
|
+
/** Column number if available */
|
|
317
|
+
column?: number;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Per-scenario validation result
|
|
322
|
+
*/
|
|
323
|
+
export interface ScenarioValidation {
|
|
324
|
+
/** File path */
|
|
325
|
+
file: string;
|
|
326
|
+
/** Scenario name */
|
|
327
|
+
name: string;
|
|
328
|
+
/** Number of test cases */
|
|
329
|
+
caseCount: number;
|
|
330
|
+
/** Whether this scenario is valid */
|
|
331
|
+
valid: boolean;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Result from validate() method
|
|
336
|
+
*/
|
|
337
|
+
export interface ValidationResult {
|
|
338
|
+
/** Overall validation passed */
|
|
339
|
+
valid: boolean;
|
|
340
|
+
/** Individual scenario validation results */
|
|
341
|
+
scenarios: ScenarioValidation[];
|
|
342
|
+
/** Validation errors */
|
|
343
|
+
errors: ValidationError[];
|
|
344
|
+
/** Validation warnings (informational) */
|
|
345
|
+
warnings: ValidationWarning[];
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// ============================================================================
|
|
349
|
+
// Comparison Types (v0.3.2+)
|
|
350
|
+
// ============================================================================
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* Options for comparing test runs
|
|
354
|
+
*/
|
|
355
|
+
export interface CompareOptions {
|
|
356
|
+
/**
|
|
357
|
+
* Baseline run identifier
|
|
358
|
+
* Can be a run ID, 'latest', or a named baseline
|
|
359
|
+
*/
|
|
360
|
+
baseline: string;
|
|
361
|
+
|
|
362
|
+
/**
|
|
363
|
+
* Current run identifier
|
|
364
|
+
* Can be a run ID
|
|
365
|
+
*/
|
|
366
|
+
current: string;
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Regression threshold (0-1)
|
|
370
|
+
* A regression is detected if success rate drops by more than this amount
|
|
371
|
+
* @default 0.05 (5%)
|
|
372
|
+
*/
|
|
373
|
+
threshold?: number;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Summary of a run for comparison
|
|
378
|
+
*/
|
|
379
|
+
export interface RunSummary {
|
|
380
|
+
/** Run identifier */
|
|
381
|
+
runId: string;
|
|
382
|
+
/** Success rate (0-1) */
|
|
383
|
+
successRate: number;
|
|
384
|
+
/** Total number of test cases */
|
|
385
|
+
totalCases: number;
|
|
386
|
+
/** Number of passed cases */
|
|
387
|
+
passedCases: number;
|
|
388
|
+
/** Number of failed cases */
|
|
389
|
+
failedCases: number;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Comparison details between two runs
|
|
394
|
+
*/
|
|
395
|
+
export interface ComparisonDetails {
|
|
396
|
+
/** Change in success rate (current - baseline) */
|
|
397
|
+
successRateDelta: number;
|
|
398
|
+
/** Cases that passed in baseline but failed in current */
|
|
399
|
+
newFailures: Array<{
|
|
400
|
+
caseId: string;
|
|
401
|
+
caseName?: string;
|
|
402
|
+
baselineStatus: string;
|
|
403
|
+
currentStatus: string;
|
|
404
|
+
}>;
|
|
405
|
+
/** Cases that failed in baseline but passed in current */
|
|
406
|
+
newPasses: Array<{
|
|
407
|
+
caseId: string;
|
|
408
|
+
caseName?: string;
|
|
409
|
+
baselineStatus: string;
|
|
410
|
+
currentStatus: string;
|
|
411
|
+
}>;
|
|
412
|
+
/** Cases with unchanged status */
|
|
413
|
+
unchanged: Array<{
|
|
414
|
+
caseId: string;
|
|
415
|
+
status: string;
|
|
416
|
+
}>;
|
|
417
|
+
/** Cases that exist in current run but not in baseline (new test cases) */
|
|
418
|
+
addedCases: Array<{
|
|
419
|
+
caseId: string;
|
|
420
|
+
caseName?: string;
|
|
421
|
+
status: string;
|
|
422
|
+
}>;
|
|
423
|
+
/** Cases that exist in baseline but not in current run (removed test cases) */
|
|
424
|
+
removedCases: Array<{
|
|
425
|
+
caseId: string;
|
|
426
|
+
caseName?: string;
|
|
427
|
+
status: string;
|
|
428
|
+
}>;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Result from compare() method
|
|
433
|
+
*/
|
|
434
|
+
export interface CompareResult {
|
|
435
|
+
/** Baseline run summary */
|
|
436
|
+
baseline: RunSummary;
|
|
437
|
+
/** Current run summary */
|
|
438
|
+
current: RunSummary;
|
|
439
|
+
/** Detailed comparison */
|
|
440
|
+
comparison: ComparisonDetails;
|
|
441
|
+
/** Whether a regression was detected */
|
|
442
|
+
hasRegression: boolean;
|
|
443
|
+
/** Threshold used for regression detection */
|
|
444
|
+
threshold: number;
|
|
445
|
+
}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Utility types and helper functions
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
AdapterConfig,
|
|
8
|
+
AnyManifest,
|
|
9
|
+
CaseResult,
|
|
10
|
+
Expected,
|
|
11
|
+
RedTeamManifest,
|
|
12
|
+
RunManifest,
|
|
13
|
+
Scenario,
|
|
14
|
+
StressManifest,
|
|
15
|
+
} from '@artemiskit/core';
|
|
16
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
17
|
+
|
|
18
|
+
// ============================================================================
|
|
19
|
+
// Provider Name Types
|
|
20
|
+
// ============================================================================
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* All supported provider names as a union type
|
|
24
|
+
*/
|
|
25
|
+
export type ProviderName = AdapterConfig['provider'];
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* All supported expectation types
|
|
29
|
+
*/
|
|
30
|
+
export type ExpectationType = Expected['type'];
|
|
31
|
+
|
|
32
|
+
// ============================================================================
|
|
33
|
+
// Result Union Types
|
|
34
|
+
// ============================================================================
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Union of all SDK result types
|
|
38
|
+
*/
|
|
39
|
+
export type AnyResult = RunResult | RedTeamResult | StressResult;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Union of all manifest types (re-export from core for convenience)
|
|
43
|
+
*/
|
|
44
|
+
export type { AnyManifest };
|
|
45
|
+
|
|
46
|
+
// ============================================================================
|
|
47
|
+
// Type Extraction Utilities
|
|
48
|
+
// ============================================================================
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Extract case results from a RunResult
|
|
52
|
+
*/
|
|
53
|
+
export type ExtractRunCases = RunResult['cases'];
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Extract red team cases from a RedTeamResult
|
|
57
|
+
*/
|
|
58
|
+
export type ExtractRedTeamCases = RedTeamResult['manifest']['results'];
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Extract stress results from a StressResult
|
|
62
|
+
*/
|
|
63
|
+
export type ExtractStressResults = StressResult['manifest']['sample_results'];
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Extract the manifest type from any result type
|
|
67
|
+
*/
|
|
68
|
+
export type ExtractManifest<T extends AnyResult> = T extends RunResult
|
|
69
|
+
? RunManifest
|
|
70
|
+
: T extends RedTeamResult
|
|
71
|
+
? RedTeamManifest
|
|
72
|
+
: T extends StressResult
|
|
73
|
+
? StressManifest
|
|
74
|
+
: never;
|
|
75
|
+
|
|
76
|
+
// ============================================================================
|
|
77
|
+
// Partial Types for Construction
|
|
78
|
+
// ============================================================================
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Deep partial utility type - makes all nested properties optional
|
|
82
|
+
*/
|
|
83
|
+
export type DeepPartial<T> = T extends object
|
|
84
|
+
? {
|
|
85
|
+
[P in keyof T]?: DeepPartial<T[P]>;
|
|
86
|
+
}
|
|
87
|
+
: T;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Partial scenario for incremental construction
|
|
91
|
+
*/
|
|
92
|
+
export type PartialScenario = DeepPartial<Scenario>;
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Make specific fields required
|
|
96
|
+
*/
|
|
97
|
+
export type RequireFields<T, K extends keyof T> = T & Required<Pick<T, K>>;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Make specific fields optional
|
|
101
|
+
*/
|
|
102
|
+
export type OptionalFields<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
|
|
103
|
+
|
|
104
|
+
// ============================================================================
|
|
105
|
+
// Strict Configuration Types
|
|
106
|
+
// ============================================================================
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Strict version of AdapterConfig with all common fields required
|
|
110
|
+
*/
|
|
111
|
+
export type StrictAdapterConfig = RequireFields<AdapterConfig, 'provider' | 'apiKey'>;
|
|
112
|
+
|
|
113
|
+
// ============================================================================
|
|
114
|
+
// Type Guards
|
|
115
|
+
// ============================================================================
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Type guard for RunResult
|
|
119
|
+
*/
|
|
120
|
+
export function isRunResult(result: AnyResult): result is RunResult {
|
|
121
|
+
return 'cases' in result && Array.isArray(result.cases) && !('defenseRate' in result);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Type guard for RedTeamResult
|
|
126
|
+
*/
|
|
127
|
+
export function isRedTeamResult(result: AnyResult): result is RedTeamResult {
|
|
128
|
+
return 'defenseRate' in result && 'unsafeCount' in result;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Type guard for StressResult
|
|
133
|
+
*/
|
|
134
|
+
export function isStressResult(result: AnyResult): result is StressResult {
|
|
135
|
+
return 'rps' in result && 'p95LatencyMs' in result;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Type guard for RunManifest
|
|
140
|
+
*
|
|
141
|
+
* Uses structural field checks rather than absence-of-field logic for robustness.
|
|
142
|
+
* Checks for RunManifest-specific fields: 'metrics.success_rate' and 'cases' array.
|
|
143
|
+
*/
|
|
144
|
+
export function isRunManifestType(manifest: AnyManifest): manifest is RunManifest {
|
|
145
|
+
return (
|
|
146
|
+
'metrics' in manifest &&
|
|
147
|
+
manifest.metrics !== null &&
|
|
148
|
+
typeof manifest.metrics === 'object' &&
|
|
149
|
+
'success_rate' in manifest.metrics &&
|
|
150
|
+
'cases' in manifest &&
|
|
151
|
+
Array.isArray((manifest as RunManifest).cases)
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Type guard for RedTeamManifest
|
|
157
|
+
*/
|
|
158
|
+
export function isRedTeamManifestType(manifest: AnyManifest): manifest is RedTeamManifest {
|
|
159
|
+
return 'type' in manifest && manifest.type === 'redteam';
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Type guard for StressManifest
|
|
164
|
+
*/
|
|
165
|
+
export function isStressManifestType(manifest: AnyManifest): manifest is StressManifest {
|
|
166
|
+
return 'type' in manifest && manifest.type === 'stress';
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// ============================================================================
|
|
170
|
+
// Assertion Helpers
|
|
171
|
+
// ============================================================================
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Assert that a value is defined (non-null, non-undefined)
|
|
175
|
+
*/
|
|
176
|
+
export function assertDefined<T>(
|
|
177
|
+
value: T | null | undefined,
|
|
178
|
+
message = 'Value is null or undefined'
|
|
179
|
+
): asserts value is T {
|
|
180
|
+
if (value === null || value === undefined) {
|
|
181
|
+
throw new Error(message);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Assert that a condition is true
|
|
187
|
+
*/
|
|
188
|
+
export function assert(condition: boolean, message = 'Assertion failed'): asserts condition {
|
|
189
|
+
if (!condition) {
|
|
190
|
+
throw new Error(message);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ============================================================================
|
|
195
|
+
// Result Analysis Helpers
|
|
196
|
+
// ============================================================================
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Get failed cases from a run result
|
|
200
|
+
*/
|
|
201
|
+
export function getFailedCases(result: RunResult): CaseResult[] {
|
|
202
|
+
return result.cases.filter((c) => !c.ok);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Get passed cases from a run result
|
|
207
|
+
*/
|
|
208
|
+
export function getPassedCases(result: RunResult): CaseResult[] {
|
|
209
|
+
return result.cases.filter((c) => c.ok);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Get cases by tag from a run result
|
|
214
|
+
*/
|
|
215
|
+
export function getCasesByTag(result: RunResult, tag: string): CaseResult[] {
|
|
216
|
+
return result.cases.filter((c) => c.tags.includes(tag));
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Calculate success rate from a run result
|
|
221
|
+
*/
|
|
222
|
+
export function calculateSuccessRate(result: RunResult): number {
|
|
223
|
+
if (result.cases.length === 0) return 0;
|
|
224
|
+
return result.cases.filter((c) => c.ok).length / result.cases.length;
|
|
225
|
+
}
|