@artemiskit/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +229 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +352 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +354 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +53 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +44918 -28644
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/index.js +13 -59
  39. package/dist/matchers/jest.d.ts +46 -0
  40. package/dist/matchers/jest.d.ts.map +1 -0
  41. package/dist/matchers/jest.js +0 -46
  42. package/dist/matchers/vitest.d.ts +24 -0
  43. package/dist/matchers/vitest.d.ts.map +1 -0
  44. package/dist/matchers/vitest.js +0 -46
  45. package/dist/types-only.d.ts +31 -0
  46. package/dist/types-only.d.ts.map +1 -0
  47. package/dist/types-only.js +1 -0
  48. package/dist/types.d.ts +379 -0
  49. package/dist/types.d.ts.map +1 -0
  50. package/dist/utils/index.d.ts +112 -0
  51. package/dist/utils/index.d.ts.map +1 -0
  52. package/dist/utils/index.js +58 -0
  53. package/package.json +19 -4
  54. package/src/__tests__/builders.test.ts +450 -0
  55. package/src/__tests__/circuit-breaker.test.ts +704 -0
  56. package/src/__tests__/compare.test.ts +469 -0
  57. package/src/__tests__/contracts.test.ts +354 -0
  58. package/src/__tests__/guardian-modes.test.ts +215 -0
  59. package/src/__tests__/guardrails.test.ts +404 -0
  60. package/src/__tests__/semantic-validator.test.ts +421 -0
  61. package/src/__tests__/utils.test.ts +324 -0
  62. package/src/__tests__/validate.test.ts +310 -0
  63. package/src/artemiskit.ts +479 -5
  64. package/src/builders/index.ts +700 -0
  65. package/src/contracts/index.ts +357 -0
  66. package/src/guardian/circuit-breaker.ts +74 -28
  67. package/src/guardian/guardian.ts +307 -20
  68. package/src/guardian/guardrails.ts +143 -1
  69. package/src/guardian/index.ts +35 -1
  70. package/src/guardian/interceptor.ts +87 -11
  71. package/src/guardian/semantic-validator.ts +399 -0
  72. package/src/guardian/types.ts +413 -2
  73. package/src/index.ts +258 -14
  74. package/src/types-only.ts +276 -0
  75. package/src/types.ts +186 -0
  76. package/src/utils/index.ts +225 -0
  77. package/tsconfig.json +2 -1
  78. package/adapters/openai/dist/index.js +0 -5625
@@ -0,0 +1,379 @@
1
+ /**
2
+ * @artemiskit/sdk
3
+ * Type definitions for the programmatic SDK
4
+ */
5
+ import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult, ToolExecutor } from '@artemiskit/core';
6
+ import type { RedactionConfig } from '@artemiskit/core';
7
+ import type { Scenario } from '@artemiskit/core';
8
+ /**
9
+ * Event emitted when a test case starts
10
+ */
11
+ export interface CaseStartEvent {
12
+ caseId: string;
13
+ caseName?: string;
14
+ index: number;
15
+ total: number;
16
+ }
17
+ /**
18
+ * Event emitted when a test case completes
19
+ */
20
+ export interface CaseCompleteEvent {
21
+ result: CaseResult;
22
+ index: number;
23
+ total: number;
24
+ }
25
+ /**
26
+ * Event emitted for progress updates
27
+ */
28
+ export interface ProgressEvent {
29
+ message: string;
30
+ phase: 'setup' | 'running' | 'teardown';
31
+ progress?: number;
32
+ }
33
+ /**
34
+ * Event emitted when a red team mutation starts
35
+ */
36
+ export interface RedTeamMutationStartEvent {
37
+ mutation: string;
38
+ caseId: string;
39
+ index: number;
40
+ total: number;
41
+ }
42
+ /**
43
+ * Event emitted when a red team mutation completes
44
+ */
45
+ export interface RedTeamMutationCompleteEvent {
46
+ mutation: string;
47
+ caseId: string;
48
+ status: 'safe' | 'unsafe' | 'blocked' | 'error';
49
+ severity: 'none' | 'low' | 'medium' | 'high' | 'critical';
50
+ index: number;
51
+ total: number;
52
+ }
53
+ /**
54
+ * Event emitted for stress test request completion
55
+ */
56
+ export interface StressRequestCompleteEvent {
57
+ result: StressRequestResult;
58
+ index: number;
59
+ total: number;
60
+ currentRPS: number;
61
+ }
62
+ export type CaseStartHandler = (event: CaseStartEvent) => void;
63
+ export type CaseCompleteHandler = (event: CaseCompleteEvent) => void;
64
+ export type ProgressHandler = (event: ProgressEvent) => void;
65
+ export type RedTeamMutationStartHandler = (event: RedTeamMutationStartEvent) => void;
66
+ export type RedTeamMutationCompleteHandler = (event: RedTeamMutationCompleteEvent) => void;
67
+ export type StressRequestCompleteHandler = (event: StressRequestCompleteEvent) => void;
68
+ /**
69
+ * SDK-level configuration options
70
+ */
71
+ export interface ArtemisKitConfig {
72
+ /** Project name for manifest grouping */
73
+ project?: string;
74
+ /** Default provider to use */
75
+ provider?: AdapterConfig['provider'];
76
+ /** Default model to use */
77
+ model?: string;
78
+ /** Provider-specific configuration */
79
+ providerConfig?: Partial<AdapterConfig>;
80
+ /** Default redaction settings */
81
+ redaction?: RedactionConfig;
82
+ /** Default timeout per case in milliseconds */
83
+ timeout?: number;
84
+ /** Default number of retries per case */
85
+ retries?: number;
86
+ /** Default concurrency for parallel execution */
87
+ concurrency?: number;
88
+ /**
89
+ * Storage configuration for persisting and loading run manifests
90
+ * Required for compare() method to load historical runs
91
+ * @since 0.3.2
92
+ */
93
+ storage?: StorageConfig;
94
+ }
95
+ /**
96
+ * Options for running test scenarios
97
+ */
98
+ export interface RunOptions {
99
+ /** Path to scenario file or inline Scenario object */
100
+ scenario: string | Scenario;
101
+ /** Override provider */
102
+ provider?: AdapterConfig['provider'];
103
+ /** Override model */
104
+ model?: string;
105
+ /** Provider-specific configuration override */
106
+ providerConfig?: Partial<AdapterConfig>;
107
+ /** Pre-configured model client (skips adapter creation) */
108
+ client?: ModelClient;
109
+ /** Filter test cases by tags */
110
+ tags?: string[];
111
+ /** Number of concurrent requests */
112
+ concurrency?: number;
113
+ /** Timeout per case in milliseconds */
114
+ timeout?: number;
115
+ /** Number of retries per case */
116
+ retries?: number;
117
+ /** Redaction configuration */
118
+ redaction?: RedactionConfig;
119
+ /** Explicit executor for real tools; CLI scenarios remain fixture-only. */
120
+ toolExecutor?: ToolExecutor;
121
+ }
122
+ /**
123
+ * Options for red team testing
124
+ */
125
+ export interface RedTeamOptions {
126
+ /** Path to scenario file or inline Scenario object */
127
+ scenario: string | Scenario;
128
+ /** Override provider */
129
+ provider?: AdapterConfig['provider'];
130
+ /** Override model */
131
+ model?: string;
132
+ /** Provider-specific configuration override */
133
+ providerConfig?: Partial<AdapterConfig>;
134
+ /** Pre-configured model client (skips adapter creation) */
135
+ client?: ModelClient;
136
+ /** Mutations to apply (default: all) */
137
+ mutations?: string[];
138
+ /** Number of mutations per case */
139
+ countPerCase?: number;
140
+ /** Filter test cases by tags */
141
+ tags?: string[];
142
+ /** Timeout per case in milliseconds */
143
+ timeout?: number;
144
+ /** Redaction configuration */
145
+ redaction?: RedactionConfig;
146
+ }
147
+ /**
148
+ * Options for stress testing
149
+ */
150
+ export interface StressOptions {
151
+ /** Path to scenario file or inline Scenario object */
152
+ scenario: string | Scenario;
153
+ /** Override provider */
154
+ provider?: AdapterConfig['provider'];
155
+ /** Override model */
156
+ model?: string;
157
+ /** Provider-specific configuration override */
158
+ providerConfig?: Partial<AdapterConfig>;
159
+ /** Pre-configured model client (skips adapter creation) */
160
+ client?: ModelClient;
161
+ /** Number of concurrent requests */
162
+ concurrency?: number;
163
+ /** Test duration in seconds */
164
+ duration?: number;
165
+ /** Ramp-up period in seconds */
166
+ rampUp?: number;
167
+ /** Maximum number of requests (optional limit) */
168
+ maxRequests?: number;
169
+ /** Redaction configuration */
170
+ redaction?: RedactionConfig;
171
+ }
172
+ /**
173
+ * Result from a test scenario run
174
+ */
175
+ export interface RunResult {
176
+ /** The generated manifest */
177
+ manifest: RunManifest;
178
+ /** Individual case results */
179
+ cases: CaseResult[];
180
+ /** Whether all cases passed */
181
+ success: boolean;
182
+ }
183
+ /**
184
+ * Result from a red team test run
185
+ */
186
+ export interface RedTeamResult {
187
+ /** The generated manifest */
188
+ manifest: RedTeamManifest;
189
+ /** Whether the defense rate is acceptable (>= threshold) */
190
+ success: boolean;
191
+ /** Defense rate (0-1) */
192
+ defenseRate: number;
193
+ /** Count of unsafe responses */
194
+ unsafeCount: number;
195
+ }
196
+ /**
197
+ * Result from a stress test run
198
+ */
199
+ export interface StressResult {
200
+ /** The generated manifest */
201
+ manifest: StressManifest;
202
+ /** Whether the test passed success rate threshold */
203
+ success: boolean;
204
+ /** Success rate (0-1) */
205
+ successRate: number;
206
+ /** Requests per second achieved */
207
+ rps: number;
208
+ /** P95 latency in milliseconds */
209
+ p95LatencyMs: number;
210
+ }
211
+ /**
212
+ * Event types supported by ArtemisKit
213
+ */
214
+ export interface ArtemisKitEvents {
215
+ caseStart: CaseStartEvent;
216
+ caseComplete: CaseCompleteEvent;
217
+ progress: ProgressEvent;
218
+ redteamMutationStart: RedTeamMutationStartEvent;
219
+ redteamMutationComplete: RedTeamMutationCompleteEvent;
220
+ stressRequestComplete: StressRequestCompleteEvent;
221
+ }
222
+ export type ArtemisKitEventName = keyof ArtemisKitEvents;
223
+ /**
224
+ * Options for validating scenario files
225
+ */
226
+ export interface ValidateOptions {
227
+ /**
228
+ * Scenario file path(s) or glob pattern
229
+ * Can be a single path, array of paths, or glob pattern
230
+ */
231
+ scenario: string | string[];
232
+ /**
233
+ * Strict mode - fail on warnings as well as errors
234
+ * @default false
235
+ */
236
+ strict?: boolean;
237
+ }
238
+ /**
239
+ * Validation error details
240
+ */
241
+ export interface ValidationError {
242
+ /** File path where error occurred */
243
+ file: string;
244
+ /** Error message */
245
+ message: string;
246
+ /** Line number if available */
247
+ line?: number;
248
+ /** Column number if available */
249
+ column?: number;
250
+ }
251
+ /**
252
+ * Validation warning details
253
+ */
254
+ export interface ValidationWarning {
255
+ /** File path where warning occurred */
256
+ file: string;
257
+ /** Warning message */
258
+ message: string;
259
+ /** Line number if available */
260
+ line?: number;
261
+ /** Column number if available */
262
+ column?: number;
263
+ }
264
+ /**
265
+ * Per-scenario validation result
266
+ */
267
+ export interface ScenarioValidation {
268
+ /** File path */
269
+ file: string;
270
+ /** Scenario name */
271
+ name: string;
272
+ /** Number of test cases */
273
+ caseCount: number;
274
+ /** Whether this scenario is valid */
275
+ valid: boolean;
276
+ }
277
+ /**
278
+ * Result from validate() method
279
+ */
280
+ export interface ValidationResult {
281
+ /** Overall validation passed */
282
+ valid: boolean;
283
+ /** Individual scenario validation results */
284
+ scenarios: ScenarioValidation[];
285
+ /** Validation errors */
286
+ errors: ValidationError[];
287
+ /** Validation warnings (informational) */
288
+ warnings: ValidationWarning[];
289
+ }
290
+ /**
291
+ * Options for comparing test runs
292
+ */
293
+ export interface CompareOptions {
294
+ /**
295
+ * Baseline run identifier
296
+ * Can be a run ID, 'latest', or a named baseline
297
+ */
298
+ baseline: string;
299
+ /**
300
+ * Current run identifier
301
+ * Can be a run ID
302
+ */
303
+ current: string;
304
+ /**
305
+ * Regression threshold (0-1)
306
+ * A regression is detected if success rate drops by more than this amount
307
+ * @default 0.05 (5%)
308
+ */
309
+ threshold?: number;
310
+ }
311
+ /**
312
+ * Summary of a run for comparison
313
+ */
314
+ export interface RunSummary {
315
+ /** Run identifier */
316
+ runId: string;
317
+ /** Success rate (0-1) */
318
+ successRate: number;
319
+ /** Total number of test cases */
320
+ totalCases: number;
321
+ /** Number of passed cases */
322
+ passedCases: number;
323
+ /** Number of failed cases */
324
+ failedCases: number;
325
+ }
326
+ /**
327
+ * Comparison details between two runs
328
+ */
329
+ export interface ComparisonDetails {
330
+ /** Change in success rate (current - baseline) */
331
+ successRateDelta: number;
332
+ /** Cases that passed in baseline but failed in current */
333
+ newFailures: Array<{
334
+ caseId: string;
335
+ caseName?: string;
336
+ baselineStatus: string;
337
+ currentStatus: string;
338
+ }>;
339
+ /** Cases that failed in baseline but passed in current */
340
+ newPasses: Array<{
341
+ caseId: string;
342
+ caseName?: string;
343
+ baselineStatus: string;
344
+ currentStatus: string;
345
+ }>;
346
+ /** Cases with unchanged status */
347
+ unchanged: Array<{
348
+ caseId: string;
349
+ status: string;
350
+ }>;
351
+ /** Cases that exist in current run but not in baseline (new test cases) */
352
+ addedCases: Array<{
353
+ caseId: string;
354
+ caseName?: string;
355
+ status: string;
356
+ }>;
357
+ /** Cases that exist in baseline but not in current run (removed test cases) */
358
+ removedCases: Array<{
359
+ caseId: string;
360
+ caseName?: string;
361
+ status: string;
362
+ }>;
363
+ }
364
+ /**
365
+ * Result from compare() method
366
+ */
367
+ export interface CompareResult {
368
+ /** Baseline run summary */
369
+ baseline: RunSummary;
370
+ /** Current run summary */
371
+ current: RunSummary;
372
+ /** Detailed comparison */
373
+ comparison: ComparisonDetails;
374
+ /** Whether a regression was detected */
375
+ hasRegression: boolean;
376
+ /** Threshold used for regression detection */
377
+ threshold: number;
378
+ }
379
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,YAAY,EACb,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,wCAAwC;IACxC,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
@@ -0,0 +1,112 @@
1
+ /**
2
+ * @artemiskit/sdk
3
+ * Utility types and helper functions
4
+ */
5
+ import type { AdapterConfig, AnyManifest, CaseResult, Expected, RedTeamManifest, RunManifest, Scenario, StressManifest } from '@artemiskit/core';
6
+ import type { RedTeamResult, RunResult, StressResult } from '../types';
7
+ /**
8
+ * All supported provider names as a union type
9
+ */
10
+ export type ProviderName = AdapterConfig['provider'];
11
+ /**
12
+ * All supported expectation types
13
+ */
14
+ export type ExpectationType = Expected['type'];
15
+ /**
16
+ * Union of all SDK result types
17
+ */
18
+ export type AnyResult = RunResult | RedTeamResult | StressResult;
19
+ /**
20
+ * Union of all manifest types (re-export from core for convenience)
21
+ */
22
+ export type { AnyManifest };
23
+ /**
24
+ * Extract case results from a RunResult
25
+ */
26
+ export type ExtractRunCases = RunResult['cases'];
27
+ /**
28
+ * Extract red team cases from a RedTeamResult
29
+ */
30
+ export type ExtractRedTeamCases = RedTeamResult['manifest']['results'];
31
+ /**
32
+ * Extract stress results from a StressResult
33
+ */
34
+ export type ExtractStressResults = StressResult['manifest']['sample_results'];
35
+ /**
36
+ * Extract the manifest type from any result type
37
+ */
38
+ export type ExtractManifest<T extends AnyResult> = T extends RunResult ? RunManifest : T extends RedTeamResult ? RedTeamManifest : T extends StressResult ? StressManifest : never;
39
+ /**
40
+ * Deep partial utility type - makes all nested properties optional
41
+ */
42
+ export type DeepPartial<T> = T extends object ? {
43
+ [P in keyof T]?: DeepPartial<T[P]>;
44
+ } : T;
45
+ /**
46
+ * Partial scenario for incremental construction
47
+ */
48
+ export type PartialScenario = DeepPartial<Scenario>;
49
+ /**
50
+ * Make specific fields required
51
+ */
52
+ export type RequireFields<T, K extends keyof T> = T & Required<Pick<T, K>>;
53
+ /**
54
+ * Make specific fields optional
55
+ */
56
+ export type OptionalFields<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
57
+ /**
58
+ * Strict version of AdapterConfig with all common fields required
59
+ */
60
+ export type StrictAdapterConfig = RequireFields<AdapterConfig, 'provider' | 'apiKey'>;
61
+ /**
62
+ * Type guard for RunResult
63
+ */
64
+ export declare function isRunResult(result: AnyResult): result is RunResult;
65
+ /**
66
+ * Type guard for RedTeamResult
67
+ */
68
+ export declare function isRedTeamResult(result: AnyResult): result is RedTeamResult;
69
+ /**
70
+ * Type guard for StressResult
71
+ */
72
+ export declare function isStressResult(result: AnyResult): result is StressResult;
73
+ /**
74
+ * Type guard for RunManifest
75
+ *
76
+ * Uses structural field checks rather than absence-of-field logic for robustness.
77
+ * Checks for RunManifest-specific fields: 'metrics.success_rate' and 'cases' array.
78
+ */
79
+ export declare function isRunManifestType(manifest: AnyManifest): manifest is RunManifest;
80
+ /**
81
+ * Type guard for RedTeamManifest
82
+ */
83
+ export declare function isRedTeamManifestType(manifest: AnyManifest): manifest is RedTeamManifest;
84
+ /**
85
+ * Type guard for StressManifest
86
+ */
87
+ export declare function isStressManifestType(manifest: AnyManifest): manifest is StressManifest;
88
+ /**
89
+ * Assert that a value is defined (non-null, non-undefined)
90
+ */
91
+ export declare function assertDefined<T>(value: T | null | undefined, message?: string): asserts value is T;
92
+ /**
93
+ * Assert that a condition is true
94
+ */
95
+ export declare function assert(condition: boolean, message?: string): asserts condition;
96
+ /**
97
+ * Get failed cases from a run result
98
+ */
99
+ export declare function getFailedCases(result: RunResult): CaseResult[];
100
+ /**
101
+ * Get passed cases from a run result
102
+ */
103
+ export declare function getPassedCases(result: RunResult): CaseResult[];
104
+ /**
105
+ * Get cases by tag from a run result
106
+ */
107
+ export declare function getCasesByTag(result: RunResult, tag: string): CaseResult[];
108
+ /**
109
+ * Calculate success rate from a run result
110
+ */
111
+ export declare function calculateSuccessRate(result: RunResult): number;
112
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,WAAW,EACX,UAAU,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,QAAQ,EACR,cAAc,EACf,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAMvE;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAErD;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;AAM/C;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,aAAa,GAAG,YAAY,CAAC;AAEjE;;GAEG;AACH,YAAY,EAAE,WAAW,EAAE,CAAC;AAM5B;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;AAEjD;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,gBAAgB,CAAC,CAAC;AAE9E;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,SAAS,IAAI,CAAC,SAAS,SAAS,GAClE,WAAW,GACX,CAAC,SAAS,aAAa,GACrB,eAAe,GACf,CAAC,SAAS,YAAY,GACpB,cAAc,GACd,KAAK,CAAC;AAMd;;GAEG;AACH,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,GACzC;KACG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CACnC,GACD,CAAC,CAAC;AAEN;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;AAEpD;;GAEG;AACH,MAAM,MAAM,aAAa,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAE3E;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAMpF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,aAAa,CAAC,aAAa,EAAE,UAAU,GAAG,QAAQ,CAAC,CAAC;AAMtF;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,SAAS,CAElE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,aAAa,CAE1E;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,IAAI,YAAY,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,WAAW,CAShF;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,eAAe,CAExF;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,IAAI,cAAc,CAEtF;AAMD;;GAEG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAC7B,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,SAAS,EAC3B,OAAO,SAA+B,GACrC,OAAO,CAAC,KAAK,IAAI,CAAC,CAIpB;AAED;;GAEG;AACH,wBAAgB,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,SAAqB,GAAG,OAAO,CAAC,SAAS,CAI1F;AAMD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,EAAE,CAE9D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,EAAE,CAE9D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,UAAU,EAAE,CAE1E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CAG9D"}
@@ -0,0 +1,58 @@
1
+ // @bun
2
+ // src/utils/index.ts
3
+ function isRunResult(result) {
4
+ return "cases" in result && Array.isArray(result.cases) && !("defenseRate" in result);
5
+ }
6
+ function isRedTeamResult(result) {
7
+ return "defenseRate" in result && "unsafeCount" in result;
8
+ }
9
+ function isStressResult(result) {
10
+ return "rps" in result && "p95LatencyMs" in result;
11
+ }
12
+ function isRunManifestType(manifest) {
13
+ return "metrics" in manifest && manifest.metrics !== null && typeof manifest.metrics === "object" && "success_rate" in manifest.metrics && "cases" in manifest && Array.isArray(manifest.cases);
14
+ }
15
+ function isRedTeamManifestType(manifest) {
16
+ return "type" in manifest && manifest.type === "redteam";
17
+ }
18
+ function isStressManifestType(manifest) {
19
+ return "type" in manifest && manifest.type === "stress";
20
+ }
21
+ function assertDefined(value, message = "Value is null or undefined") {
22
+ if (value === null || value === undefined) {
23
+ throw new Error(message);
24
+ }
25
+ }
26
+ function assert(condition, message = "Assertion failed") {
27
+ if (!condition) {
28
+ throw new Error(message);
29
+ }
30
+ }
31
+ function getFailedCases(result) {
32
+ return result.cases.filter((c) => !c.ok);
33
+ }
34
+ function getPassedCases(result) {
35
+ return result.cases.filter((c) => c.ok);
36
+ }
37
+ function getCasesByTag(result, tag) {
38
+ return result.cases.filter((c) => c.tags.includes(tag));
39
+ }
40
+ function calculateSuccessRate(result) {
41
+ if (result.cases.length === 0)
42
+ return 0;
43
+ return result.cases.filter((c) => c.ok).length / result.cases.length;
44
+ }
45
+ export {
46
+ assert,
47
+ assertDefined,
48
+ calculateSuccessRate,
49
+ getCasesByTag,
50
+ getFailedCases,
51
+ getPassedCases,
52
+ isRedTeamManifestType,
53
+ isRedTeamResult,
54
+ isRunManifestType,
55
+ isRunResult,
56
+ isStressManifestType,
57
+ isStressResult
58
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@artemiskit/sdk",
3
- "version": "0.3.1",
3
+ "version": "0.4.0",
4
4
  "description": "Programmatic SDK for ArtemisKit LLM evaluation toolkit - integrate into your Node.js apps",
5
5
  "type": "module",
6
6
  "license": "Apache-2.0",
@@ -46,18 +46,33 @@
46
46
  "./jest": {
47
47
  "import": "./dist/matchers/jest.js",
48
48
  "types": "./dist/matchers/jest.d.ts"
49
+ },
50
+ "./builders": {
51
+ "import": "./dist/builders/index.js",
52
+ "types": "./dist/builders/index.d.ts"
53
+ },
54
+ "./contracts": {
55
+ "import": "./dist/contracts/index.js",
56
+ "types": "./dist/contracts/index.d.ts"
57
+ },
58
+ "./utils": {
59
+ "import": "./dist/utils/index.js",
60
+ "types": "./dist/utils/index.d.ts"
61
+ },
62
+ "./types": {
63
+ "types": "./dist/types-only.d.ts"
49
64
  }
50
65
  },
51
66
  "scripts": {
52
- "build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts --outdir ./dist --target bun",
67
+ "build": "tsc --emitDeclarationOnly && bun build ./src/index.ts ./src/matchers/index.ts ./src/matchers/vitest.ts ./src/matchers/jest.ts ./src/builders/index.ts ./src/contracts/index.ts ./src/utils/index.ts ./src/types-only.ts --outdir ./dist --target bun",
53
68
  "typecheck": "tsc --noEmit",
54
69
  "clean": "rm -rf dist",
55
70
  "test": "bun test",
56
71
  "test:coverage": "bun test --coverage"
57
72
  },
58
73
  "dependencies": {
59
- "@artemiskit/core": "0.3.0",
60
- "@artemiskit/redteam": "0.3.0"
74
+ "@artemiskit/core": "0.4.0",
75
+ "@artemiskit/redteam": "0.3.1"
61
76
  },
62
77
  "devDependencies": {
63
78
  "@types/bun": "^1.1.0",