@artemiskit/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +213 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +350 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +398 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +99 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +24847 -13653
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/jest.d.ts +46 -0
  39. package/dist/matchers/jest.d.ts.map +1 -0
  40. package/dist/matchers/vitest.d.ts +24 -0
  41. package/dist/matchers/vitest.d.ts.map +1 -0
  42. package/dist/types-only.d.ts +31 -0
  43. package/dist/types-only.d.ts.map +1 -0
  44. package/dist/types-only.js +1 -0
  45. package/dist/types.d.ts +377 -0
  46. package/dist/types.d.ts.map +1 -0
  47. package/dist/utils/index.d.ts +112 -0
  48. package/dist/utils/index.d.ts.map +1 -0
  49. package/dist/utils/index.js +104 -0
  50. package/package.json +17 -2
  51. package/src/__tests__/builders.test.ts +445 -0
  52. package/src/__tests__/circuit-breaker.test.ts +704 -0
  53. package/src/__tests__/compare.test.ts +469 -0
  54. package/src/__tests__/contracts.test.ts +354 -0
  55. package/src/__tests__/guardian-modes.test.ts +215 -0
  56. package/src/__tests__/guardrails.test.ts +404 -0
  57. package/src/__tests__/semantic-validator.test.ts +421 -0
  58. package/src/__tests__/utils.test.ts +324 -0
  59. package/src/__tests__/validate.test.ts +307 -0
  60. package/src/artemiskit.ts +478 -5
  61. package/src/builders/index.ts +698 -0
  62. package/src/contracts/index.ts +357 -0
  63. package/src/guardian/circuit-breaker.ts +74 -28
  64. package/src/guardian/guardian.ts +307 -20
  65. package/src/guardian/guardrails.ts +143 -1
  66. package/src/guardian/index.ts +35 -1
  67. package/src/guardian/interceptor.ts +87 -11
  68. package/src/guardian/semantic-validator.ts +399 -0
  69. package/src/guardian/types.ts +413 -2
  70. package/src/index.ts +258 -14
  71. package/src/types-only.ts +276 -0
  72. package/src/types.ts +183 -0
  73. package/src/utils/index.ts +225 -0
  74. package/tsconfig.json +2 -1
package/src/types.ts CHANGED
@@ -9,6 +9,7 @@ import type {
9
9
  ModelClient,
10
10
  RedTeamManifest,
11
11
  RunManifest,
12
+ StorageConfig,
12
13
  StressManifest,
13
14
  StressRequestResult,
14
15
  } from '@artemiskit/core';
@@ -114,6 +115,12 @@ export interface ArtemisKitConfig {
114
115
  retries?: number;
115
116
  /** Default concurrency for parallel execution */
116
117
  concurrency?: number;
118
+ /**
119
+ * Storage configuration for persisting and loading run manifests
120
+ * Required for compare() method to load historical runs
121
+ * @since 0.3.2
122
+ */
123
+ storage?: StorageConfig;
117
124
  }
118
125
 
119
126
  /**
@@ -257,3 +264,179 @@ export interface ArtemisKitEvents {
257
264
  }
258
265
 
259
266
  export type ArtemisKitEventName = keyof ArtemisKitEvents;
267
+
268
+ // ============================================================================
269
+ // Validation Types (v0.3.2+)
270
+ // ============================================================================
271
+
272
+ /**
273
+ * Options for validating scenario files
274
+ */
275
+ export interface ValidateOptions {
276
+ /**
277
+ * Scenario file path(s) or glob pattern
278
+ * Can be a single path, array of paths, or glob pattern
279
+ */
280
+ scenario: string | string[];
281
+
282
+ /**
283
+ * Strict mode - fail on warnings as well as errors
284
+ * @default false
285
+ */
286
+ strict?: boolean;
287
+ }
288
+
289
+ /**
290
+ * Validation error details
291
+ */
292
+ export interface ValidationError {
293
+ /** File path where error occurred */
294
+ file: string;
295
+ /** Error message */
296
+ message: string;
297
+ /** Line number if available */
298
+ line?: number;
299
+ /** Column number if available */
300
+ column?: number;
301
+ }
302
+
303
+ /**
304
+ * Validation warning details
305
+ */
306
+ export interface ValidationWarning {
307
+ /** File path where warning occurred */
308
+ file: string;
309
+ /** Warning message */
310
+ message: string;
311
+ /** Line number if available */
312
+ line?: number;
313
+ /** Column number if available */
314
+ column?: number;
315
+ }
316
+
317
+ /**
318
+ * Per-scenario validation result
319
+ */
320
+ export interface ScenarioValidation {
321
+ /** File path */
322
+ file: string;
323
+ /** Scenario name */
324
+ name: string;
325
+ /** Number of test cases */
326
+ caseCount: number;
327
+ /** Whether this scenario is valid */
328
+ valid: boolean;
329
+ }
330
+
331
+ /**
332
+ * Result from validate() method
333
+ */
334
+ export interface ValidationResult {
335
+ /** Overall validation passed */
336
+ valid: boolean;
337
+ /** Individual scenario validation results */
338
+ scenarios: ScenarioValidation[];
339
+ /** Validation errors */
340
+ errors: ValidationError[];
341
+ /** Validation warnings (informational) */
342
+ warnings: ValidationWarning[];
343
+ }
344
+
345
+ // ============================================================================
346
+ // Comparison Types (v0.3.2+)
347
+ // ============================================================================
348
+
349
+ /**
350
+ * Options for comparing test runs
351
+ */
352
+ export interface CompareOptions {
353
+ /**
354
+ * Baseline run identifier
355
+ * Can be a run ID, 'latest', or a named baseline
356
+ */
357
+ baseline: string;
358
+
359
+ /**
360
+ * Current run identifier
361
+ * Can be a run ID
362
+ */
363
+ current: string;
364
+
365
+ /**
366
+ * Regression threshold (0-1)
367
+ * A regression is detected if success rate drops by more than this amount
368
+ * @default 0.05 (5%)
369
+ */
370
+ threshold?: number;
371
+ }
372
+
373
+ /**
374
+ * Summary of a run for comparison
375
+ */
376
+ export interface RunSummary {
377
+ /** Run identifier */
378
+ runId: string;
379
+ /** Success rate (0-1) */
380
+ successRate: number;
381
+ /** Total number of test cases */
382
+ totalCases: number;
383
+ /** Number of passed cases */
384
+ passedCases: number;
385
+ /** Number of failed cases */
386
+ failedCases: number;
387
+ }
388
+
389
+ /**
390
+ * Comparison details between two runs
391
+ */
392
+ export interface ComparisonDetails {
393
+ /** Change in success rate (current - baseline) */
394
+ successRateDelta: number;
395
+ /** Cases that passed in baseline but failed in current */
396
+ newFailures: Array<{
397
+ caseId: string;
398
+ caseName?: string;
399
+ baselineStatus: string;
400
+ currentStatus: string;
401
+ }>;
402
+ /** Cases that failed in baseline but passed in current */
403
+ newPasses: Array<{
404
+ caseId: string;
405
+ caseName?: string;
406
+ baselineStatus: string;
407
+ currentStatus: string;
408
+ }>;
409
+ /** Cases with unchanged status */
410
+ unchanged: Array<{
411
+ caseId: string;
412
+ status: string;
413
+ }>;
414
+ /** Cases that exist in current run but not in baseline (new test cases) */
415
+ addedCases: Array<{
416
+ caseId: string;
417
+ caseName?: string;
418
+ status: string;
419
+ }>;
420
+ /** Cases that exist in baseline but not in current run (removed test cases) */
421
+ removedCases: Array<{
422
+ caseId: string;
423
+ caseName?: string;
424
+ status: string;
425
+ }>;
426
+ }
427
+
428
+ /**
429
+ * Result from compare() method
430
+ */
431
+ export interface CompareResult {
432
+ /** Baseline run summary */
433
+ baseline: RunSummary;
434
+ /** Current run summary */
435
+ current: RunSummary;
436
+ /** Detailed comparison */
437
+ comparison: ComparisonDetails;
438
+ /** Whether a regression was detected */
439
+ hasRegression: boolean;
440
+ /** Threshold used for regression detection */
441
+ threshold: number;
442
+ }
@@ -0,0 +1,225 @@
1
+ /**
2
+ * @artemiskit/sdk
3
+ * Utility types and helper functions
4
+ */
5
+
6
+ import type {
7
+ AdapterConfig,
8
+ AnyManifest,
9
+ CaseResult,
10
+ Expected,
11
+ RedTeamManifest,
12
+ RunManifest,
13
+ Scenario,
14
+ StressManifest,
15
+ } from '@artemiskit/core';
16
+ import type { RedTeamResult, RunResult, StressResult } from '../types';
17
+
18
+ // ============================================================================
19
+ // Provider Name Types
20
+ // ============================================================================
21
+
22
+ /**
23
+ * All supported provider names as a union type
24
+ */
25
+ export type ProviderName = AdapterConfig['provider'];
26
+
27
+ /**
28
+ * All supported expectation types
29
+ */
30
+ export type ExpectationType = Expected['type'];
31
+
32
+ // ============================================================================
33
+ // Result Union Types
34
+ // ============================================================================
35
+
36
+ /**
37
+ * Union of all SDK result types
38
+ */
39
+ export type AnyResult = RunResult | RedTeamResult | StressResult;
40
+
41
+ /**
42
+ * Union of all manifest types (re-export from core for convenience)
43
+ */
44
+ export type { AnyManifest };
45
+
46
+ // ============================================================================
47
+ // Type Extraction Utilities
48
+ // ============================================================================
49
+
50
+ /**
51
+ * Extract case results from a RunResult
52
+ */
53
+ export type ExtractRunCases = RunResult['cases'];
54
+
55
+ /**
56
+ * Extract red team cases from a RedTeamResult
57
+ */
58
+ export type ExtractRedTeamCases = RedTeamResult['manifest']['results'];
59
+
60
+ /**
61
+ * Extract stress results from a StressResult
62
+ */
63
+ export type ExtractStressResults = StressResult['manifest']['sample_results'];
64
+
65
+ /**
66
+ * Extract the manifest type from any result type
67
+ */
68
+ export type ExtractManifest<T extends AnyResult> = T extends RunResult
69
+ ? RunManifest
70
+ : T extends RedTeamResult
71
+ ? RedTeamManifest
72
+ : T extends StressResult
73
+ ? StressManifest
74
+ : never;
75
+
76
+ // ============================================================================
77
+ // Partial Types for Construction
78
+ // ============================================================================
79
+
80
+ /**
81
+ * Deep partial utility type - makes all nested properties optional
82
+ */
83
+ export type DeepPartial<T> = T extends object
84
+ ? {
85
+ [P in keyof T]?: DeepPartial<T[P]>;
86
+ }
87
+ : T;
88
+
89
+ /**
90
+ * Partial scenario for incremental construction
91
+ */
92
+ export type PartialScenario = DeepPartial<Scenario>;
93
+
94
+ /**
95
+ * Make specific fields required
96
+ */
97
+ export type RequireFields<T, K extends keyof T> = T & Required<Pick<T, K>>;
98
+
99
+ /**
100
+ * Make specific fields optional
101
+ */
102
+ export type OptionalFields<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
103
+
104
+ // ============================================================================
105
+ // Strict Configuration Types
106
+ // ============================================================================
107
+
108
+ /**
109
+ * Strict version of AdapterConfig with all common fields required
110
+ */
111
+ export type StrictAdapterConfig = RequireFields<AdapterConfig, 'provider' | 'apiKey'>;
112
+
113
+ // ============================================================================
114
+ // Type Guards
115
+ // ============================================================================
116
+
117
+ /**
118
+ * Type guard for RunResult
119
+ */
120
+ export function isRunResult(result: AnyResult): result is RunResult {
121
+ return 'cases' in result && Array.isArray(result.cases) && !('defenseRate' in result);
122
+ }
123
+
124
+ /**
125
+ * Type guard for RedTeamResult
126
+ */
127
+ export function isRedTeamResult(result: AnyResult): result is RedTeamResult {
128
+ return 'defenseRate' in result && 'unsafeCount' in result;
129
+ }
130
+
131
+ /**
132
+ * Type guard for StressResult
133
+ */
134
+ export function isStressResult(result: AnyResult): result is StressResult {
135
+ return 'rps' in result && 'p95LatencyMs' in result;
136
+ }
137
+
138
+ /**
139
+ * Type guard for RunManifest
140
+ *
141
+ * Uses structural field checks rather than absence-of-field logic for robustness.
142
+ * Checks for RunManifest-specific fields: 'metrics.success_rate' and 'cases' array.
143
+ */
144
+ export function isRunManifestType(manifest: AnyManifest): manifest is RunManifest {
145
+ return (
146
+ 'metrics' in manifest &&
147
+ manifest.metrics !== null &&
148
+ typeof manifest.metrics === 'object' &&
149
+ 'success_rate' in manifest.metrics &&
150
+ 'cases' in manifest &&
151
+ Array.isArray((manifest as RunManifest).cases)
152
+ );
153
+ }
154
+
155
+ /**
156
+ * Type guard for RedTeamManifest
157
+ */
158
+ export function isRedTeamManifestType(manifest: AnyManifest): manifest is RedTeamManifest {
159
+ return 'type' in manifest && manifest.type === 'redteam';
160
+ }
161
+
162
+ /**
163
+ * Type guard for StressManifest
164
+ */
165
+ export function isStressManifestType(manifest: AnyManifest): manifest is StressManifest {
166
+ return 'type' in manifest && manifest.type === 'stress';
167
+ }
168
+
169
+ // ============================================================================
170
+ // Assertion Helpers
171
+ // ============================================================================
172
+
173
+ /**
174
+ * Assert that a value is defined (non-null, non-undefined)
175
+ */
176
+ export function assertDefined<T>(
177
+ value: T | null | undefined,
178
+ message = 'Value is null or undefined'
179
+ ): asserts value is T {
180
+ if (value === null || value === undefined) {
181
+ throw new Error(message);
182
+ }
183
+ }
184
+
185
+ /**
186
+ * Assert that a condition is true
187
+ */
188
+ export function assert(condition: boolean, message = 'Assertion failed'): asserts condition {
189
+ if (!condition) {
190
+ throw new Error(message);
191
+ }
192
+ }
193
+
194
+ // ============================================================================
195
+ // Result Analysis Helpers
196
+ // ============================================================================
197
+
198
+ /**
199
+ * Get failed cases from a run result
200
+ */
201
+ export function getFailedCases(result: RunResult): CaseResult[] {
202
+ return result.cases.filter((c) => !c.ok);
203
+ }
204
+
205
+ /**
206
+ * Get passed cases from a run result
207
+ */
208
+ export function getPassedCases(result: RunResult): CaseResult[] {
209
+ return result.cases.filter((c) => c.ok);
210
+ }
211
+
212
+ /**
213
+ * Get cases by tag from a run result
214
+ */
215
+ export function getCasesByTag(result: RunResult, tag: string): CaseResult[] {
216
+ return result.cases.filter((c) => c.tags.includes(tag));
217
+ }
218
+
219
+ /**
220
+ * Calculate success rate from a run result
221
+ */
222
+ export function calculateSuccessRate(result: RunResult): number {
223
+ if (result.cases.length === 0) return 0;
224
+ return result.cases.filter((c) => c.ok).length / result.cases.length;
225
+ }
package/tsconfig.json CHANGED
@@ -4,7 +4,8 @@
4
4
  "outDir": "./dist",
5
5
  "rootDir": "./src",
6
6
  "declaration": true,
7
- "declarationMap": true
7
+ "declarationMap": true,
8
+ "noEmit": false
8
9
  },
9
10
  "include": ["src/**/*"],
10
11
  "exclude": ["node_modules", "dist", "**/*.test.ts"]