@artemiskit/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +229 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +352 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +354 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +53 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +44918 -28644
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/index.js +13 -59
  39. package/dist/matchers/jest.d.ts +46 -0
  40. package/dist/matchers/jest.d.ts.map +1 -0
  41. package/dist/matchers/jest.js +0 -46
  42. package/dist/matchers/vitest.d.ts +24 -0
  43. package/dist/matchers/vitest.d.ts.map +1 -0
  44. package/dist/matchers/vitest.js +0 -46
  45. package/dist/types-only.d.ts +31 -0
  46. package/dist/types-only.d.ts.map +1 -0
  47. package/dist/types-only.js +1 -0
  48. package/dist/types.d.ts +379 -0
  49. package/dist/types.d.ts.map +1 -0
  50. package/dist/utils/index.d.ts +112 -0
  51. package/dist/utils/index.d.ts.map +1 -0
  52. package/dist/utils/index.js +58 -0
  53. package/package.json +19 -4
  54. package/src/__tests__/builders.test.ts +450 -0
  55. package/src/__tests__/circuit-breaker.test.ts +704 -0
  56. package/src/__tests__/compare.test.ts +469 -0
  57. package/src/__tests__/contracts.test.ts +354 -0
  58. package/src/__tests__/guardian-modes.test.ts +215 -0
  59. package/src/__tests__/guardrails.test.ts +404 -0
  60. package/src/__tests__/semantic-validator.test.ts +421 -0
  61. package/src/__tests__/utils.test.ts +324 -0
  62. package/src/__tests__/validate.test.ts +310 -0
  63. package/src/artemiskit.ts +479 -5
  64. package/src/builders/index.ts +700 -0
  65. package/src/contracts/index.ts +357 -0
  66. package/src/guardian/circuit-breaker.ts +74 -28
  67. package/src/guardian/guardian.ts +307 -20
  68. package/src/guardian/guardrails.ts +143 -1
  69. package/src/guardian/index.ts +35 -1
  70. package/src/guardian/interceptor.ts +87 -11
  71. package/src/guardian/semantic-validator.ts +399 -0
  72. package/src/guardian/types.ts +413 -2
  73. package/src/index.ts +258 -14
  74. package/src/types-only.ts +276 -0
  75. package/src/types.ts +186 -0
  76. package/src/utils/index.ts +225 -0
  77. package/tsconfig.json +2 -1
  78. package/adapters/openai/dist/index.js +0 -5625
package/src/types.ts CHANGED
@@ -9,8 +9,10 @@ import type {
9
9
  ModelClient,
10
10
  RedTeamManifest,
11
11
  RunManifest,
12
+ StorageConfig,
12
13
  StressManifest,
13
14
  StressRequestResult,
15
+ ToolExecutor,
14
16
  } from '@artemiskit/core';
15
17
  import type { RedactionConfig } from '@artemiskit/core';
16
18
  import type { Scenario } from '@artemiskit/core';
@@ -114,6 +116,12 @@ export interface ArtemisKitConfig {
114
116
  retries?: number;
115
117
  /** Default concurrency for parallel execution */
116
118
  concurrency?: number;
119
+ /**
120
+ * Storage configuration for persisting and loading run manifests
121
+ * Required for compare() method to load historical runs
122
+ * @since 0.3.2
123
+ */
124
+ storage?: StorageConfig;
117
125
  }
118
126
 
119
127
  /**
@@ -140,6 +148,8 @@ export interface RunOptions {
140
148
  retries?: number;
141
149
  /** Redaction configuration */
142
150
  redaction?: RedactionConfig;
151
+ /** Explicit executor for real tools; CLI scenarios remain fixture-only. */
152
+ toolExecutor?: ToolExecutor;
143
153
  }
144
154
 
145
155
  /**
@@ -257,3 +267,179 @@ export interface ArtemisKitEvents {
257
267
  }
258
268
 
259
269
  export type ArtemisKitEventName = keyof ArtemisKitEvents;
270
+
271
+ // ============================================================================
272
+ // Validation Types (v0.3.2+)
273
+ // ============================================================================
274
+
275
+ /**
276
+ * Options for validating scenario files
277
+ */
278
+ export interface ValidateOptions {
279
+ /**
280
+ * Scenario file path(s) or glob pattern
281
+ * Can be a single path, array of paths, or glob pattern
282
+ */
283
+ scenario: string | string[];
284
+
285
+ /**
286
+ * Strict mode - fail on warnings as well as errors
287
+ * @default false
288
+ */
289
+ strict?: boolean;
290
+ }
291
+
292
+ /**
293
+ * Validation error details
294
+ */
295
+ export interface ValidationError {
296
+ /** File path where error occurred */
297
+ file: string;
298
+ /** Error message */
299
+ message: string;
300
+ /** Line number if available */
301
+ line?: number;
302
+ /** Column number if available */
303
+ column?: number;
304
+ }
305
+
306
+ /**
307
+ * Validation warning details
308
+ */
309
+ export interface ValidationWarning {
310
+ /** File path where warning occurred */
311
+ file: string;
312
+ /** Warning message */
313
+ message: string;
314
+ /** Line number if available */
315
+ line?: number;
316
+ /** Column number if available */
317
+ column?: number;
318
+ }
319
+
320
+ /**
321
+ * Per-scenario validation result
322
+ */
323
+ export interface ScenarioValidation {
324
+ /** File path */
325
+ file: string;
326
+ /** Scenario name */
327
+ name: string;
328
+ /** Number of test cases */
329
+ caseCount: number;
330
+ /** Whether this scenario is valid */
331
+ valid: boolean;
332
+ }
333
+
334
+ /**
335
+ * Result from validate() method
336
+ */
337
+ export interface ValidationResult {
338
+ /** Overall validation passed */
339
+ valid: boolean;
340
+ /** Individual scenario validation results */
341
+ scenarios: ScenarioValidation[];
342
+ /** Validation errors */
343
+ errors: ValidationError[];
344
+ /** Validation warnings (informational) */
345
+ warnings: ValidationWarning[];
346
+ }
347
+
348
+ // ============================================================================
349
+ // Comparison Types (v0.3.2+)
350
+ // ============================================================================
351
+
352
+ /**
353
+ * Options for comparing test runs
354
+ */
355
+ export interface CompareOptions {
356
+ /**
357
+ * Baseline run identifier
358
+ * Can be a run ID, 'latest', or a named baseline
359
+ */
360
+ baseline: string;
361
+
362
+ /**
363
+ * Current run identifier
364
+ * Can be a run ID
365
+ */
366
+ current: string;
367
+
368
+ /**
369
+ * Regression threshold (0-1)
370
+ * A regression is detected if success rate drops by more than this amount
371
+ * @default 0.05 (5%)
372
+ */
373
+ threshold?: number;
374
+ }
375
+
376
+ /**
377
+ * Summary of a run for comparison
378
+ */
379
+ export interface RunSummary {
380
+ /** Run identifier */
381
+ runId: string;
382
+ /** Success rate (0-1) */
383
+ successRate: number;
384
+ /** Total number of test cases */
385
+ totalCases: number;
386
+ /** Number of passed cases */
387
+ passedCases: number;
388
+ /** Number of failed cases */
389
+ failedCases: number;
390
+ }
391
+
392
+ /**
393
+ * Comparison details between two runs
394
+ */
395
+ export interface ComparisonDetails {
396
+ /** Change in success rate (current - baseline) */
397
+ successRateDelta: number;
398
+ /** Cases that passed in baseline but failed in current */
399
+ newFailures: Array<{
400
+ caseId: string;
401
+ caseName?: string;
402
+ baselineStatus: string;
403
+ currentStatus: string;
404
+ }>;
405
+ /** Cases that failed in baseline but passed in current */
406
+ newPasses: Array<{
407
+ caseId: string;
408
+ caseName?: string;
409
+ baselineStatus: string;
410
+ currentStatus: string;
411
+ }>;
412
+ /** Cases with unchanged status */
413
+ unchanged: Array<{
414
+ caseId: string;
415
+ status: string;
416
+ }>;
417
+ /** Cases that exist in current run but not in baseline (new test cases) */
418
+ addedCases: Array<{
419
+ caseId: string;
420
+ caseName?: string;
421
+ status: string;
422
+ }>;
423
+ /** Cases that exist in baseline but not in current run (removed test cases) */
424
+ removedCases: Array<{
425
+ caseId: string;
426
+ caseName?: string;
427
+ status: string;
428
+ }>;
429
+ }
430
+
431
+ /**
432
+ * Result from compare() method
433
+ */
434
+ export interface CompareResult {
435
+ /** Baseline run summary */
436
+ baseline: RunSummary;
437
+ /** Current run summary */
438
+ current: RunSummary;
439
+ /** Detailed comparison */
440
+ comparison: ComparisonDetails;
441
+ /** Whether a regression was detected */
442
+ hasRegression: boolean;
443
+ /** Threshold used for regression detection */
444
+ threshold: number;
445
+ }
@@ -0,0 +1,225 @@
1
+ /**
2
+ * @artemiskit/sdk
3
+ * Utility types and helper functions
4
+ */
5
+
6
+ import type {
7
+ AdapterConfig,
8
+ AnyManifest,
9
+ CaseResult,
10
+ Expected,
11
+ RedTeamManifest,
12
+ RunManifest,
13
+ Scenario,
14
+ StressManifest,
15
+ } from '@artemiskit/core';
16
+ import type { RedTeamResult, RunResult, StressResult } from '../types';
17
+
18
+ // ============================================================================
19
+ // Provider Name Types
20
+ // ============================================================================
21
+
22
+ /**
23
+ * All supported provider names as a union type
24
+ */
25
+ export type ProviderName = AdapterConfig['provider'];
26
+
27
+ /**
28
+ * All supported expectation types
29
+ */
30
+ export type ExpectationType = Expected['type'];
31
+
32
+ // ============================================================================
33
+ // Result Union Types
34
+ // ============================================================================
35
+
36
+ /**
37
+ * Union of all SDK result types
38
+ */
39
+ export type AnyResult = RunResult | RedTeamResult | StressResult;
40
+
41
+ /**
42
+ * Union of all manifest types (re-export from core for convenience)
43
+ */
44
+ export type { AnyManifest };
45
+
46
+ // ============================================================================
47
+ // Type Extraction Utilities
48
+ // ============================================================================
49
+
50
+ /**
51
+ * Extract case results from a RunResult
52
+ */
53
+ export type ExtractRunCases = RunResult['cases'];
54
+
55
+ /**
56
+ * Extract red team cases from a RedTeamResult
57
+ */
58
+ export type ExtractRedTeamCases = RedTeamResult['manifest']['results'];
59
+
60
+ /**
61
+ * Extract stress results from a StressResult
62
+ */
63
+ export type ExtractStressResults = StressResult['manifest']['sample_results'];
64
+
65
+ /**
66
+ * Extract the manifest type from any result type
67
+ */
68
+ export type ExtractManifest<T extends AnyResult> = T extends RunResult
69
+ ? RunManifest
70
+ : T extends RedTeamResult
71
+ ? RedTeamManifest
72
+ : T extends StressResult
73
+ ? StressManifest
74
+ : never;
75
+
76
+ // ============================================================================
77
+ // Partial Types for Construction
78
+ // ============================================================================
79
+
80
+ /**
81
+ * Deep partial utility type - makes all nested properties optional
82
+ */
83
+ export type DeepPartial<T> = T extends object
84
+ ? {
85
+ [P in keyof T]?: DeepPartial<T[P]>;
86
+ }
87
+ : T;
88
+
89
+ /**
90
+ * Partial scenario for incremental construction
91
+ */
92
+ export type PartialScenario = DeepPartial<Scenario>;
93
+
94
+ /**
95
+ * Make specific fields required
96
+ */
97
+ export type RequireFields<T, K extends keyof T> = T & Required<Pick<T, K>>;
98
+
99
+ /**
100
+ * Make specific fields optional
101
+ */
102
+ export type OptionalFields<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
103
+
104
+ // ============================================================================
105
+ // Strict Configuration Types
106
+ // ============================================================================
107
+
108
+ /**
109
+ * Strict version of AdapterConfig with all common fields required
110
+ */
111
+ export type StrictAdapterConfig = RequireFields<AdapterConfig, 'provider' | 'apiKey'>;
112
+
113
+ // ============================================================================
114
+ // Type Guards
115
+ // ============================================================================
116
+
117
+ /**
118
+ * Type guard for RunResult
119
+ */
120
+ export function isRunResult(result: AnyResult): result is RunResult {
121
+ return 'cases' in result && Array.isArray(result.cases) && !('defenseRate' in result);
122
+ }
123
+
124
+ /**
125
+ * Type guard for RedTeamResult
126
+ */
127
+ export function isRedTeamResult(result: AnyResult): result is RedTeamResult {
128
+ return 'defenseRate' in result && 'unsafeCount' in result;
129
+ }
130
+
131
+ /**
132
+ * Type guard for StressResult
133
+ */
134
+ export function isStressResult(result: AnyResult): result is StressResult {
135
+ return 'rps' in result && 'p95LatencyMs' in result;
136
+ }
137
+
138
+ /**
139
+ * Type guard for RunManifest
140
+ *
141
+ * Uses structural field checks rather than absence-of-field logic for robustness.
142
+ * Checks for RunManifest-specific fields: 'metrics.success_rate' and 'cases' array.
143
+ */
144
+ export function isRunManifestType(manifest: AnyManifest): manifest is RunManifest {
145
+ return (
146
+ 'metrics' in manifest &&
147
+ manifest.metrics !== null &&
148
+ typeof manifest.metrics === 'object' &&
149
+ 'success_rate' in manifest.metrics &&
150
+ 'cases' in manifest &&
151
+ Array.isArray((manifest as RunManifest).cases)
152
+ );
153
+ }
154
+
155
+ /**
156
+ * Type guard for RedTeamManifest
157
+ */
158
+ export function isRedTeamManifestType(manifest: AnyManifest): manifest is RedTeamManifest {
159
+ return 'type' in manifest && manifest.type === 'redteam';
160
+ }
161
+
162
+ /**
163
+ * Type guard for StressManifest
164
+ */
165
+ export function isStressManifestType(manifest: AnyManifest): manifest is StressManifest {
166
+ return 'type' in manifest && manifest.type === 'stress';
167
+ }
168
+
169
+ // ============================================================================
170
+ // Assertion Helpers
171
+ // ============================================================================
172
+
173
+ /**
174
+ * Assert that a value is defined (non-null, non-undefined)
175
+ */
176
+ export function assertDefined<T>(
177
+ value: T | null | undefined,
178
+ message = 'Value is null or undefined'
179
+ ): asserts value is T {
180
+ if (value === null || value === undefined) {
181
+ throw new Error(message);
182
+ }
183
+ }
184
+
185
+ /**
186
+ * Assert that a condition is true
187
+ */
188
+ export function assert(condition: boolean, message = 'Assertion failed'): asserts condition {
189
+ if (!condition) {
190
+ throw new Error(message);
191
+ }
192
+ }
193
+
194
+ // ============================================================================
195
+ // Result Analysis Helpers
196
+ // ============================================================================
197
+
198
+ /**
199
+ * Get failed cases from a run result
200
+ */
201
+ export function getFailedCases(result: RunResult): CaseResult[] {
202
+ return result.cases.filter((c) => !c.ok);
203
+ }
204
+
205
+ /**
206
+ * Get passed cases from a run result
207
+ */
208
+ export function getPassedCases(result: RunResult): CaseResult[] {
209
+ return result.cases.filter((c) => c.ok);
210
+ }
211
+
212
+ /**
213
+ * Get cases by tag from a run result
214
+ */
215
+ export function getCasesByTag(result: RunResult, tag: string): CaseResult[] {
216
+ return result.cases.filter((c) => c.tags.includes(tag));
217
+ }
218
+
219
+ /**
220
+ * Calculate success rate from a run result
221
+ */
222
+ export function calculateSuccessRate(result: RunResult): number {
223
+ if (result.cases.length === 0) return 0;
224
+ return result.cases.filter((c) => c.ok).length / result.cases.length;
225
+ }
package/tsconfig.json CHANGED
@@ -4,7 +4,8 @@
4
4
  "outDir": "./dist",
5
5
  "rootDir": "./src",
6
6
  "declaration": true,
7
- "declarationMap": true
7
+ "declarationMap": true,
8
+ "noEmit": false
8
9
  },
9
10
  "include": ["src/**/*"],
10
11
  "exclude": ["node_modules", "dist", "**/*.test.ts"]