@artemiskit/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,230 @@
|
|
|
1
1
|
# @artemiskit/sdk
|
|
2
2
|
|
|
3
|
+
## 0.3.3
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 14ee2d8: ## SDK v0.3.2 - Guardian Enhancements & SDK Parity
|
|
8
|
+
|
|
9
|
+
### New Features
|
|
10
|
+
|
|
11
|
+
- **SDK Parity Methods**: Added `kit.validate()` for pre-flight scenario validation and `kit.compare()` for regression detection
|
|
12
|
+
- **Guardian Mode Normalization**: Canonical modes (observe/selective/strict) with deprecation warnings for legacy modes
|
|
13
|
+
- **Semantic Validation**: LLM-as-judge pattern for content validation with configurable thresholds
|
|
14
|
+
- **Programmatic Scenario Builders**: Type-safe fluent API for building scenarios programmatically
|
|
15
|
+
- **Type Contracts**: Strict type contracts for safe SDK usage
|
|
16
|
+
|
|
17
|
+
### Guardian Security Improvements
|
|
18
|
+
|
|
19
|
+
- Added `shouldBlockViolation` callback for mode-aware per-violation blocking decisions
|
|
20
|
+
- Hardened `parseResponse` to detect multiple JSON candidates as potential prompt injection
|
|
21
|
+
- Added `circuit_breaker` to GuardrailType for type safety
|
|
22
|
+
- Fixed streaming bypass security gap - streaming now throws instead of bypassing protections
|
|
23
|
+
|
|
24
|
+
### Bug Fixes
|
|
25
|
+
|
|
26
|
+
- Fixed case-insensitive pattern matching in guardrails
|
|
27
|
+
- Added `addedCases`/`removedCases` to comparison results
|
|
28
|
+
- Fixed lint warnings in semantic validator
|
|
29
|
+
|
|
30
|
+
## 0.3.2
|
|
31
|
+
|
|
32
|
+
### Patch Changes
|
|
33
|
+
|
|
34
|
+
- SDK Parity + Guardian Semantic Validation (v0.3.2)
|
|
35
|
+
|
|
36
|
+
## New Features
|
|
37
|
+
|
|
38
|
+
### SDK Parity Methods
|
|
39
|
+
|
|
40
|
+
#### `kit.validate()` - Scenario Validation
|
|
41
|
+
|
|
42
|
+
Validate scenario files without execution for CI/CD pre-flight checks.
|
|
43
|
+
|
|
44
|
+
```typescript
|
|
45
|
+
const result = await kit.validate({
|
|
46
|
+
scenario: "./scenarios/**/*.yaml",
|
|
47
|
+
strict: false,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
if (!result.valid) {
|
|
51
|
+
console.error("Validation errors:", result.errors);
|
|
52
|
+
process.exit(1);
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
**Returns:** `ValidationResult` with `valid`, `scenarios`, `errors`, and `warnings` fields.
|
|
57
|
+
|
|
58
|
+
#### `kit.compare()` - Run Comparison
|
|
59
|
+
|
|
60
|
+
Compare two test runs for regression detection.
|
|
61
|
+
|
|
62
|
+
```typescript
|
|
63
|
+
const comparison = await kit.compare({
|
|
64
|
+
baseline: "run_abc123",
|
|
65
|
+
current: "run_xyz789",
|
|
66
|
+
threshold: 0.05,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
if (comparison.hasRegression) {
|
|
70
|
+
console.error("Regression detected!", comparison.summary);
|
|
71
|
+
}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Guardian Mode Normalization
|
|
75
|
+
|
|
76
|
+
Canonical modes replace legacy mode names with deprecation warnings:
|
|
77
|
+
|
|
78
|
+
| Canonical Mode | Legacy Mode (Deprecated) | Behavior |
|
|
79
|
+
| -------------- | ------------------------ | ---------------------------- |
|
|
80
|
+
| `observe` | `testing` | Log only, never block |
|
|
81
|
+
| `selective` | `hybrid` | Block high-confidence only |
|
|
82
|
+
| `strict` | `guardian` | Block all detected (default) |
|
|
83
|
+
|
|
84
|
+
```typescript
|
|
85
|
+
// Using canonical modes (recommended)
|
|
86
|
+
const guardian = createGuardian({ mode: "selective" });
|
|
87
|
+
|
|
88
|
+
// Legacy modes still work but emit deprecation warnings
|
|
89
|
+
const guardian = createGuardian({ mode: "testing" });
|
|
90
|
+
// Console: [ArtemisKit Guardian] Mode 'testing' is deprecated. Use 'observe' instead.
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
### SemanticValidator - LLM-as-Judge Content Validation
|
|
94
|
+
|
|
95
|
+
New `SemanticValidator` class for LLM-based content analysis, now the default validation strategy.
|
|
96
|
+
|
|
97
|
+
```typescript
|
|
98
|
+
import { SemanticValidator, createSemanticValidator } from "@artemiskit/sdk";
|
|
99
|
+
|
|
100
|
+
const validator = createSemanticValidator(llmClient, {
|
|
101
|
+
strategy: "semantic",
|
|
102
|
+
semanticThreshold: 0.9,
|
|
103
|
+
categories: ["prompt_injection", "jailbreak", "pii_disclosure"],
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
// Validate input
|
|
107
|
+
const inputResult = await validator.validateInput("User message here");
|
|
108
|
+
if (inputResult.shouldBlock) {
|
|
109
|
+
// Handle violation
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Validate output
|
|
113
|
+
const outputResult = await validator.validateOutput("LLM response here");
|
|
114
|
+
|
|
115
|
+
// Use as Guardian guardrail
|
|
116
|
+
const guardrail = validator.asGuardrail("input");
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
**Features:**
|
|
120
|
+
|
|
121
|
+
- Input validation: Detects prompt injection, jailbreak attempts, role manipulation
|
|
122
|
+
- Output validation: Detects PII disclosure, harmful content, system leakage
|
|
123
|
+
- Configurable threshold: Control blocking sensitivity (0-1)
|
|
124
|
+
- Fail-open behavior: On LLM errors, defaults to allowing (for availability)
|
|
125
|
+
- Guardrail integration: Use `asGuardrail()` for Guardian integration
|
|
126
|
+
|
|
127
|
+
### Semantic Validation as Default
|
|
128
|
+
|
|
129
|
+
Guardian now defaults to semantic (LLM-as-judge) content validation when an `llmClient` is provided.
|
|
130
|
+
|
|
131
|
+
```typescript
|
|
132
|
+
const guardian = createGuardian({
|
|
133
|
+
llmClient: myLLMClient, // Required for semantic validation
|
|
134
|
+
contentValidation: {
|
|
135
|
+
strategy: "semantic", // NEW DEFAULT (was 'pattern')
|
|
136
|
+
semanticThreshold: 0.9,
|
|
137
|
+
},
|
|
138
|
+
});
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
**Validation Strategies:**
|
|
142
|
+
|
|
143
|
+
| Strategy | Description | Requires LLM |
|
|
144
|
+
| ---------- | ------------------------------------ | ------------ |
|
|
145
|
+
| `semantic` | LLM-based content analysis (default) | Yes |
|
|
146
|
+
| `pattern` | Regex/pattern matching only | No |
|
|
147
|
+
| `hybrid` | Both semantic and pattern | Yes |
|
|
148
|
+
| `off` | Disable content validation | No |
|
|
149
|
+
|
|
150
|
+
## New Exports
|
|
151
|
+
|
|
152
|
+
```typescript
|
|
153
|
+
import {
|
|
154
|
+
// Existing
|
|
155
|
+
ArtemisKit,
|
|
156
|
+
createGuardian,
|
|
157
|
+
Guardian,
|
|
158
|
+
|
|
159
|
+
// New in 0.3.2
|
|
160
|
+
normalizeGuardianMode,
|
|
161
|
+
SemanticValidator,
|
|
162
|
+
createSemanticValidator,
|
|
163
|
+
} from "@artemiskit/sdk";
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Type Changes
|
|
167
|
+
|
|
168
|
+
```typescript
|
|
169
|
+
// New canonical modes
|
|
170
|
+
type GuardianModeCanonical = "observe" | "selective" | "strict";
|
|
171
|
+
|
|
172
|
+
// Legacy modes (deprecated)
|
|
173
|
+
type GuardianModeLegacy = "testing" | "guardian" | "hybrid";
|
|
174
|
+
|
|
175
|
+
// Union type for backwards compatibility
|
|
176
|
+
type GuardianMode = GuardianModeCanonical | GuardianModeLegacy;
|
|
177
|
+
|
|
178
|
+
// Validation categories
|
|
179
|
+
type ValidationCategory =
|
|
180
|
+
| "prompt_injection"
|
|
181
|
+
| "jailbreak"
|
|
182
|
+
| "pii_disclosure"
|
|
183
|
+
| "role_manipulation"
|
|
184
|
+
| "data_extraction"
|
|
185
|
+
| "content_safety";
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
## Migration Notes
|
|
189
|
+
|
|
190
|
+
### Content Validation Default Change
|
|
191
|
+
|
|
192
|
+
If you relied on pattern-only validation, explicitly set:
|
|
193
|
+
|
|
194
|
+
```typescript
|
|
195
|
+
const guardian = createGuardian({
|
|
196
|
+
contentValidation: { strategy: "pattern" },
|
|
197
|
+
});
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
### Mode Name Updates (Optional)
|
|
201
|
+
|
|
202
|
+
Legacy mode names continue to work but emit deprecation warnings:
|
|
203
|
+
|
|
204
|
+
```typescript
|
|
205
|
+
// Before (still works)
|
|
206
|
+
createGuardian({ mode: "testing" });
|
|
207
|
+
|
|
208
|
+
// After (recommended)
|
|
209
|
+
createGuardian({ mode: "observe" });
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## Breaking Changes
|
|
213
|
+
|
|
214
|
+
None. All changes are backwards compatible.
|
|
215
|
+
|
|
216
|
+
## 0.3.1
|
|
217
|
+
|
|
218
|
+
### Patch Changes
|
|
219
|
+
|
|
220
|
+
- 29e29d6: Fix npm install error caused by unresolved workspace:\* dependencies
|
|
221
|
+
|
|
222
|
+
The published package contained workspace:\* protocol references for @artemiskit/adapter-deepagents and @artemiskit/adapter-langchain, which npm doesn't support. These are now properly resolved to version numbers during publish.
|
|
223
|
+
|
|
224
|
+
Fix npm install error caused by unresolved workspace:\* dependencies
|
|
225
|
+
|
|
226
|
+
The published package contained workspace:\* protocol references for @artemiskit/adapter-deepagents and @artemiskit/adapter-langchain, which npm doesn't support. These are now properly resolved to version numbers during publish.
|
|
227
|
+
|
|
3
228
|
## 0.3.0
|
|
4
229
|
|
|
5
230
|
### Minor Changes
|
package/README.md
CHANGED
|
@@ -7,7 +7,8 @@ Programmatic SDK for [ArtemisKit](https://github.com/code-sensei/artemiskit) - i
|
|
|
7
7
|
- 🚀 **Simple API** - Run tests, red team evaluations, and stress tests programmatically
|
|
8
8
|
- 📊 **Event Emitters** - Real-time progress tracking with `onCaseStart`, `onCaseComplete`, `onProgress`
|
|
9
9
|
- 🧪 **Test Framework Integration** - Custom matchers for Jest and Vitest
|
|
10
|
-
- 🔴 **Red Team Testing** - Adversarial security testing
|
|
10
|
+
- 🔴 **Red Team Testing** - Adversarial security testing with OWASP LLM Top 10 coverage
|
|
11
|
+
- 🛡️ **Guardian Mode** - Runtime AI protection with injection detection and PII filtering
|
|
11
12
|
- ⚡ **Stress Testing** - Load testing with configurable concurrency
|
|
12
13
|
- 📝 **TypeScript First** - Full type definitions included
|
|
13
14
|
|
|
@@ -168,6 +169,48 @@ describe('LLM Tests', () => {
|
|
|
168
169
|
- `toAchieveRPS(rps)`
|
|
169
170
|
- `toHaveStressP95LatencyBelow(ms)`
|
|
170
171
|
|
|
172
|
+
## Guardian Mode
|
|
173
|
+
|
|
174
|
+
Protect your LLM applications at runtime:
|
|
175
|
+
|
|
176
|
+
```typescript
|
|
177
|
+
import { createGuardian } from '@artemiskit/sdk/guardian';
|
|
178
|
+
|
|
179
|
+
const guardian = createGuardian({
|
|
180
|
+
mode: 'guardian',
|
|
181
|
+
blockOnFailure: true,
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
// Wrap your LLM client
|
|
185
|
+
const protectedClient = guardian.protect(myLLMClient);
|
|
186
|
+
|
|
187
|
+
// Validate inputs before sending to LLM
|
|
188
|
+
const inputResult = await guardian.validateInput(userMessage);
|
|
189
|
+
if (!inputResult.safe) {
|
|
190
|
+
console.error('Blocked:', inputResult.reason);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Validate outputs before returning to user
|
|
194
|
+
const outputResult = await guardian.validateOutput(llmResponse);
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Guardian Features
|
|
198
|
+
|
|
199
|
+
- **Prompt Injection Detection** - Pattern matching for injection attempts
|
|
200
|
+
- **PII Detection & Redaction** - Email, phone, SSN, API keys
|
|
201
|
+
- **Action Validation** - Validate agent tool/function calls
|
|
202
|
+
- **Intent Classification** - Risk scoring for user intents
|
|
203
|
+
- **Circuit Breaker** - Auto-block on repeated violations
|
|
204
|
+
- **Rate & Cost Limiting** - Control usage
|
|
205
|
+
|
|
206
|
+
### Guardian Modes
|
|
207
|
+
|
|
208
|
+
| Mode | Description |
|
|
209
|
+
|------|-------------|
|
|
210
|
+
| `testing` | Log violations but don't block |
|
|
211
|
+
| `guardian` | Block violations and log |
|
|
212
|
+
| `hybrid` | Block critical, log others |
|
|
213
|
+
|
|
171
214
|
## License
|
|
172
215
|
|
|
173
216
|
Apache-2.0
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Main ArtemisKit class - programmatic API for LLM testing
|
|
4
|
+
*/
|
|
5
|
+
import type { ArtemisKitConfig, ArtemisKitEventName, ArtemisKitEvents, CaseCompleteHandler, CaseStartHandler, CompareOptions, CompareResult, ProgressHandler, RedTeamMutationCompleteHandler, RedTeamMutationStartHandler, RedTeamOptions, RedTeamResult, RunOptions, RunResult, StressOptions, StressRequestCompleteHandler, StressResult, ValidateOptions, ValidationResult } from './types';
|
|
6
|
+
/**
|
|
7
|
+
* ArtemisKit SDK - programmatic API for LLM evaluation testing
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```typescript
|
|
11
|
+
* import { ArtemisKit } from '@artemiskit/sdk';
|
|
12
|
+
*
|
|
13
|
+
* const kit = new ArtemisKit({
|
|
14
|
+
* provider: 'openai',
|
|
15
|
+
* model: 'gpt-4',
|
|
16
|
+
* });
|
|
17
|
+
*
|
|
18
|
+
* // Run a test scenario
|
|
19
|
+
* const result = await kit.run({ scenario: './my-tests.yaml' });
|
|
20
|
+
* console.log(result.success); // true/false
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
export declare class ArtemisKit {
|
|
24
|
+
private config;
|
|
25
|
+
private eventHandlers;
|
|
26
|
+
private storage;
|
|
27
|
+
constructor(config?: ArtemisKitConfig);
|
|
28
|
+
/**
|
|
29
|
+
* Register an event handler
|
|
30
|
+
*/
|
|
31
|
+
on<E extends ArtemisKitEventName>(event: E, handler: (event: ArtemisKitEvents[E]) => void): this;
|
|
32
|
+
/**
|
|
33
|
+
* Remove an event handler
|
|
34
|
+
*/
|
|
35
|
+
off<E extends ArtemisKitEventName>(event: E, handler: (event: ArtemisKitEvents[E]) => void): this;
|
|
36
|
+
/**
|
|
37
|
+
* Register a one-time event handler
|
|
38
|
+
*/
|
|
39
|
+
once<E extends ArtemisKitEventName>(event: E, handler: (event: ArtemisKitEvents[E]) => void): this;
|
|
40
|
+
/**
|
|
41
|
+
* Emit an event to all registered handlers
|
|
42
|
+
*/
|
|
43
|
+
private emit;
|
|
44
|
+
/**
|
|
45
|
+
* Register handler for when a test case starts
|
|
46
|
+
*/
|
|
47
|
+
onCaseStart(handler: CaseStartHandler): this;
|
|
48
|
+
/**
|
|
49
|
+
* Register handler for when a test case completes
|
|
50
|
+
*/
|
|
51
|
+
onCaseComplete(handler: CaseCompleteHandler): this;
|
|
52
|
+
/**
|
|
53
|
+
* Register handler for progress updates
|
|
54
|
+
*/
|
|
55
|
+
onProgress(handler: ProgressHandler): this;
|
|
56
|
+
/**
|
|
57
|
+
* Register handler for when a red team mutation starts
|
|
58
|
+
*/
|
|
59
|
+
onRedTeamMutationStart(handler: RedTeamMutationStartHandler): this;
|
|
60
|
+
/**
|
|
61
|
+
* Register handler for when a red team mutation completes
|
|
62
|
+
*/
|
|
63
|
+
onRedTeamMutationComplete(handler: RedTeamMutationCompleteHandler): this;
|
|
64
|
+
/**
|
|
65
|
+
* Register handler for stress test request completion
|
|
66
|
+
*/
|
|
67
|
+
onStressRequestComplete(handler: StressRequestCompleteHandler): this;
|
|
68
|
+
/**
|
|
69
|
+
* Run a test scenario
|
|
70
|
+
*/
|
|
71
|
+
run(options: RunOptions): Promise<RunResult>;
|
|
72
|
+
/**
|
|
73
|
+
* Run red team adversarial testing
|
|
74
|
+
*/
|
|
75
|
+
redteam(options: RedTeamOptions): Promise<RedTeamResult>;
|
|
76
|
+
/**
|
|
77
|
+
* Run stress/load testing
|
|
78
|
+
*/
|
|
79
|
+
stress(options: StressOptions): Promise<StressResult>;
|
|
80
|
+
/**
|
|
81
|
+
* Validate scenario files without execution (pre-flight checks)
|
|
82
|
+
*
|
|
83
|
+
* Use this for CI/CD pipelines to catch configuration errors before running tests.
|
|
84
|
+
*
|
|
85
|
+
* @example
|
|
86
|
+
* ```typescript
|
|
87
|
+
* const result = await kit.validate({ scenario: './scenarios/**\/*.yaml' });
|
|
88
|
+
* if (!result.valid) {
|
|
89
|
+
* console.error('Validation errors:', result.errors);
|
|
90
|
+
* process.exit(1);
|
|
91
|
+
* }
|
|
92
|
+
* ```
|
|
93
|
+
*/
|
|
94
|
+
validate(options: ValidateOptions): Promise<ValidationResult>;
|
|
95
|
+
/**
|
|
96
|
+
* Compare two test runs for regression detection
|
|
97
|
+
*
|
|
98
|
+
* Requires storage configuration to be set in ArtemisKitConfig.
|
|
99
|
+
*
|
|
100
|
+
* @example
|
|
101
|
+
* ```typescript
|
|
102
|
+
* const kit = new ArtemisKit({
|
|
103
|
+
* storage: { type: 'local', basePath: './artemis-runs' }
|
|
104
|
+
* });
|
|
105
|
+
*
|
|
106
|
+
* const comparison = await kit.compare({
|
|
107
|
+
* baseline: 'run_abc123',
|
|
108
|
+
* current: 'run_def456',
|
|
109
|
+
* threshold: 0.05, // 5% regression threshold
|
|
110
|
+
* });
|
|
111
|
+
*
|
|
112
|
+
* if (comparison.hasRegression) {
|
|
113
|
+
* console.error('Regression detected!', comparison.regressions);
|
|
114
|
+
* }
|
|
115
|
+
* ```
|
|
116
|
+
*
|
|
117
|
+
* @since 0.3.2
|
|
118
|
+
*/
|
|
119
|
+
compare(options: CompareOptions): Promise<CompareResult>;
|
|
120
|
+
/**
|
|
121
|
+
* Extract run metrics from a manifest
|
|
122
|
+
*/
|
|
123
|
+
private extractRunMetrics;
|
|
124
|
+
/**
|
|
125
|
+
* Compare cases between baseline and current manifests
|
|
126
|
+
*/
|
|
127
|
+
private compareCases;
|
|
128
|
+
/**
|
|
129
|
+
* Extract cases from a manifest
|
|
130
|
+
*/
|
|
131
|
+
private extractCases;
|
|
132
|
+
/**
|
|
133
|
+
* Load a scenario from file or use inline object
|
|
134
|
+
*/
|
|
135
|
+
private loadScenario;
|
|
136
|
+
/**
|
|
137
|
+
* Create a model client based on options
|
|
138
|
+
*/
|
|
139
|
+
private createClient;
|
|
140
|
+
/**
|
|
141
|
+
* Build mutation instances from mutation names
|
|
142
|
+
*/
|
|
143
|
+
private buildMutations;
|
|
144
|
+
/**
|
|
145
|
+
* Get available mutations for red team testing
|
|
146
|
+
*/
|
|
147
|
+
getAvailableMutations(): string[];
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=artemiskit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artemiskit.d.ts","sourceRoot":"","sources":["../src/artemiskit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuCH,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,aAAa,EACb,eAAe,EACf,8BAA8B,EAC9B,2BAA2B,EAC3B,cAAc,EACd,aAAa,EACb,UAAU,EACV,SAAS,EACT,aAAa,EACb,4BAA4B,EAC5B,YAAY,EACZ,eAAe,EACf,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAsBjB;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,aAAa,CAA6D;IAClF,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,GAAE,gBAAqB;IAuBzC;;OAEG;IACH,EAAE,CAAC,CAAC,SAAS,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI;IAQhG;;OAEG;IACH,GAAG,CAAC,CAAC,SAAS,mBAAmB,EAC/B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,IAAI,CAAC,CAAC,SAAS,mBAAmB,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC,KAAK,IAAI,GAC5C,IAAI;IAQP;;OAEG;IACH,OAAO,CAAC,IAAI;IAiBZ;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI5C;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,mBAAmB,GAAG,IAAI;IAIlD;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAI1C;;OAEG;IACH,sBAAsB,CAAC,OAAO,EAAE,2BAA2B,GAAG,IAAI;IAIlE;;OAEG;IACH,yBAAyB,CAAC,OAAO,EAAE,8BAA8B,GAAG,IAAI;IAIxE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,4BAA4B,GAAG,IAAI;IAQpE;;OAEG;IACG,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IA8DlD;;OAEG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAgN9D;;OAEG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAwO3D;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA2LnE;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IA6F9D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAmBzB;;OAEG;IACH,OAAO,CAAC,YAAY;IA4EpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAepB;;OAEG;YACW,YAAY;IAO1B;;OAEG;YACW,YAAY;IAgB1B;;OAEG;IACH,OAAO,CAAC,cAAc;IA6BtB;;OAEG;IACH,qBAAqB,IAAI,MAAM,EAAE;CAGlC"}
|