@artemiskit/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +213 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +17 -2
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Core matcher implementations for Jest/Vitest integration
|
|
4
|
+
*/
|
|
5
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
6
|
+
/**
|
|
7
|
+
* Matcher result interface compatible with Jest/Vitest
|
|
8
|
+
*/
|
|
9
|
+
export interface MatcherResult {
|
|
10
|
+
pass: boolean;
|
|
11
|
+
message: () => string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Check if a run result passed all test cases
|
|
15
|
+
*/
|
|
16
|
+
export declare function toPassAllCases(result: RunResult): MatcherResult;
|
|
17
|
+
/**
|
|
18
|
+
* Check if success rate meets threshold
|
|
19
|
+
*/
|
|
20
|
+
export declare function toHaveSuccessRate(result: RunResult, expectedRate: number): MatcherResult;
|
|
21
|
+
/**
|
|
22
|
+
* Check if all cases with specific tags passed
|
|
23
|
+
*/
|
|
24
|
+
export declare function toPassCasesWithTag(result: RunResult, tag: string): MatcherResult;
|
|
25
|
+
/**
|
|
26
|
+
* Check if median latency is within threshold
|
|
27
|
+
*/
|
|
28
|
+
export declare function toHaveMedianLatencyBelow(result: RunResult, maxLatencyMs: number): MatcherResult;
|
|
29
|
+
/**
|
|
30
|
+
* Check if P95 latency is within threshold
|
|
31
|
+
*/
|
|
32
|
+
export declare function toHaveP95LatencyBelow(result: RunResult, maxLatencyMs: number): MatcherResult;
|
|
33
|
+
/**
|
|
34
|
+
* Check if defense rate meets threshold
|
|
35
|
+
*/
|
|
36
|
+
export declare function toHaveDefenseRate(result: RedTeamResult, expectedRate: number): MatcherResult;
|
|
37
|
+
/**
|
|
38
|
+
* Check that no critical vulnerabilities were found
|
|
39
|
+
*/
|
|
40
|
+
export declare function toHaveNoCriticalVulnerabilities(result: RedTeamResult): MatcherResult;
|
|
41
|
+
/**
|
|
42
|
+
* Check that no high or critical vulnerabilities were found
|
|
43
|
+
*/
|
|
44
|
+
export declare function toHaveNoHighSeverityVulnerabilities(result: RedTeamResult): MatcherResult;
|
|
45
|
+
/**
|
|
46
|
+
* Check if red team test passed (default 95% defense rate)
|
|
47
|
+
*/
|
|
48
|
+
export declare function toPassRedTeam(result: RedTeamResult): MatcherResult;
|
|
49
|
+
/**
|
|
50
|
+
* Check if stress test success rate meets threshold
|
|
51
|
+
*/
|
|
52
|
+
export declare function toHaveStressSuccessRate(result: StressResult, expectedRate: number): MatcherResult;
|
|
53
|
+
/**
|
|
54
|
+
* Check if stress test achieved target RPS
|
|
55
|
+
*/
|
|
56
|
+
export declare function toAchieveRPS(result: StressResult, targetRPS: number): MatcherResult;
|
|
57
|
+
/**
|
|
58
|
+
* Check if stress test P95 latency is within threshold
|
|
59
|
+
*/
|
|
60
|
+
export declare function toHaveStressP95LatencyBelow(result: StressResult, maxLatencyMs: number): MatcherResult;
|
|
61
|
+
/**
|
|
62
|
+
* Check if stress test passed (default 95% success rate)
|
|
63
|
+
*/
|
|
64
|
+
export declare function toPassStressTest(result: StressResult): MatcherResult;
|
|
65
|
+
export declare const artemiskitMatchers: {
|
|
66
|
+
toPassAllCases: typeof toPassAllCases;
|
|
67
|
+
toHaveSuccessRate: typeof toHaveSuccessRate;
|
|
68
|
+
toPassCasesWithTag: typeof toPassCasesWithTag;
|
|
69
|
+
toHaveMedianLatencyBelow: typeof toHaveMedianLatencyBelow;
|
|
70
|
+
toHaveP95LatencyBelow: typeof toHaveP95LatencyBelow;
|
|
71
|
+
toHaveDefenseRate: typeof toHaveDefenseRate;
|
|
72
|
+
toHaveNoCriticalVulnerabilities: typeof toHaveNoCriticalVulnerabilities;
|
|
73
|
+
toHaveNoHighSeverityVulnerabilities: typeof toHaveNoHighSeverityVulnerabilities;
|
|
74
|
+
toPassRedTeam: typeof toPassRedTeam;
|
|
75
|
+
toHaveStressSuccessRate: typeof toHaveStressSuccessRate;
|
|
76
|
+
toAchieveRPS: typeof toAchieveRPS;
|
|
77
|
+
toHaveStressP95LatencyBelow: typeof toHaveStressP95LatencyBelow;
|
|
78
|
+
toPassStressTest: typeof toPassStressTest;
|
|
79
|
+
};
|
|
80
|
+
export type ArtemisKitMatchers = typeof artemiskitMatchers;
|
|
81
|
+
//# sourceMappingURL=core.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../../src/matchers/core.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,MAAM,CAAC;CACvB;AAoCD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,SAAS,GAAG,aAAa,CAW/D;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAYxF;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,aAAa,CAahF;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAW/F;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAW5F;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAc5F;AAED;;GAEG;AACH,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAWpF;AAED;;GAEG;AACH,wBAAgB,mCAAmC,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAaxF;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAiBlE;AAMD;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,GAAG,aAAa,CAWjG;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CAWnF;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,YAAY,EACpB,YAAY,EAAE,MAAM,GACnB,aAAa,CAWf;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,YAAY,GAAG,aAAa,CAiBpE;AAMD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;CAmB9B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Test matchers - core implementations without framework bindings
|
|
4
|
+
*/
|
|
5
|
+
export { artemiskitMatchers, type ArtemisKitMatchers, type MatcherResult, toPassAllCases, toHaveSuccessRate, toPassCasesWithTag, toHaveMedianLatencyBelow, toHaveP95LatencyBelow, toHaveDefenseRate, toHaveNoCriticalVulnerabilities, toHaveNoHighSeverityVulnerabilities, toPassRedTeam, toHaveStressSuccessRate, toAchieveRPS, toHaveStressP95LatencyBelow, toPassStressTest, } from './core';
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAElB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EAErB,iBAAiB,EACjB,+BAA+B,EAC/B,mCAAmC,EACnC,aAAa,EAEb,uBAAuB,EACvB,YAAY,EACZ,2BAA2B,EAC3B,gBAAgB,GACjB,MAAM,QAAQ,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Jest integration - custom matchers for ArtemisKit
|
|
4
|
+
*/
|
|
5
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
6
|
+
/**
|
|
7
|
+
* Jest matchers implementation
|
|
8
|
+
*/
|
|
9
|
+
export declare const jestMatchers: {
|
|
10
|
+
toPassAllCases(received: RunResult): import("./core").MatcherResult;
|
|
11
|
+
toHaveSuccessRate(received: RunResult, expectedRate: number): import("./core").MatcherResult;
|
|
12
|
+
toPassCasesWithTag(received: RunResult, tag: string): import("./core").MatcherResult;
|
|
13
|
+
toHaveMedianLatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
14
|
+
toHaveP95LatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
15
|
+
toHaveDefenseRate(received: RedTeamResult, expectedRate: number): import("./core").MatcherResult;
|
|
16
|
+
toHaveNoCriticalVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
17
|
+
toHaveNoHighSeverityVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
18
|
+
toPassRedTeam(received: RedTeamResult): import("./core").MatcherResult;
|
|
19
|
+
toHaveStressSuccessRate(received: StressResult, expectedRate: number): import("./core").MatcherResult;
|
|
20
|
+
toAchieveRPS(received: StressResult, targetRPS: number): import("./core").MatcherResult;
|
|
21
|
+
toHaveStressP95LatencyBelow(received: StressResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
22
|
+
toPassStressTest(received: StressResult): import("./core").MatcherResult;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* TypeScript declarations for Jest matchers
|
|
26
|
+
*/
|
|
27
|
+
declare global {
|
|
28
|
+
namespace jest {
|
|
29
|
+
interface Matchers<R> {
|
|
30
|
+
toPassAllCases(): R;
|
|
31
|
+
toHaveSuccessRate(expectedRate: number): R;
|
|
32
|
+
toPassCasesWithTag(tag: string): R;
|
|
33
|
+
toHaveMedianLatencyBelow(maxLatencyMs: number): R;
|
|
34
|
+
toHaveP95LatencyBelow(maxLatencyMs: number): R;
|
|
35
|
+
toHaveDefenseRate(expectedRate: number): R;
|
|
36
|
+
toHaveNoCriticalVulnerabilities(): R;
|
|
37
|
+
toHaveNoHighSeverityVulnerabilities(): R;
|
|
38
|
+
toPassRedTeam(): R;
|
|
39
|
+
toHaveStressSuccessRate(expectedRate: number): R;
|
|
40
|
+
toAchieveRPS(targetRPS: number): R;
|
|
41
|
+
toHaveStressP95LatencyBelow(maxLatencyMs: number): R;
|
|
42
|
+
toPassStressTest(): R;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=jest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jest.d.ts","sourceRoot":"","sources":["../../src/matchers/jest.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAiBvE;;GAEG;AACH,eAAO,MAAM,YAAY;6BACE,SAAS;gCAIN,SAAS,gBAAgB,MAAM;iCAI9B,SAAS,OAAO,MAAM;uCAIhB,SAAS,gBAAgB,MAAM;oCAIlC,SAAS,gBAAgB,MAAM;gCAInC,aAAa,gBAAgB,MAAM;8CAIrB,aAAa;kDAIT,aAAa;4BAInC,aAAa;sCAIH,YAAY,gBAAgB,MAAM;2BAI7C,YAAY,aAAa,MAAM;0CAIhB,YAAY,gBAAgB,MAAM;+BAI7C,YAAY;CAGxC,CAAC;AAQF;;GAEG;AACH,OAAO,CAAC,MAAM,CAAC;IACb,UAAU,IAAI,CAAC;QACb,UAAU,QAAQ,CAAC,CAAC;YAElB,cAAc,IAAI,CAAC,CAAC;YACpB,iBAAiB,CAAC,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC;YAC3C,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC;YACnC,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC;YAClD,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC;YAG/C,iBAAiB,CAAC,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC;YAC3C,+BAA+B,IAAI,CAAC,CAAC;YACrC,mCAAmC,IAAI,CAAC,CAAC;YACzC,aAAa,IAAI,CAAC,CAAC;YAGnB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC;YACjD,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,CAAC,CAAC;YACnC,2BAA2B,CAAC,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC;YACrD,gBAAgB,IAAI,CAAC,CAAC;SACvB;KACF;CACF"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Vitest integration - custom matchers for ArtemisKit
|
|
4
|
+
*/
|
|
5
|
+
import type { RedTeamResult, RunResult, StressResult } from '../types';
|
|
6
|
+
/**
|
|
7
|
+
* Vitest matchers implementation
|
|
8
|
+
*/
|
|
9
|
+
export declare const vitestMatchers: {
|
|
10
|
+
toPassAllCases(received: RunResult): import("./core").MatcherResult;
|
|
11
|
+
toHaveSuccessRate(received: RunResult, expectedRate: number): import("./core").MatcherResult;
|
|
12
|
+
toPassCasesWithTag(received: RunResult, tag: string): import("./core").MatcherResult;
|
|
13
|
+
toHaveMedianLatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
14
|
+
toHaveP95LatencyBelow(received: RunResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
15
|
+
toHaveDefenseRate(received: RedTeamResult, expectedRate: number): import("./core").MatcherResult;
|
|
16
|
+
toHaveNoCriticalVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
17
|
+
toHaveNoHighSeverityVulnerabilities(received: RedTeamResult): import("./core").MatcherResult;
|
|
18
|
+
toPassRedTeam(received: RedTeamResult): import("./core").MatcherResult;
|
|
19
|
+
toHaveStressSuccessRate(received: StressResult, expectedRate: number): import("./core").MatcherResult;
|
|
20
|
+
toAchieveRPS(received: StressResult, targetRPS: number): import("./core").MatcherResult;
|
|
21
|
+
toHaveStressP95LatencyBelow(received: StressResult, maxLatencyMs: number): import("./core").MatcherResult;
|
|
22
|
+
toPassStressTest(received: StressResult): import("./core").MatcherResult;
|
|
23
|
+
};
|
|
24
|
+
//# sourceMappingURL=vitest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vitest.d.ts","sourceRoot":"","sources":["../../src/matchers/vitest.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAiBvE;;GAEG;AACH,eAAO,MAAM,cAAc;6BACA,SAAS;gCAIN,SAAS,gBAAgB,MAAM;iCAI9B,SAAS,OAAO,MAAM;uCAIhB,SAAS,gBAAgB,MAAM;oCAIlC,SAAS,gBAAgB,MAAM;gCAInC,aAAa,gBAAgB,MAAM;8CAIrB,aAAa;kDAIT,aAAa;4BAInC,aAAa;sCAIH,YAAY,gBAAgB,MAAM;2BAI7C,YAAY,aAAa,MAAM;0CAIhB,YAAY,gBAAgB,MAAM;+BAI7C,YAAY;CAGxC,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk/types
|
|
3
|
+
* Types-only exports for consumers who only need type definitions
|
|
4
|
+
*
|
|
5
|
+
* Use this export when you only need types for type-checking
|
|
6
|
+
* without importing any runtime code.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import type {
|
|
11
|
+
* Scenario,
|
|
12
|
+
* TestCase,
|
|
13
|
+
* RunResult,
|
|
14
|
+
* ArtemisKitConfig
|
|
15
|
+
* } from '@artemiskit/sdk/types'
|
|
16
|
+
* ```
|
|
17
|
+
*/
|
|
18
|
+
export type { ArtemisKitConfig, RunOptions, RedTeamOptions, StressOptions, RunResult, RedTeamResult, StressResult, CaseStartEvent, CaseCompleteEvent, ProgressEvent, RedTeamMutationStartEvent, RedTeamMutationCompleteEvent, StressRequestCompleteEvent, CaseStartHandler, CaseCompleteHandler, ProgressHandler, RedTeamMutationStartHandler, RedTeamMutationCompleteHandler, StressRequestCompleteHandler, ArtemisKitEvents, ArtemisKitEventName, } from './types';
|
|
19
|
+
export type { Scenario, TestCase, Expected, Provider, ProviderConfig, ChatMessageType, Variables, } from '@artemiskit/core';
|
|
20
|
+
export type { CaseResult, CaseRedactionInfo, RunManifest, RunConfig, RunMetrics, RedTeamManifest, RedTeamConfig, RedTeamMetrics, RedTeamCaseResult, RedTeamStatus, RedTeamSeverity, StressManifest, StressConfig, StressMetrics, StressRequestResult, AnyManifest, ResolvedConfig, ConfigSource, ManifestRedactionInfo, CostEstimateInfo, GitInfo, ProvenanceInfo, } from '@artemiskit/core';
|
|
21
|
+
export type { ModelClient, ModelCapabilities, GenerateOptions, GenerateResult, TokenUsage, ChatMessage, FunctionDefinition, ToolDefinition, ToolCall, AdapterConfig, BaseAdapterConfig, OpenAIAdapterConfig, AzureOpenAIAdapterConfig, VercelAIAdapterConfig, AnthropicAdapterConfig, LangChainAdapterConfig, DeepAgentsAdapterConfig, ProviderType, } from '@artemiskit/core';
|
|
22
|
+
export type { Evaluator, EvaluatorContext, EvaluatorResult } from '@artemiskit/core';
|
|
23
|
+
export type { StorageAdapter, StorageConfig, RunListItem, ListOptions, ComparisonResult, BaselineMetadata, BaselineStorageAdapter, CaseResultRecord, CaseResultStatus, CaseResultQueryOptions, MetricsSnapshot, MetricsTrendOptions, TrendDataPoint, AnalyticsStorageAdapter, } from '@artemiskit/core';
|
|
24
|
+
export type { RedactionConfig } from '@artemiskit/core';
|
|
25
|
+
export type { Severity, SeverityInfo, CvssScore } from '@artemiskit/redteam';
|
|
26
|
+
export type { GuardianConfig, InterceptorConfig, ActionValidatorConfig, IntentClassifierConfig, GuardrailsConfig, RateLimiterConfig, GuardianMode, ViolationSeverity, ViolationAction, GuardrailType, Violation, GuardrailResult, PolicyRule, PolicyCondition, GuardianPolicy, CircuitBreakerConfig, RateLimitConfig, CostLimitConfig, ActionDefinition, ActionParameter, ParameterValidation, IntentClassification, IntentCategory, PIIDetection, PIIType, PIILocation, InjectionDetection, InjectionType, ContentFilterResult, ContentFlag, ContentCategory, GuardianMetrics, CircuitBreakerState, CostTracking, GuardianEventType, GuardianEvent, GuardianEventHandler, InterceptedRequest, InterceptedResponse, InterceptedToolCall, InterceptedAgentStep, FrameworkType, FrameworkIntegrationConfig, } from './guardian';
|
|
27
|
+
export type { ArtemisKitMatchers, MatcherResult } from './matchers';
|
|
28
|
+
export type { ProviderName, ExpectationType, AnyResult, ExtractRunCases, ExtractRedTeamCases, ExtractStressResults, ExtractManifest, DeepPartial, PartialScenario, RequireFields, OptionalFields, StrictAdapterConfig, } from './utils';
|
|
29
|
+
export type { AdapterContract, EvaluatorContract, StorageContract, AdapterFactory, EvaluatorFactory, StorageFactory, ArtemisKitPlugin, } from './contracts';
|
|
30
|
+
export type { ScenarioBuilder, TestCaseBuilder } from './builders';
|
|
31
|
+
//# sourceMappingURL=types-only.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-only.d.ts","sourceRoot":"","sources":["../src/types-only.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAMH,YAAY,EAEV,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,aAAa,EAEb,SAAS,EACT,aAAa,EACb,YAAY,EAEZ,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,yBAAyB,EACzB,4BAA4B,EAC5B,0BAA0B,EAE1B,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,2BAA2B,EAC3B,8BAA8B,EAC9B,4BAA4B,EAE5B,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,UAAU,EACV,iBAAiB,EAEjB,WAAW,EACX,SAAS,EACT,UAAU,EAEV,eAAe,EACf,aAAa,EACb,cAAc,EACd,iBAAiB,EACjB,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,WAAW,EACX,cAAc,EACd,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,OAAO,EACP,cAAc,GACf,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAEV,WAAW,EACX,iBAAiB,EAEjB,eAAe,EACf,cAAc,EACd,UAAU,EAEV,WAAW,EAEX,kBAAkB,EAClB,cAAc,EACd,QAAQ,EAER,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,YAAY,GACb,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMrF,YAAY,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EAEtB,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,kBAAkB,CAAC;AAM1B,YAAY,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAMxD,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAM7E,YAAY,EAEV,cAAc,EACd,iBAAiB,EACjB,qBAAqB,EACrB,sBAAsB,EACtB,gBAAgB,EAChB,iBAAiB,EAEjB,YAAY,EACZ,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EAEf,UAAU,EACV,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,eAAe,EAEf,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EAEnB,oBAAoB,EACpB,cAAc,EAEd,YAAY,EACZ,OAAO,EACP,WAAW,EACX,kBAAkB,EAClB,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EAEf,eAAe,EACf,mBAAmB,EACnB,YAAY,EAEZ,iBAAiB,EACjB,aAAa,EACb,oBAAoB,EAEpB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EAEpB,aAAa,EACb,0BAA0B,GAC3B,MAAM,YAAY,CAAC;AAMpB,YAAY,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMpE,YAAY,EAEV,YAAY,EACZ,eAAe,EAEf,SAAS,EACT,eAAe,EACf,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EAEf,WAAW,EACX,eAAe,EACf,aAAa,EACb,cAAc,EACd,mBAAmB,GACpB,MAAM,SAAS,CAAC;AAMjB,YAAY,EACV,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
// @bun
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @artemiskit/sdk
|
|
3
|
+
* Type definitions for the programmatic SDK
|
|
4
|
+
*/
|
|
5
|
+
import type { AdapterConfig, CaseResult, ModelClient, RedTeamManifest, RunManifest, StorageConfig, StressManifest, StressRequestResult } from '@artemiskit/core';
|
|
6
|
+
import type { RedactionConfig } from '@artemiskit/core';
|
|
7
|
+
import type { Scenario } from '@artemiskit/core';
|
|
8
|
+
/**
|
|
9
|
+
* Event emitted when a test case starts
|
|
10
|
+
*/
|
|
11
|
+
export interface CaseStartEvent {
|
|
12
|
+
caseId: string;
|
|
13
|
+
caseName?: string;
|
|
14
|
+
index: number;
|
|
15
|
+
total: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Event emitted when a test case completes
|
|
19
|
+
*/
|
|
20
|
+
export interface CaseCompleteEvent {
|
|
21
|
+
result: CaseResult;
|
|
22
|
+
index: number;
|
|
23
|
+
total: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Event emitted for progress updates
|
|
27
|
+
*/
|
|
28
|
+
export interface ProgressEvent {
|
|
29
|
+
message: string;
|
|
30
|
+
phase: 'setup' | 'running' | 'teardown';
|
|
31
|
+
progress?: number;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Event emitted when a red team mutation starts
|
|
35
|
+
*/
|
|
36
|
+
export interface RedTeamMutationStartEvent {
|
|
37
|
+
mutation: string;
|
|
38
|
+
caseId: string;
|
|
39
|
+
index: number;
|
|
40
|
+
total: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Event emitted when a red team mutation completes
|
|
44
|
+
*/
|
|
45
|
+
export interface RedTeamMutationCompleteEvent {
|
|
46
|
+
mutation: string;
|
|
47
|
+
caseId: string;
|
|
48
|
+
status: 'safe' | 'unsafe' | 'blocked' | 'error';
|
|
49
|
+
severity: 'none' | 'low' | 'medium' | 'high' | 'critical';
|
|
50
|
+
index: number;
|
|
51
|
+
total: number;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Event emitted for stress test request completion
|
|
55
|
+
*/
|
|
56
|
+
export interface StressRequestCompleteEvent {
|
|
57
|
+
result: StressRequestResult;
|
|
58
|
+
index: number;
|
|
59
|
+
total: number;
|
|
60
|
+
currentRPS: number;
|
|
61
|
+
}
|
|
62
|
+
export type CaseStartHandler = (event: CaseStartEvent) => void;
|
|
63
|
+
export type CaseCompleteHandler = (event: CaseCompleteEvent) => void;
|
|
64
|
+
export type ProgressHandler = (event: ProgressEvent) => void;
|
|
65
|
+
export type RedTeamMutationStartHandler = (event: RedTeamMutationStartEvent) => void;
|
|
66
|
+
export type RedTeamMutationCompleteHandler = (event: RedTeamMutationCompleteEvent) => void;
|
|
67
|
+
export type StressRequestCompleteHandler = (event: StressRequestCompleteEvent) => void;
|
|
68
|
+
/**
|
|
69
|
+
* SDK-level configuration options
|
|
70
|
+
*/
|
|
71
|
+
export interface ArtemisKitConfig {
|
|
72
|
+
/** Project name for manifest grouping */
|
|
73
|
+
project?: string;
|
|
74
|
+
/** Default provider to use */
|
|
75
|
+
provider?: AdapterConfig['provider'];
|
|
76
|
+
/** Default model to use */
|
|
77
|
+
model?: string;
|
|
78
|
+
/** Provider-specific configuration */
|
|
79
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
80
|
+
/** Default redaction settings */
|
|
81
|
+
redaction?: RedactionConfig;
|
|
82
|
+
/** Default timeout per case in milliseconds */
|
|
83
|
+
timeout?: number;
|
|
84
|
+
/** Default number of retries per case */
|
|
85
|
+
retries?: number;
|
|
86
|
+
/** Default concurrency for parallel execution */
|
|
87
|
+
concurrency?: number;
|
|
88
|
+
/**
|
|
89
|
+
* Storage configuration for persisting and loading run manifests
|
|
90
|
+
* Required for compare() method to load historical runs
|
|
91
|
+
* @since 0.3.2
|
|
92
|
+
*/
|
|
93
|
+
storage?: StorageConfig;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Options for running test scenarios
|
|
97
|
+
*/
|
|
98
|
+
export interface RunOptions {
|
|
99
|
+
/** Path to scenario file or inline Scenario object */
|
|
100
|
+
scenario: string | Scenario;
|
|
101
|
+
/** Override provider */
|
|
102
|
+
provider?: AdapterConfig['provider'];
|
|
103
|
+
/** Override model */
|
|
104
|
+
model?: string;
|
|
105
|
+
/** Provider-specific configuration override */
|
|
106
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
107
|
+
/** Pre-configured model client (skips adapter creation) */
|
|
108
|
+
client?: ModelClient;
|
|
109
|
+
/** Filter test cases by tags */
|
|
110
|
+
tags?: string[];
|
|
111
|
+
/** Number of concurrent requests */
|
|
112
|
+
concurrency?: number;
|
|
113
|
+
/** Timeout per case in milliseconds */
|
|
114
|
+
timeout?: number;
|
|
115
|
+
/** Number of retries per case */
|
|
116
|
+
retries?: number;
|
|
117
|
+
/** Redaction configuration */
|
|
118
|
+
redaction?: RedactionConfig;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Options for red team testing
|
|
122
|
+
*/
|
|
123
|
+
export interface RedTeamOptions {
|
|
124
|
+
/** Path to scenario file or inline Scenario object */
|
|
125
|
+
scenario: string | Scenario;
|
|
126
|
+
/** Override provider */
|
|
127
|
+
provider?: AdapterConfig['provider'];
|
|
128
|
+
/** Override model */
|
|
129
|
+
model?: string;
|
|
130
|
+
/** Provider-specific configuration override */
|
|
131
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
132
|
+
/** Pre-configured model client (skips adapter creation) */
|
|
133
|
+
client?: ModelClient;
|
|
134
|
+
/** Mutations to apply (default: all) */
|
|
135
|
+
mutations?: string[];
|
|
136
|
+
/** Number of mutations per case */
|
|
137
|
+
countPerCase?: number;
|
|
138
|
+
/** Filter test cases by tags */
|
|
139
|
+
tags?: string[];
|
|
140
|
+
/** Timeout per case in milliseconds */
|
|
141
|
+
timeout?: number;
|
|
142
|
+
/** Redaction configuration */
|
|
143
|
+
redaction?: RedactionConfig;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Options for stress testing
|
|
147
|
+
*/
|
|
148
|
+
export interface StressOptions {
|
|
149
|
+
/** Path to scenario file or inline Scenario object */
|
|
150
|
+
scenario: string | Scenario;
|
|
151
|
+
/** Override provider */
|
|
152
|
+
provider?: AdapterConfig['provider'];
|
|
153
|
+
/** Override model */
|
|
154
|
+
model?: string;
|
|
155
|
+
/** Provider-specific configuration override */
|
|
156
|
+
providerConfig?: Partial<AdapterConfig>;
|
|
157
|
+
/** Pre-configured model client (skips adapter creation) */
|
|
158
|
+
client?: ModelClient;
|
|
159
|
+
/** Number of concurrent requests */
|
|
160
|
+
concurrency?: number;
|
|
161
|
+
/** Test duration in seconds */
|
|
162
|
+
duration?: number;
|
|
163
|
+
/** Ramp-up period in seconds */
|
|
164
|
+
rampUp?: number;
|
|
165
|
+
/** Maximum number of requests (optional limit) */
|
|
166
|
+
maxRequests?: number;
|
|
167
|
+
/** Redaction configuration */
|
|
168
|
+
redaction?: RedactionConfig;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Result from a test scenario run
|
|
172
|
+
*/
|
|
173
|
+
export interface RunResult {
|
|
174
|
+
/** The generated manifest */
|
|
175
|
+
manifest: RunManifest;
|
|
176
|
+
/** Individual case results */
|
|
177
|
+
cases: CaseResult[];
|
|
178
|
+
/** Whether all cases passed */
|
|
179
|
+
success: boolean;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Result from a red team test run
|
|
183
|
+
*/
|
|
184
|
+
export interface RedTeamResult {
|
|
185
|
+
/** The generated manifest */
|
|
186
|
+
manifest: RedTeamManifest;
|
|
187
|
+
/** Whether the defense rate is acceptable (>= threshold) */
|
|
188
|
+
success: boolean;
|
|
189
|
+
/** Defense rate (0-1) */
|
|
190
|
+
defenseRate: number;
|
|
191
|
+
/** Count of unsafe responses */
|
|
192
|
+
unsafeCount: number;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Result from a stress test run
|
|
196
|
+
*/
|
|
197
|
+
export interface StressResult {
|
|
198
|
+
/** The generated manifest */
|
|
199
|
+
manifest: StressManifest;
|
|
200
|
+
/** Whether the test passed success rate threshold */
|
|
201
|
+
success: boolean;
|
|
202
|
+
/** Success rate (0-1) */
|
|
203
|
+
successRate: number;
|
|
204
|
+
/** Requests per second achieved */
|
|
205
|
+
rps: number;
|
|
206
|
+
/** P95 latency in milliseconds */
|
|
207
|
+
p95LatencyMs: number;
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Event types supported by ArtemisKit
|
|
211
|
+
*/
|
|
212
|
+
export interface ArtemisKitEvents {
|
|
213
|
+
caseStart: CaseStartEvent;
|
|
214
|
+
caseComplete: CaseCompleteEvent;
|
|
215
|
+
progress: ProgressEvent;
|
|
216
|
+
redteamMutationStart: RedTeamMutationStartEvent;
|
|
217
|
+
redteamMutationComplete: RedTeamMutationCompleteEvent;
|
|
218
|
+
stressRequestComplete: StressRequestCompleteEvent;
|
|
219
|
+
}
|
|
220
|
+
export type ArtemisKitEventName = keyof ArtemisKitEvents;
|
|
221
|
+
/**
|
|
222
|
+
* Options for validating scenario files
|
|
223
|
+
*/
|
|
224
|
+
export interface ValidateOptions {
|
|
225
|
+
/**
|
|
226
|
+
* Scenario file path(s) or glob pattern
|
|
227
|
+
* Can be a single path, array of paths, or glob pattern
|
|
228
|
+
*/
|
|
229
|
+
scenario: string | string[];
|
|
230
|
+
/**
|
|
231
|
+
* Strict mode - fail on warnings as well as errors
|
|
232
|
+
* @default false
|
|
233
|
+
*/
|
|
234
|
+
strict?: boolean;
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Validation error details
|
|
238
|
+
*/
|
|
239
|
+
export interface ValidationError {
|
|
240
|
+
/** File path where error occurred */
|
|
241
|
+
file: string;
|
|
242
|
+
/** Error message */
|
|
243
|
+
message: string;
|
|
244
|
+
/** Line number if available */
|
|
245
|
+
line?: number;
|
|
246
|
+
/** Column number if available */
|
|
247
|
+
column?: number;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Validation warning details
|
|
251
|
+
*/
|
|
252
|
+
export interface ValidationWarning {
|
|
253
|
+
/** File path where warning occurred */
|
|
254
|
+
file: string;
|
|
255
|
+
/** Warning message */
|
|
256
|
+
message: string;
|
|
257
|
+
/** Line number if available */
|
|
258
|
+
line?: number;
|
|
259
|
+
/** Column number if available */
|
|
260
|
+
column?: number;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Per-scenario validation result
|
|
264
|
+
*/
|
|
265
|
+
export interface ScenarioValidation {
|
|
266
|
+
/** File path */
|
|
267
|
+
file: string;
|
|
268
|
+
/** Scenario name */
|
|
269
|
+
name: string;
|
|
270
|
+
/** Number of test cases */
|
|
271
|
+
caseCount: number;
|
|
272
|
+
/** Whether this scenario is valid */
|
|
273
|
+
valid: boolean;
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Result from validate() method
|
|
277
|
+
*/
|
|
278
|
+
export interface ValidationResult {
|
|
279
|
+
/** Overall validation passed */
|
|
280
|
+
valid: boolean;
|
|
281
|
+
/** Individual scenario validation results */
|
|
282
|
+
scenarios: ScenarioValidation[];
|
|
283
|
+
/** Validation errors */
|
|
284
|
+
errors: ValidationError[];
|
|
285
|
+
/** Validation warnings (informational) */
|
|
286
|
+
warnings: ValidationWarning[];
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Options for comparing test runs
|
|
290
|
+
*/
|
|
291
|
+
export interface CompareOptions {
|
|
292
|
+
/**
|
|
293
|
+
* Baseline run identifier
|
|
294
|
+
* Can be a run ID, 'latest', or a named baseline
|
|
295
|
+
*/
|
|
296
|
+
baseline: string;
|
|
297
|
+
/**
|
|
298
|
+
* Current run identifier
|
|
299
|
+
* Can be a run ID
|
|
300
|
+
*/
|
|
301
|
+
current: string;
|
|
302
|
+
/**
|
|
303
|
+
* Regression threshold (0-1)
|
|
304
|
+
* A regression is detected if success rate drops by more than this amount
|
|
305
|
+
* @default 0.05 (5%)
|
|
306
|
+
*/
|
|
307
|
+
threshold?: number;
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Summary of a run for comparison
|
|
311
|
+
*/
|
|
312
|
+
export interface RunSummary {
|
|
313
|
+
/** Run identifier */
|
|
314
|
+
runId: string;
|
|
315
|
+
/** Success rate (0-1) */
|
|
316
|
+
successRate: number;
|
|
317
|
+
/** Total number of test cases */
|
|
318
|
+
totalCases: number;
|
|
319
|
+
/** Number of passed cases */
|
|
320
|
+
passedCases: number;
|
|
321
|
+
/** Number of failed cases */
|
|
322
|
+
failedCases: number;
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Comparison details between two runs
|
|
326
|
+
*/
|
|
327
|
+
export interface ComparisonDetails {
|
|
328
|
+
/** Change in success rate (current - baseline) */
|
|
329
|
+
successRateDelta: number;
|
|
330
|
+
/** Cases that passed in baseline but failed in current */
|
|
331
|
+
newFailures: Array<{
|
|
332
|
+
caseId: string;
|
|
333
|
+
caseName?: string;
|
|
334
|
+
baselineStatus: string;
|
|
335
|
+
currentStatus: string;
|
|
336
|
+
}>;
|
|
337
|
+
/** Cases that failed in baseline but passed in current */
|
|
338
|
+
newPasses: Array<{
|
|
339
|
+
caseId: string;
|
|
340
|
+
caseName?: string;
|
|
341
|
+
baselineStatus: string;
|
|
342
|
+
currentStatus: string;
|
|
343
|
+
}>;
|
|
344
|
+
/** Cases with unchanged status */
|
|
345
|
+
unchanged: Array<{
|
|
346
|
+
caseId: string;
|
|
347
|
+
status: string;
|
|
348
|
+
}>;
|
|
349
|
+
/** Cases that exist in current run but not in baseline (new test cases) */
|
|
350
|
+
addedCases: Array<{
|
|
351
|
+
caseId: string;
|
|
352
|
+
caseName?: string;
|
|
353
|
+
status: string;
|
|
354
|
+
}>;
|
|
355
|
+
/** Cases that exist in baseline but not in current run (removed test cases) */
|
|
356
|
+
removedCases: Array<{
|
|
357
|
+
caseId: string;
|
|
358
|
+
caseName?: string;
|
|
359
|
+
status: string;
|
|
360
|
+
}>;
|
|
361
|
+
}
|
|
362
|
+
/**
|
|
363
|
+
* Result from compare() method
|
|
364
|
+
*/
|
|
365
|
+
export interface CompareResult {
|
|
366
|
+
/** Baseline run summary */
|
|
367
|
+
baseline: RunSummary;
|
|
368
|
+
/** Current run summary */
|
|
369
|
+
current: RunSummary;
|
|
370
|
+
/** Detailed comparison */
|
|
371
|
+
comparison: ComparisonDetails;
|
|
372
|
+
/** Whether a regression was detected */
|
|
373
|
+
hasRegression: boolean;
|
|
374
|
+
/** Threshold used for regression detection */
|
|
375
|
+
threshold: number;
|
|
376
|
+
}
|
|
377
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,UAAU,EACV,WAAW,EACX,eAAe,EACf,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAMjD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAAC;IACxC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,OAAO,CAAC;IAChD,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,CAAC;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;AAC/D,MAAM,MAAM,mBAAmB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;AACrE,MAAM,MAAM,eAAe,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAC7D,MAAM,MAAM,2BAA2B,GAAG,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;AACrF,MAAM,MAAM,8BAA8B,GAAG,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;AAC3F,MAAM,MAAM,4BAA4B,GAAG,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;AAMvF;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,2BAA2B;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,iCAAiC;IACjC,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,mCAAmC;IACnC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,sDAAsD;IACtD,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC5B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,qBAAqB;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,+CAA+C;IAC/C,cAAc,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,2DAA2D;IAC3D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+BAA+B;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kDAAkD;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,eAAe,CAAC;CAC7B;AAMD;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,QAAQ,EAAE,WAAW,CAAC;IACtB,8BAA8B;IAC9B,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,6BAA6B;IAC7B,QAAQ,EAAE,eAAe,CAAC;IAC1B,4DAA4D;IAC5D,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,6BAA6B;IAC7B,QAAQ,EAAE,cAAc,CAAC;IACzB,qDAAqD;IACrD,OAAO,EAAE,OAAO,CAAC;IACjB,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,iBAAiB,CAAC;IAChC,QAAQ,EAAE,aAAa,CAAC;IACxB,oBAAoB,EAAE,yBAAyB,CAAC;IAChD,uBAAuB,EAAE,4BAA4B,CAAC;IACtD,qBAAqB,EAAE,0BAA0B,CAAC;CACnD;AAED,MAAM,MAAM,mBAAmB,GAAG,MAAM,gBAAgB,CAAC;AAMzD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAE5B;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,KAAK,EAAE,OAAO,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gCAAgC;IAChC,KAAK,EAAE,OAAO,CAAC;IACf,6CAA6C;IAC7C,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,wBAAwB;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,0CAA0C;IAC1C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAMD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,qBAAqB;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,yBAAyB;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IACnB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,kDAAkD;IAClD,gBAAgB,EAAE,MAAM,CAAC;IACzB,0DAA0D;IAC1D,WAAW,EAAE,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,0DAA0D;IAC1D,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC,CAAC;IACH,kCAAkC;IAClC,SAAS,EAAE,KAAK,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,2EAA2E;IAC3E,UAAU,EAAE,KAAK,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;IACH,+EAA+E;IAC/E,YAAY,EAAE,KAAK,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,2BAA2B;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,0BAA0B;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,0BAA0B;IAC1B,UAAU,EAAE,iBAAiB,CAAC;IAC9B,wCAAwC;IACxC,aAAa,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB"}
|