@artemiskit/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +229 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +352 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +354 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +53 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +44918 -28644
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/index.js +13 -59
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/jest.js +0 -46
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/matchers/vitest.js +0 -46
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +379 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +58 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +450 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +310 -0
- package/src/artemiskit.ts +479 -5
- package/src/builders/index.ts +700 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +186 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
- package/adapters/openai/dist/index.js +0 -5625
package/src/artemiskit.ts
CHANGED
|
@@ -5,27 +5,36 @@
|
|
|
5
5
|
|
|
6
6
|
import {
|
|
7
7
|
type AdapterConfig,
|
|
8
|
+
type AnyManifest,
|
|
8
9
|
type ModelClient,
|
|
9
10
|
type RedTeamCaseResult,
|
|
10
11
|
type RedTeamManifest,
|
|
11
12
|
type RedTeamSeverity,
|
|
13
|
+
type RunManifest,
|
|
14
|
+
type StorageAdapter,
|
|
12
15
|
type StressManifest,
|
|
13
16
|
type StressMetrics,
|
|
14
17
|
type StressRequestResult,
|
|
15
18
|
runScenario as coreRunScenario,
|
|
16
19
|
createAdapter,
|
|
20
|
+
createStorageAdapter,
|
|
17
21
|
getGitInfo,
|
|
18
22
|
parseScenarioFile,
|
|
23
|
+
resolveScenarioPaths,
|
|
19
24
|
} from '@artemiskit/core';
|
|
20
25
|
import type { Scenario } from '@artemiskit/core';
|
|
21
26
|
import {
|
|
27
|
+
AgentConfusionMutation,
|
|
28
|
+
ChainManipulationMutation,
|
|
22
29
|
CotInjectionMutation,
|
|
23
30
|
EncodingMutation,
|
|
24
31
|
InstructionFlipMutation,
|
|
32
|
+
MemoryPoisoningMutation,
|
|
25
33
|
MultiTurnMutation,
|
|
26
34
|
type Mutation,
|
|
27
35
|
RedTeamGenerator,
|
|
28
36
|
RoleSpoofMutation,
|
|
37
|
+
ToolAbuseMutation,
|
|
29
38
|
TypoMutation,
|
|
30
39
|
UnsafeResponseDetector,
|
|
31
40
|
} from '@artemiskit/redteam';
|
|
@@ -37,6 +46,8 @@ import type {
|
|
|
37
46
|
ArtemisKitEvents,
|
|
38
47
|
CaseCompleteHandler,
|
|
39
48
|
CaseStartHandler,
|
|
49
|
+
CompareOptions,
|
|
50
|
+
CompareResult,
|
|
40
51
|
ProgressHandler,
|
|
41
52
|
RedTeamMutationCompleteHandler,
|
|
42
53
|
RedTeamMutationStartHandler,
|
|
@@ -47,18 +58,26 @@ import type {
|
|
|
47
58
|
StressOptions,
|
|
48
59
|
StressRequestCompleteHandler,
|
|
49
60
|
StressResult,
|
|
61
|
+
ValidateOptions,
|
|
62
|
+
ValidationResult,
|
|
50
63
|
} from './types';
|
|
51
64
|
|
|
52
65
|
/**
|
|
53
66
|
* Available mutation name to class mapping
|
|
54
67
|
*/
|
|
55
68
|
const MUTATION_MAP: Record<string, new () => Mutation> = {
|
|
69
|
+
// Core mutations (v0.1.x - v0.2.x)
|
|
56
70
|
typo: TypoMutation,
|
|
57
71
|
'role-spoof': RoleSpoofMutation,
|
|
58
72
|
'instruction-flip': InstructionFlipMutation,
|
|
59
73
|
'cot-injection': CotInjectionMutation,
|
|
60
74
|
encoding: EncodingMutation,
|
|
61
75
|
'multi-turn': MultiTurnMutation,
|
|
76
|
+
// Agent-specific mutations (v0.3.1)
|
|
77
|
+
'agent-confusion': AgentConfusionMutation,
|
|
78
|
+
'tool-abuse': ToolAbuseMutation,
|
|
79
|
+
'memory-poisoning': MemoryPoisoningMutation,
|
|
80
|
+
'chain-manipulation': ChainManipulationMutation,
|
|
62
81
|
};
|
|
63
82
|
|
|
64
83
|
type AnyEventHandler = (event: unknown) => void;
|
|
@@ -83,6 +102,7 @@ type AnyEventHandler = (event: unknown) => void;
|
|
|
83
102
|
export class ArtemisKit {
|
|
84
103
|
private config: ArtemisKitConfig;
|
|
85
104
|
private eventHandlers: Map<ArtemisKitEventName, Set<AnyEventHandler>> = new Map();
|
|
105
|
+
private storage: StorageAdapter | null = null;
|
|
86
106
|
|
|
87
107
|
constructor(config: ArtemisKitConfig = {}) {
|
|
88
108
|
this.config = {
|
|
@@ -94,7 +114,13 @@ export class ArtemisKit {
|
|
|
94
114
|
timeout: config.timeout,
|
|
95
115
|
retries: config.retries ?? 0,
|
|
96
116
|
concurrency: config.concurrency ?? 1,
|
|
117
|
+
storage: config.storage,
|
|
97
118
|
};
|
|
119
|
+
|
|
120
|
+
// Initialize storage adapter if configured
|
|
121
|
+
if (config.storage) {
|
|
122
|
+
this.storage = createStorageAdapter(config.storage);
|
|
123
|
+
}
|
|
98
124
|
}
|
|
99
125
|
|
|
100
126
|
// ==========================================================================
|
|
@@ -244,6 +270,7 @@ export class ArtemisKit {
|
|
|
244
270
|
timeout: options.timeout ?? this.config.timeout,
|
|
245
271
|
retries: options.retries ?? this.config.retries,
|
|
246
272
|
redaction: options.redaction ?? this.config.redaction,
|
|
273
|
+
toolExecutor: options.toolExecutor,
|
|
247
274
|
onCaseComplete: (caseResult, index, total) => {
|
|
248
275
|
this.emit('caseComplete', { result: caseResult, index, total });
|
|
249
276
|
this.emit('progress', {
|
|
@@ -713,6 +740,443 @@ export class ArtemisKit {
|
|
|
713
740
|
};
|
|
714
741
|
}
|
|
715
742
|
|
|
743
|
+
// ==========================================================================
|
|
744
|
+
// Validation & Comparison Methods
|
|
745
|
+
// ==========================================================================
|
|
746
|
+
|
|
747
|
+
/**
|
|
748
|
+
* Validate scenario files without execution (pre-flight checks)
|
|
749
|
+
*
|
|
750
|
+
* Use this for CI/CD pipelines to catch configuration errors before running tests.
|
|
751
|
+
*
|
|
752
|
+
* @example
|
|
753
|
+
* ```typescript
|
|
754
|
+
* const result = await kit.validate({ scenario: './scenarios/**\/*.yaml' });
|
|
755
|
+
* if (!result.valid) {
|
|
756
|
+
* console.error('Validation errors:', result.errors);
|
|
757
|
+
* process.exit(1);
|
|
758
|
+
* }
|
|
759
|
+
* ```
|
|
760
|
+
*/
|
|
761
|
+
async validate(options: ValidateOptions): Promise<ValidationResult> {
|
|
762
|
+
const inputPaths = Array.isArray(options.scenario) ? options.scenario : [options.scenario];
|
|
763
|
+
const strict = options.strict ?? false;
|
|
764
|
+
|
|
765
|
+
const errors: Array<{
|
|
766
|
+
file: string;
|
|
767
|
+
message: string;
|
|
768
|
+
line?: number;
|
|
769
|
+
column?: number;
|
|
770
|
+
}> = [];
|
|
771
|
+
|
|
772
|
+
const warnings: Array<{
|
|
773
|
+
file: string;
|
|
774
|
+
message: string;
|
|
775
|
+
line?: number;
|
|
776
|
+
column?: number;
|
|
777
|
+
}> = [];
|
|
778
|
+
|
|
779
|
+
const scenarios: Array<{
|
|
780
|
+
file: string;
|
|
781
|
+
name: string;
|
|
782
|
+
caseCount: number;
|
|
783
|
+
valid: boolean;
|
|
784
|
+
}> = [];
|
|
785
|
+
|
|
786
|
+
// Resolve glob patterns and directories to actual file paths
|
|
787
|
+
const resolvedPaths: string[] = [];
|
|
788
|
+
for (const inputPath of inputPaths) {
|
|
789
|
+
try {
|
|
790
|
+
const paths = await resolveScenarioPaths(inputPath);
|
|
791
|
+
resolvedPaths.push(...paths);
|
|
792
|
+
} catch (error) {
|
|
793
|
+
errors.push({
|
|
794
|
+
file: inputPath,
|
|
795
|
+
message: `Failed to resolve path: ${(error as Error).message}`,
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
this.emit('progress', {
|
|
801
|
+
message: `Validating ${resolvedPaths.length} scenario file(s)...`,
|
|
802
|
+
phase: 'setup',
|
|
803
|
+
progress: 0,
|
|
804
|
+
});
|
|
805
|
+
|
|
806
|
+
for (let i = 0; i < resolvedPaths.length; i++) {
|
|
807
|
+
const scenarioPath = resolvedPaths[i];
|
|
808
|
+
|
|
809
|
+
try {
|
|
810
|
+
// Parse and validate scenario file
|
|
811
|
+
const scenario = await parseScenarioFile(scenarioPath);
|
|
812
|
+
|
|
813
|
+
// Check for common issues
|
|
814
|
+
const fileWarnings: typeof warnings = [];
|
|
815
|
+
const fileErrors: typeof errors = [];
|
|
816
|
+
|
|
817
|
+
// Validate cases
|
|
818
|
+
if (!scenario.cases || scenario.cases.length === 0) {
|
|
819
|
+
fileErrors.push({
|
|
820
|
+
file: scenarioPath,
|
|
821
|
+
message: 'Scenario has no test cases defined',
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
// Validate each case
|
|
826
|
+
for (let j = 0; j < (scenario.cases ?? []).length; j++) {
|
|
827
|
+
const testCase = scenario.cases[j];
|
|
828
|
+
|
|
829
|
+
if (!testCase.id) {
|
|
830
|
+
fileErrors.push({
|
|
831
|
+
file: scenarioPath,
|
|
832
|
+
message: `Case at index ${j} is missing required 'id' field`,
|
|
833
|
+
});
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
if (!testCase.prompt) {
|
|
837
|
+
fileErrors.push({
|
|
838
|
+
file: scenarioPath,
|
|
839
|
+
message: `Case '${testCase.id ?? j}' is missing required 'prompt' field`,
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
// Check for empty expectations (warning)
|
|
844
|
+
if (!testCase.expected) {
|
|
845
|
+
fileWarnings.push({
|
|
846
|
+
file: scenarioPath,
|
|
847
|
+
message: `Case '${testCase.id ?? j}' has no expectations defined - will only check for LLM response`,
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
// Validate expectation type
|
|
852
|
+
if (testCase.expected) {
|
|
853
|
+
const validTypes = [
|
|
854
|
+
'contains',
|
|
855
|
+
'not_contains',
|
|
856
|
+
'exact',
|
|
857
|
+
'regex',
|
|
858
|
+
'fuzzy',
|
|
859
|
+
'similarity',
|
|
860
|
+
'llm_grader',
|
|
861
|
+
'json_schema',
|
|
862
|
+
'combined',
|
|
863
|
+
'inline',
|
|
864
|
+
];
|
|
865
|
+
if (!validTypes.includes(testCase.expected.type)) {
|
|
866
|
+
fileErrors.push({
|
|
867
|
+
file: scenarioPath,
|
|
868
|
+
message: `Case '${testCase.id ?? j}' has invalid expectation type '${testCase.expected.type}'`,
|
|
869
|
+
});
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
// Check for duplicate case IDs
|
|
875
|
+
const caseIds = (scenario.cases ?? []).map((c) => c.id).filter(Boolean);
|
|
876
|
+
const seenIds = new Set<string>();
|
|
877
|
+
for (const id of caseIds) {
|
|
878
|
+
if (seenIds.has(id)) {
|
|
879
|
+
fileErrors.push({
|
|
880
|
+
file: scenarioPath,
|
|
881
|
+
message: `Duplicate case ID '${id}' found in scenario`,
|
|
882
|
+
});
|
|
883
|
+
}
|
|
884
|
+
seenIds.add(id);
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
// Check provider/model configuration
|
|
888
|
+
if (!scenario.provider && !this.config.provider) {
|
|
889
|
+
fileWarnings.push({
|
|
890
|
+
file: scenarioPath,
|
|
891
|
+
message: 'No provider specified in scenario or ArtemisKit config',
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
if (!scenario.model && !this.config.model) {
|
|
896
|
+
fileWarnings.push({
|
|
897
|
+
file: scenarioPath,
|
|
898
|
+
message: 'No model specified in scenario or ArtemisKit config',
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
errors.push(...fileErrors);
|
|
903
|
+
warnings.push(...fileWarnings);
|
|
904
|
+
|
|
905
|
+
scenarios.push({
|
|
906
|
+
file: scenarioPath,
|
|
907
|
+
name: scenario.name ?? scenarioPath,
|
|
908
|
+
caseCount: scenario.cases?.length ?? 0,
|
|
909
|
+
valid: fileErrors.length === 0,
|
|
910
|
+
});
|
|
911
|
+
} catch (error) {
|
|
912
|
+
errors.push({
|
|
913
|
+
file: scenarioPath,
|
|
914
|
+
message: `Failed to parse scenario: ${(error as Error).message}`,
|
|
915
|
+
});
|
|
916
|
+
|
|
917
|
+
scenarios.push({
|
|
918
|
+
file: scenarioPath,
|
|
919
|
+
name: scenarioPath,
|
|
920
|
+
caseCount: 0,
|
|
921
|
+
valid: false,
|
|
922
|
+
});
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
this.emit('progress', {
|
|
926
|
+
message: `Validated ${i + 1}/${resolvedPaths.length} scenarios`,
|
|
927
|
+
phase: 'running',
|
|
928
|
+
progress: Math.round(((i + 1) / resolvedPaths.length) * 100),
|
|
929
|
+
});
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
const valid = errors.length === 0 && (!strict || warnings.length === 0);
|
|
933
|
+
|
|
934
|
+
this.emit('progress', {
|
|
935
|
+
message: `Validation ${valid ? 'passed' : 'failed'}: ${errors.length} error(s), ${warnings.length} warning(s)`,
|
|
936
|
+
phase: 'teardown',
|
|
937
|
+
progress: 100,
|
|
938
|
+
});
|
|
939
|
+
|
|
940
|
+
return {
|
|
941
|
+
valid,
|
|
942
|
+
scenarios,
|
|
943
|
+
errors,
|
|
944
|
+
warnings,
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
/**
|
|
949
|
+
* Compare two test runs for regression detection
|
|
950
|
+
*
|
|
951
|
+
* Requires storage configuration to be set in ArtemisKitConfig.
|
|
952
|
+
*
|
|
953
|
+
* @example
|
|
954
|
+
* ```typescript
|
|
955
|
+
* const kit = new ArtemisKit({
|
|
956
|
+
* storage: { type: 'local', basePath: './artemis-runs' }
|
|
957
|
+
* });
|
|
958
|
+
*
|
|
959
|
+
* const comparison = await kit.compare({
|
|
960
|
+
* baseline: 'run_abc123',
|
|
961
|
+
* current: 'run_def456',
|
|
962
|
+
* threshold: 0.05, // 5% regression threshold
|
|
963
|
+
* });
|
|
964
|
+
*
|
|
965
|
+
* if (comparison.hasRegression) {
|
|
966
|
+
* console.error('Regression detected!', comparison.regressions);
|
|
967
|
+
* }
|
|
968
|
+
* ```
|
|
969
|
+
*
|
|
970
|
+
* @since 0.3.2
|
|
971
|
+
*/
|
|
972
|
+
async compare(options: CompareOptions): Promise<CompareResult> {
|
|
973
|
+
const { baseline, current, threshold = 0.05 } = options;
|
|
974
|
+
|
|
975
|
+
// Validate storage is configured
|
|
976
|
+
if (!this.storage) {
|
|
977
|
+
throw new Error(
|
|
978
|
+
'Storage configuration required for compare(). ' +
|
|
979
|
+
'Set storage in ArtemisKitConfig: { storage: { type: "local", basePath: "./artemis-runs" } }'
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
this.emit('progress', {
|
|
984
|
+
message: `Comparing runs: ${baseline} vs ${current}`,
|
|
985
|
+
phase: 'setup',
|
|
986
|
+
progress: 0,
|
|
987
|
+
});
|
|
988
|
+
|
|
989
|
+
// Resolve "latest" baseline if specified
|
|
990
|
+
let baselineRunId = baseline;
|
|
991
|
+
if (baseline === 'latest') {
|
|
992
|
+
this.emit('progress', {
|
|
993
|
+
message: 'Resolving latest run as baseline...',
|
|
994
|
+
phase: 'setup',
|
|
995
|
+
progress: 10,
|
|
996
|
+
});
|
|
997
|
+
|
|
998
|
+
// StorageAdapter.list() returns runs sorted by startTime descending (most recent first)
|
|
999
|
+
// so runs[0] is guaranteed to be the latest run
|
|
1000
|
+
const runs = await this.storage.list({ limit: 1, type: 'run' });
|
|
1001
|
+
if (runs.length === 0) {
|
|
1002
|
+
throw new Error('No runs found in storage. Cannot use "latest" as baseline.');
|
|
1003
|
+
}
|
|
1004
|
+
baselineRunId = runs[0].runId;
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
this.emit('progress', {
|
|
1008
|
+
message: 'Loading manifests...',
|
|
1009
|
+
phase: 'running',
|
|
1010
|
+
progress: 20,
|
|
1011
|
+
});
|
|
1012
|
+
|
|
1013
|
+
// Load both manifests
|
|
1014
|
+
const [baselineManifest, currentManifest] = await Promise.all([
|
|
1015
|
+
this.storage.load(baselineRunId),
|
|
1016
|
+
this.storage.load(current),
|
|
1017
|
+
]);
|
|
1018
|
+
|
|
1019
|
+
this.emit('progress', {
|
|
1020
|
+
message: 'Analyzing results...',
|
|
1021
|
+
phase: 'running',
|
|
1022
|
+
progress: 50,
|
|
1023
|
+
});
|
|
1024
|
+
|
|
1025
|
+
// Extract metrics from manifests
|
|
1026
|
+
const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
|
|
1027
|
+
const currentMetrics = this.extractRunMetrics(currentManifest, current);
|
|
1028
|
+
|
|
1029
|
+
// Calculate success rate delta
|
|
1030
|
+
const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
|
|
1031
|
+
|
|
1032
|
+
this.emit('progress', {
|
|
1033
|
+
message: 'Comparing cases...',
|
|
1034
|
+
phase: 'running',
|
|
1035
|
+
progress: 70,
|
|
1036
|
+
});
|
|
1037
|
+
|
|
1038
|
+
// Compare case-by-case results
|
|
1039
|
+
const caseComparison = this.compareCases(baselineManifest, currentManifest);
|
|
1040
|
+
|
|
1041
|
+
// Determine if there's a regression
|
|
1042
|
+
// A regression is when the current success rate is lower than baseline by more than threshold
|
|
1043
|
+
const hasRegression = successRateDelta < -threshold;
|
|
1044
|
+
|
|
1045
|
+
this.emit('progress', {
|
|
1046
|
+
message: hasRegression
|
|
1047
|
+
? `Regression detected: ${(successRateDelta * 100).toFixed(1)}% change`
|
|
1048
|
+
: `No regression: ${(successRateDelta * 100).toFixed(1)}% change`,
|
|
1049
|
+
phase: 'teardown',
|
|
1050
|
+
progress: 100,
|
|
1051
|
+
});
|
|
1052
|
+
|
|
1053
|
+
return {
|
|
1054
|
+
baseline: baselineMetrics,
|
|
1055
|
+
current: currentMetrics,
|
|
1056
|
+
comparison: {
|
|
1057
|
+
successRateDelta,
|
|
1058
|
+
...caseComparison,
|
|
1059
|
+
},
|
|
1060
|
+
hasRegression,
|
|
1061
|
+
threshold,
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
/**
|
|
1066
|
+
* Extract run metrics from a manifest
|
|
1067
|
+
*/
|
|
1068
|
+
private extractRunMetrics(manifest: AnyManifest, runId: string): CompareResult['baseline'] {
|
|
1069
|
+
// Handle RunManifest
|
|
1070
|
+
if ('metrics' in manifest && 'success_rate' in manifest.metrics) {
|
|
1071
|
+
const runManifest = manifest as RunManifest;
|
|
1072
|
+
return {
|
|
1073
|
+
runId,
|
|
1074
|
+
successRate: runManifest.metrics.success_rate,
|
|
1075
|
+
totalCases: runManifest.metrics.total_cases,
|
|
1076
|
+
passedCases: runManifest.metrics.passed_cases,
|
|
1077
|
+
failedCases: runManifest.metrics.failed_cases,
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
// Throw for non-RunManifest types - compare() only supports RunManifest
|
|
1082
|
+
throw new Error(
|
|
1083
|
+
`Cannot extract metrics from run "${runId}": expected RunManifest but got a different manifest type. The compare() method only supports comparing scenario evaluation runs, not stress tests or red team runs.`
|
|
1084
|
+
);
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
/**
|
|
1088
|
+
* Compare cases between baseline and current manifests
|
|
1089
|
+
*/
|
|
1090
|
+
private compareCases(
|
|
1091
|
+
baselineManifest: AnyManifest,
|
|
1092
|
+
currentManifest: AnyManifest
|
|
1093
|
+
): Pick<
|
|
1094
|
+
CompareResult['comparison'],
|
|
1095
|
+
'newFailures' | 'newPasses' | 'unchanged' | 'addedCases' | 'removedCases'
|
|
1096
|
+
> {
|
|
1097
|
+
const newFailures: CompareResult['comparison']['newFailures'] = [];
|
|
1098
|
+
const newPasses: CompareResult['comparison']['newPasses'] = [];
|
|
1099
|
+
const unchanged: CompareResult['comparison']['unchanged'] = [];
|
|
1100
|
+
const addedCases: CompareResult['comparison']['addedCases'] = [];
|
|
1101
|
+
const removedCases: CompareResult['comparison']['removedCases'] = [];
|
|
1102
|
+
|
|
1103
|
+
// Get cases from manifests
|
|
1104
|
+
const baselineCases = this.extractCases(baselineManifest);
|
|
1105
|
+
const currentCases = this.extractCases(currentManifest);
|
|
1106
|
+
|
|
1107
|
+
// Build maps for quick lookup
|
|
1108
|
+
const baselineCaseMap = new Map(baselineCases.map((c) => [c.id, c]));
|
|
1109
|
+
const currentCaseMap = new Map(currentCases.map((c) => [c.id, c]));
|
|
1110
|
+
|
|
1111
|
+
// Find all unique case IDs
|
|
1112
|
+
const allCaseIds = new Set([...baselineCaseMap.keys(), ...currentCaseMap.keys()]);
|
|
1113
|
+
|
|
1114
|
+
for (const caseId of allCaseIds) {
|
|
1115
|
+
const baselineCase = baselineCaseMap.get(caseId);
|
|
1116
|
+
const currentCase = currentCaseMap.get(caseId);
|
|
1117
|
+
|
|
1118
|
+
// Case exists only in current run (new test case)
|
|
1119
|
+
if (!baselineCase && currentCase) {
|
|
1120
|
+
addedCases.push({
|
|
1121
|
+
caseId,
|
|
1122
|
+
caseName: currentCase.name,
|
|
1123
|
+
status: currentCase.ok ? 'passed' : 'failed',
|
|
1124
|
+
});
|
|
1125
|
+
continue;
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// Case exists only in baseline (removed test case)
|
|
1129
|
+
if (baselineCase && !currentCase) {
|
|
1130
|
+
removedCases.push({
|
|
1131
|
+
caseId,
|
|
1132
|
+
caseName: baselineCase.name,
|
|
1133
|
+
status: baselineCase.ok ? 'passed' : 'failed',
|
|
1134
|
+
});
|
|
1135
|
+
continue;
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
// Both exist - compare status
|
|
1139
|
+
if (baselineCase && currentCase) {
|
|
1140
|
+
const baselineStatus = baselineCase.ok ? 'passed' : 'failed';
|
|
1141
|
+
const currentStatus = currentCase.ok ? 'passed' : 'failed';
|
|
1142
|
+
|
|
1143
|
+
if (baselineStatus === currentStatus) {
|
|
1144
|
+
unchanged.push({ caseId, status: currentStatus });
|
|
1145
|
+
} else if (baselineStatus === 'passed' && currentStatus === 'failed') {
|
|
1146
|
+
newFailures.push({
|
|
1147
|
+
caseId,
|
|
1148
|
+
caseName: currentCase.name,
|
|
1149
|
+
baselineStatus,
|
|
1150
|
+
currentStatus,
|
|
1151
|
+
});
|
|
1152
|
+
} else if (baselineStatus === 'failed' && currentStatus === 'passed') {
|
|
1153
|
+
newPasses.push({
|
|
1154
|
+
caseId,
|
|
1155
|
+
caseName: currentCase.name,
|
|
1156
|
+
baselineStatus,
|
|
1157
|
+
currentStatus,
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
return { newFailures, newPasses, unchanged, addedCases, removedCases };
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
/**
|
|
1167
|
+
* Extract cases from a manifest
|
|
1168
|
+
*/
|
|
1169
|
+
private extractCases(manifest: AnyManifest): Array<{ id: string; name?: string; ok: boolean }> {
|
|
1170
|
+
if ('cases' in manifest && Array.isArray(manifest.cases)) {
|
|
1171
|
+
return manifest.cases.map((c) => ({
|
|
1172
|
+
id: c.id,
|
|
1173
|
+
name: c.name,
|
|
1174
|
+
ok: c.ok,
|
|
1175
|
+
}));
|
|
1176
|
+
}
|
|
1177
|
+
return [];
|
|
1178
|
+
}
|
|
1179
|
+
|
|
716
1180
|
// ==========================================================================
|
|
717
1181
|
// Helper Methods
|
|
718
1182
|
// ==========================================================================
|
|
@@ -750,19 +1214,29 @@ export class ArtemisKit {
|
|
|
750
1214
|
* Build mutation instances from mutation names
|
|
751
1215
|
*/
|
|
752
1216
|
private buildMutations(mutationNames?: string[]): Mutation[] {
|
|
753
|
-
|
|
1217
|
+
// If no mutation names specified, use all available mutations
|
|
1218
|
+
if (!mutationNames || mutationNames.length === 0) {
|
|
1219
|
+
return Object.values(MUTATION_MAP).map((MutationClass) => new MutationClass());
|
|
1220
|
+
}
|
|
1221
|
+
|
|
754
1222
|
const mutations: Mutation[] = [];
|
|
1223
|
+
const unknownMutations: string[] = [];
|
|
755
1224
|
|
|
756
|
-
for (const name of
|
|
1225
|
+
for (const name of mutationNames) {
|
|
757
1226
|
const MutationClass = MUTATION_MAP[name];
|
|
758
1227
|
if (MutationClass) {
|
|
759
1228
|
mutations.push(new MutationClass());
|
|
1229
|
+
} else {
|
|
1230
|
+
unknownMutations.push(name);
|
|
760
1231
|
}
|
|
761
1232
|
}
|
|
762
1233
|
|
|
763
|
-
//
|
|
764
|
-
if (
|
|
765
|
-
|
|
1234
|
+
// Throw error for unknown mutation names instead of silently ignoring
|
|
1235
|
+
if (unknownMutations.length > 0) {
|
|
1236
|
+
const availableMutations = Object.keys(MUTATION_MAP).join(', ');
|
|
1237
|
+
throw new Error(
|
|
1238
|
+
`Unknown mutation(s): ${unknownMutations.join(', ')}. Available mutations: ${availableMutations}`
|
|
1239
|
+
);
|
|
766
1240
|
}
|
|
767
1241
|
|
|
768
1242
|
return mutations;
|