@artemiskit/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -0
- package/README.md +44 -1
- package/dist/artemiskit.d.ts +149 -0
- package/dist/artemiskit.d.ts.map +1 -0
- package/dist/builders/index.d.ts +350 -0
- package/dist/builders/index.d.ts.map +1 -0
- package/dist/builders/index.js +398 -0
- package/dist/contracts/index.d.ts +253 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +99 -0
- package/dist/guardian/action-validator.d.ts +92 -0
- package/dist/guardian/action-validator.d.ts.map +1 -0
- package/dist/guardian/circuit-breaker.d.ts +211 -0
- package/dist/guardian/circuit-breaker.d.ts.map +1 -0
- package/dist/guardian/guardian.d.ts +213 -0
- package/dist/guardian/guardian.d.ts.map +1 -0
- package/dist/guardian/guardrails.d.ts +107 -0
- package/dist/guardian/guardrails.d.ts.map +1 -0
- package/dist/guardian/index.d.ts +35 -0
- package/dist/guardian/index.d.ts.map +1 -0
- package/dist/guardian/intent-classifier.d.ts +81 -0
- package/dist/guardian/intent-classifier.d.ts.map +1 -0
- package/dist/guardian/interceptor.d.ts +141 -0
- package/dist/guardian/interceptor.d.ts.map +1 -0
- package/dist/guardian/policy.d.ts +52 -0
- package/dist/guardian/policy.d.ts.map +1 -0
- package/dist/guardian/semantic-validator.d.ts +61 -0
- package/dist/guardian/semantic-validator.d.ts.map +1 -0
- package/dist/guardian/types.d.ts +653 -0
- package/dist/guardian/types.d.ts.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +24847 -13653
- package/dist/matchers/core.d.ts +81 -0
- package/dist/matchers/core.d.ts.map +1 -0
- package/dist/matchers/index.d.ts +6 -0
- package/dist/matchers/index.d.ts.map +1 -0
- package/dist/matchers/jest.d.ts +46 -0
- package/dist/matchers/jest.d.ts.map +1 -0
- package/dist/matchers/vitest.d.ts +24 -0
- package/dist/matchers/vitest.d.ts.map +1 -0
- package/dist/types-only.d.ts +31 -0
- package/dist/types-only.d.ts.map +1 -0
- package/dist/types-only.js +1 -0
- package/dist/types.d.ts +377 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/utils/index.d.ts +112 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +104 -0
- package/package.json +19 -4
- package/src/__tests__/builders.test.ts +445 -0
- package/src/__tests__/circuit-breaker.test.ts +704 -0
- package/src/__tests__/compare.test.ts +469 -0
- package/src/__tests__/contracts.test.ts +354 -0
- package/src/__tests__/guardian-modes.test.ts +215 -0
- package/src/__tests__/guardrails.test.ts +404 -0
- package/src/__tests__/semantic-validator.test.ts +421 -0
- package/src/__tests__/utils.test.ts +324 -0
- package/src/__tests__/validate.test.ts +307 -0
- package/src/artemiskit.ts +478 -5
- package/src/builders/index.ts +698 -0
- package/src/contracts/index.ts +357 -0
- package/src/guardian/circuit-breaker.ts +74 -28
- package/src/guardian/guardian.ts +307 -20
- package/src/guardian/guardrails.ts +143 -1
- package/src/guardian/index.ts +35 -1
- package/src/guardian/interceptor.ts +87 -11
- package/src/guardian/semantic-validator.ts +399 -0
- package/src/guardian/types.ts +413 -2
- package/src/index.ts +258 -14
- package/src/types-only.ts +276 -0
- package/src/types.ts +183 -0
- package/src/utils/index.ts +225 -0
- package/tsconfig.json +2 -1
package/src/artemiskit.ts
CHANGED
|
@@ -5,27 +5,36 @@
|
|
|
5
5
|
|
|
6
6
|
import {
|
|
7
7
|
type AdapterConfig,
|
|
8
|
+
type AnyManifest,
|
|
8
9
|
type ModelClient,
|
|
9
10
|
type RedTeamCaseResult,
|
|
10
11
|
type RedTeamManifest,
|
|
11
12
|
type RedTeamSeverity,
|
|
13
|
+
type RunManifest,
|
|
14
|
+
type StorageAdapter,
|
|
12
15
|
type StressManifest,
|
|
13
16
|
type StressMetrics,
|
|
14
17
|
type StressRequestResult,
|
|
15
18
|
runScenario as coreRunScenario,
|
|
16
19
|
createAdapter,
|
|
20
|
+
createStorageAdapter,
|
|
17
21
|
getGitInfo,
|
|
18
22
|
parseScenarioFile,
|
|
23
|
+
resolveScenarioPaths,
|
|
19
24
|
} from '@artemiskit/core';
|
|
20
25
|
import type { Scenario } from '@artemiskit/core';
|
|
21
26
|
import {
|
|
27
|
+
AgentConfusionMutation,
|
|
28
|
+
ChainManipulationMutation,
|
|
22
29
|
CotInjectionMutation,
|
|
23
30
|
EncodingMutation,
|
|
24
31
|
InstructionFlipMutation,
|
|
32
|
+
MemoryPoisoningMutation,
|
|
25
33
|
MultiTurnMutation,
|
|
26
34
|
type Mutation,
|
|
27
35
|
RedTeamGenerator,
|
|
28
36
|
RoleSpoofMutation,
|
|
37
|
+
ToolAbuseMutation,
|
|
29
38
|
TypoMutation,
|
|
30
39
|
UnsafeResponseDetector,
|
|
31
40
|
} from '@artemiskit/redteam';
|
|
@@ -37,6 +46,8 @@ import type {
|
|
|
37
46
|
ArtemisKitEvents,
|
|
38
47
|
CaseCompleteHandler,
|
|
39
48
|
CaseStartHandler,
|
|
49
|
+
CompareOptions,
|
|
50
|
+
CompareResult,
|
|
40
51
|
ProgressHandler,
|
|
41
52
|
RedTeamMutationCompleteHandler,
|
|
42
53
|
RedTeamMutationStartHandler,
|
|
@@ -47,18 +58,26 @@ import type {
|
|
|
47
58
|
StressOptions,
|
|
48
59
|
StressRequestCompleteHandler,
|
|
49
60
|
StressResult,
|
|
61
|
+
ValidateOptions,
|
|
62
|
+
ValidationResult,
|
|
50
63
|
} from './types';
|
|
51
64
|
|
|
52
65
|
/**
|
|
53
66
|
* Available mutation name to class mapping
|
|
54
67
|
*/
|
|
55
68
|
const MUTATION_MAP: Record<string, new () => Mutation> = {
|
|
69
|
+
// Core mutations (v0.1.x - v0.2.x)
|
|
56
70
|
typo: TypoMutation,
|
|
57
71
|
'role-spoof': RoleSpoofMutation,
|
|
58
72
|
'instruction-flip': InstructionFlipMutation,
|
|
59
73
|
'cot-injection': CotInjectionMutation,
|
|
60
74
|
encoding: EncodingMutation,
|
|
61
75
|
'multi-turn': MultiTurnMutation,
|
|
76
|
+
// Agent-specific mutations (v0.3.1)
|
|
77
|
+
'agent-confusion': AgentConfusionMutation,
|
|
78
|
+
'tool-abuse': ToolAbuseMutation,
|
|
79
|
+
'memory-poisoning': MemoryPoisoningMutation,
|
|
80
|
+
'chain-manipulation': ChainManipulationMutation,
|
|
62
81
|
};
|
|
63
82
|
|
|
64
83
|
type AnyEventHandler = (event: unknown) => void;
|
|
@@ -83,6 +102,7 @@ type AnyEventHandler = (event: unknown) => void;
|
|
|
83
102
|
export class ArtemisKit {
|
|
84
103
|
private config: ArtemisKitConfig;
|
|
85
104
|
private eventHandlers: Map<ArtemisKitEventName, Set<AnyEventHandler>> = new Map();
|
|
105
|
+
private storage: StorageAdapter | null = null;
|
|
86
106
|
|
|
87
107
|
constructor(config: ArtemisKitConfig = {}) {
|
|
88
108
|
this.config = {
|
|
@@ -94,7 +114,13 @@ export class ArtemisKit {
|
|
|
94
114
|
timeout: config.timeout,
|
|
95
115
|
retries: config.retries ?? 0,
|
|
96
116
|
concurrency: config.concurrency ?? 1,
|
|
117
|
+
storage: config.storage,
|
|
97
118
|
};
|
|
119
|
+
|
|
120
|
+
// Initialize storage adapter if configured
|
|
121
|
+
if (config.storage) {
|
|
122
|
+
this.storage = createStorageAdapter(config.storage);
|
|
123
|
+
}
|
|
98
124
|
}
|
|
99
125
|
|
|
100
126
|
// ==========================================================================
|
|
@@ -713,6 +739,443 @@ export class ArtemisKit {
|
|
|
713
739
|
};
|
|
714
740
|
}
|
|
715
741
|
|
|
742
|
+
// ==========================================================================
|
|
743
|
+
// Validation & Comparison Methods
|
|
744
|
+
// ==========================================================================
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Validate scenario files without execution (pre-flight checks)
|
|
748
|
+
*
|
|
749
|
+
* Use this for CI/CD pipelines to catch configuration errors before running tests.
|
|
750
|
+
*
|
|
751
|
+
* @example
|
|
752
|
+
* ```typescript
|
|
753
|
+
* const result = await kit.validate({ scenario: './scenarios/**\/*.yaml' });
|
|
754
|
+
* if (!result.valid) {
|
|
755
|
+
* console.error('Validation errors:', result.errors);
|
|
756
|
+
* process.exit(1);
|
|
757
|
+
* }
|
|
758
|
+
* ```
|
|
759
|
+
*/
|
|
760
|
+
async validate(options: ValidateOptions): Promise<ValidationResult> {
|
|
761
|
+
const inputPaths = Array.isArray(options.scenario) ? options.scenario : [options.scenario];
|
|
762
|
+
const strict = options.strict ?? false;
|
|
763
|
+
|
|
764
|
+
const errors: Array<{
|
|
765
|
+
file: string;
|
|
766
|
+
message: string;
|
|
767
|
+
line?: number;
|
|
768
|
+
column?: number;
|
|
769
|
+
}> = [];
|
|
770
|
+
|
|
771
|
+
const warnings: Array<{
|
|
772
|
+
file: string;
|
|
773
|
+
message: string;
|
|
774
|
+
line?: number;
|
|
775
|
+
column?: number;
|
|
776
|
+
}> = [];
|
|
777
|
+
|
|
778
|
+
const scenarios: Array<{
|
|
779
|
+
file: string;
|
|
780
|
+
name: string;
|
|
781
|
+
caseCount: number;
|
|
782
|
+
valid: boolean;
|
|
783
|
+
}> = [];
|
|
784
|
+
|
|
785
|
+
// Resolve glob patterns and directories to actual file paths
|
|
786
|
+
const resolvedPaths: string[] = [];
|
|
787
|
+
for (const inputPath of inputPaths) {
|
|
788
|
+
try {
|
|
789
|
+
const paths = await resolveScenarioPaths(inputPath);
|
|
790
|
+
resolvedPaths.push(...paths);
|
|
791
|
+
} catch (error) {
|
|
792
|
+
errors.push({
|
|
793
|
+
file: inputPath,
|
|
794
|
+
message: `Failed to resolve path: ${(error as Error).message}`,
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
this.emit('progress', {
|
|
800
|
+
message: `Validating ${resolvedPaths.length} scenario file(s)...`,
|
|
801
|
+
phase: 'setup',
|
|
802
|
+
progress: 0,
|
|
803
|
+
});
|
|
804
|
+
|
|
805
|
+
for (let i = 0; i < resolvedPaths.length; i++) {
|
|
806
|
+
const scenarioPath = resolvedPaths[i];
|
|
807
|
+
|
|
808
|
+
try {
|
|
809
|
+
// Parse and validate scenario file
|
|
810
|
+
const scenario = await parseScenarioFile(scenarioPath);
|
|
811
|
+
|
|
812
|
+
// Check for common issues
|
|
813
|
+
const fileWarnings: typeof warnings = [];
|
|
814
|
+
const fileErrors: typeof errors = [];
|
|
815
|
+
|
|
816
|
+
// Validate cases
|
|
817
|
+
if (!scenario.cases || scenario.cases.length === 0) {
|
|
818
|
+
fileErrors.push({
|
|
819
|
+
file: scenarioPath,
|
|
820
|
+
message: 'Scenario has no test cases defined',
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
// Validate each case
|
|
825
|
+
for (let j = 0; j < (scenario.cases ?? []).length; j++) {
|
|
826
|
+
const testCase = scenario.cases[j];
|
|
827
|
+
|
|
828
|
+
if (!testCase.id) {
|
|
829
|
+
fileErrors.push({
|
|
830
|
+
file: scenarioPath,
|
|
831
|
+
message: `Case at index ${j} is missing required 'id' field`,
|
|
832
|
+
});
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
if (!testCase.prompt) {
|
|
836
|
+
fileErrors.push({
|
|
837
|
+
file: scenarioPath,
|
|
838
|
+
message: `Case '${testCase.id ?? j}' is missing required 'prompt' field`,
|
|
839
|
+
});
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
// Check for empty expectations (warning)
|
|
843
|
+
if (!testCase.expected) {
|
|
844
|
+
fileWarnings.push({
|
|
845
|
+
file: scenarioPath,
|
|
846
|
+
message: `Case '${testCase.id ?? j}' has no expectations defined - will only check for LLM response`,
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
// Validate expectation type
|
|
851
|
+
if (testCase.expected) {
|
|
852
|
+
const validTypes = [
|
|
853
|
+
'contains',
|
|
854
|
+
'not_contains',
|
|
855
|
+
'exact',
|
|
856
|
+
'regex',
|
|
857
|
+
'fuzzy',
|
|
858
|
+
'similarity',
|
|
859
|
+
'llm_grader',
|
|
860
|
+
'json_schema',
|
|
861
|
+
'combined',
|
|
862
|
+
'inline',
|
|
863
|
+
];
|
|
864
|
+
if (!validTypes.includes(testCase.expected.type)) {
|
|
865
|
+
fileErrors.push({
|
|
866
|
+
file: scenarioPath,
|
|
867
|
+
message: `Case '${testCase.id ?? j}' has invalid expectation type '${testCase.expected.type}'`,
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
// Check for duplicate case IDs
|
|
874
|
+
const caseIds = (scenario.cases ?? []).map((c) => c.id).filter(Boolean);
|
|
875
|
+
const seenIds = new Set<string>();
|
|
876
|
+
for (const id of caseIds) {
|
|
877
|
+
if (seenIds.has(id)) {
|
|
878
|
+
fileErrors.push({
|
|
879
|
+
file: scenarioPath,
|
|
880
|
+
message: `Duplicate case ID '${id}' found in scenario`,
|
|
881
|
+
});
|
|
882
|
+
}
|
|
883
|
+
seenIds.add(id);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// Check provider/model configuration
|
|
887
|
+
if (!scenario.provider && !this.config.provider) {
|
|
888
|
+
fileWarnings.push({
|
|
889
|
+
file: scenarioPath,
|
|
890
|
+
message: 'No provider specified in scenario or ArtemisKit config',
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
if (!scenario.model && !this.config.model) {
|
|
895
|
+
fileWarnings.push({
|
|
896
|
+
file: scenarioPath,
|
|
897
|
+
message: 'No model specified in scenario or ArtemisKit config',
|
|
898
|
+
});
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
errors.push(...fileErrors);
|
|
902
|
+
warnings.push(...fileWarnings);
|
|
903
|
+
|
|
904
|
+
scenarios.push({
|
|
905
|
+
file: scenarioPath,
|
|
906
|
+
name: scenario.name ?? scenarioPath,
|
|
907
|
+
caseCount: scenario.cases?.length ?? 0,
|
|
908
|
+
valid: fileErrors.length === 0,
|
|
909
|
+
});
|
|
910
|
+
} catch (error) {
|
|
911
|
+
errors.push({
|
|
912
|
+
file: scenarioPath,
|
|
913
|
+
message: `Failed to parse scenario: ${(error as Error).message}`,
|
|
914
|
+
});
|
|
915
|
+
|
|
916
|
+
scenarios.push({
|
|
917
|
+
file: scenarioPath,
|
|
918
|
+
name: scenarioPath,
|
|
919
|
+
caseCount: 0,
|
|
920
|
+
valid: false,
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
this.emit('progress', {
|
|
925
|
+
message: `Validated ${i + 1}/${resolvedPaths.length} scenarios`,
|
|
926
|
+
phase: 'running',
|
|
927
|
+
progress: Math.round(((i + 1) / resolvedPaths.length) * 100),
|
|
928
|
+
});
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
const valid = errors.length === 0 && (!strict || warnings.length === 0);
|
|
932
|
+
|
|
933
|
+
this.emit('progress', {
|
|
934
|
+
message: `Validation ${valid ? 'passed' : 'failed'}: ${errors.length} error(s), ${warnings.length} warning(s)`,
|
|
935
|
+
phase: 'teardown',
|
|
936
|
+
progress: 100,
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
return {
|
|
940
|
+
valid,
|
|
941
|
+
scenarios,
|
|
942
|
+
errors,
|
|
943
|
+
warnings,
|
|
944
|
+
};
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/**
|
|
948
|
+
* Compare two test runs for regression detection
|
|
949
|
+
*
|
|
950
|
+
* Requires storage configuration to be set in ArtemisKitConfig.
|
|
951
|
+
*
|
|
952
|
+
* @example
|
|
953
|
+
* ```typescript
|
|
954
|
+
* const kit = new ArtemisKit({
|
|
955
|
+
* storage: { type: 'local', basePath: './artemis-runs' }
|
|
956
|
+
* });
|
|
957
|
+
*
|
|
958
|
+
* const comparison = await kit.compare({
|
|
959
|
+
* baseline: 'run_abc123',
|
|
960
|
+
* current: 'run_def456',
|
|
961
|
+
* threshold: 0.05, // 5% regression threshold
|
|
962
|
+
* });
|
|
963
|
+
*
|
|
964
|
+
* if (comparison.hasRegression) {
|
|
965
|
+
* console.error('Regression detected!', comparison.regressions);
|
|
966
|
+
* }
|
|
967
|
+
* ```
|
|
968
|
+
*
|
|
969
|
+
* @since 0.3.2
|
|
970
|
+
*/
|
|
971
|
+
async compare(options: CompareOptions): Promise<CompareResult> {
|
|
972
|
+
const { baseline, current, threshold = 0.05 } = options;
|
|
973
|
+
|
|
974
|
+
// Validate storage is configured
|
|
975
|
+
if (!this.storage) {
|
|
976
|
+
throw new Error(
|
|
977
|
+
'Storage configuration required for compare(). ' +
|
|
978
|
+
'Set storage in ArtemisKitConfig: { storage: { type: "local", basePath: "./artemis-runs" } }'
|
|
979
|
+
);
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
this.emit('progress', {
|
|
983
|
+
message: `Comparing runs: ${baseline} vs ${current}`,
|
|
984
|
+
phase: 'setup',
|
|
985
|
+
progress: 0,
|
|
986
|
+
});
|
|
987
|
+
|
|
988
|
+
// Resolve "latest" baseline if specified
|
|
989
|
+
let baselineRunId = baseline;
|
|
990
|
+
if (baseline === 'latest') {
|
|
991
|
+
this.emit('progress', {
|
|
992
|
+
message: 'Resolving latest run as baseline...',
|
|
993
|
+
phase: 'setup',
|
|
994
|
+
progress: 10,
|
|
995
|
+
});
|
|
996
|
+
|
|
997
|
+
// StorageAdapter.list() returns runs sorted by startTime descending (most recent first)
|
|
998
|
+
// so runs[0] is guaranteed to be the latest run
|
|
999
|
+
const runs = await this.storage.list({ limit: 1, type: 'run' });
|
|
1000
|
+
if (runs.length === 0) {
|
|
1001
|
+
throw new Error('No runs found in storage. Cannot use "latest" as baseline.');
|
|
1002
|
+
}
|
|
1003
|
+
baselineRunId = runs[0].runId;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
this.emit('progress', {
|
|
1007
|
+
message: 'Loading manifests...',
|
|
1008
|
+
phase: 'running',
|
|
1009
|
+
progress: 20,
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
// Load both manifests
|
|
1013
|
+
const [baselineManifest, currentManifest] = await Promise.all([
|
|
1014
|
+
this.storage.load(baselineRunId),
|
|
1015
|
+
this.storage.load(current),
|
|
1016
|
+
]);
|
|
1017
|
+
|
|
1018
|
+
this.emit('progress', {
|
|
1019
|
+
message: 'Analyzing results...',
|
|
1020
|
+
phase: 'running',
|
|
1021
|
+
progress: 50,
|
|
1022
|
+
});
|
|
1023
|
+
|
|
1024
|
+
// Extract metrics from manifests
|
|
1025
|
+
const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
|
|
1026
|
+
const currentMetrics = this.extractRunMetrics(currentManifest, current);
|
|
1027
|
+
|
|
1028
|
+
// Calculate success rate delta
|
|
1029
|
+
const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
|
|
1030
|
+
|
|
1031
|
+
this.emit('progress', {
|
|
1032
|
+
message: 'Comparing cases...',
|
|
1033
|
+
phase: 'running',
|
|
1034
|
+
progress: 70,
|
|
1035
|
+
});
|
|
1036
|
+
|
|
1037
|
+
// Compare case-by-case results
|
|
1038
|
+
const caseComparison = this.compareCases(baselineManifest, currentManifest);
|
|
1039
|
+
|
|
1040
|
+
// Determine if there's a regression
|
|
1041
|
+
// A regression is when the current success rate is lower than baseline by more than threshold
|
|
1042
|
+
const hasRegression = successRateDelta < -threshold;
|
|
1043
|
+
|
|
1044
|
+
this.emit('progress', {
|
|
1045
|
+
message: hasRegression
|
|
1046
|
+
? `Regression detected: ${(successRateDelta * 100).toFixed(1)}% change`
|
|
1047
|
+
: `No regression: ${(successRateDelta * 100).toFixed(1)}% change`,
|
|
1048
|
+
phase: 'teardown',
|
|
1049
|
+
progress: 100,
|
|
1050
|
+
});
|
|
1051
|
+
|
|
1052
|
+
return {
|
|
1053
|
+
baseline: baselineMetrics,
|
|
1054
|
+
current: currentMetrics,
|
|
1055
|
+
comparison: {
|
|
1056
|
+
successRateDelta,
|
|
1057
|
+
...caseComparison,
|
|
1058
|
+
},
|
|
1059
|
+
hasRegression,
|
|
1060
|
+
threshold,
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
/**
|
|
1065
|
+
* Extract run metrics from a manifest
|
|
1066
|
+
*/
|
|
1067
|
+
private extractRunMetrics(manifest: AnyManifest, runId: string): CompareResult['baseline'] {
|
|
1068
|
+
// Handle RunManifest
|
|
1069
|
+
if ('metrics' in manifest && 'success_rate' in manifest.metrics) {
|
|
1070
|
+
const runManifest = manifest as RunManifest;
|
|
1071
|
+
return {
|
|
1072
|
+
runId,
|
|
1073
|
+
successRate: runManifest.metrics.success_rate,
|
|
1074
|
+
totalCases: runManifest.metrics.total_cases,
|
|
1075
|
+
passedCases: runManifest.metrics.passed_cases,
|
|
1076
|
+
failedCases: runManifest.metrics.failed_cases,
|
|
1077
|
+
};
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
// Throw for non-RunManifest types - compare() only supports RunManifest
|
|
1081
|
+
throw new Error(
|
|
1082
|
+
`Cannot extract metrics from run "${runId}": expected RunManifest but got a different manifest type. The compare() method only supports comparing scenario evaluation runs, not stress tests or red team runs.`
|
|
1083
|
+
);
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
/**
|
|
1087
|
+
* Compare cases between baseline and current manifests
|
|
1088
|
+
*/
|
|
1089
|
+
private compareCases(
|
|
1090
|
+
baselineManifest: AnyManifest,
|
|
1091
|
+
currentManifest: AnyManifest
|
|
1092
|
+
): Pick<
|
|
1093
|
+
CompareResult['comparison'],
|
|
1094
|
+
'newFailures' | 'newPasses' | 'unchanged' | 'addedCases' | 'removedCases'
|
|
1095
|
+
> {
|
|
1096
|
+
const newFailures: CompareResult['comparison']['newFailures'] = [];
|
|
1097
|
+
const newPasses: CompareResult['comparison']['newPasses'] = [];
|
|
1098
|
+
const unchanged: CompareResult['comparison']['unchanged'] = [];
|
|
1099
|
+
const addedCases: CompareResult['comparison']['addedCases'] = [];
|
|
1100
|
+
const removedCases: CompareResult['comparison']['removedCases'] = [];
|
|
1101
|
+
|
|
1102
|
+
// Get cases from manifests
|
|
1103
|
+
const baselineCases = this.extractCases(baselineManifest);
|
|
1104
|
+
const currentCases = this.extractCases(currentManifest);
|
|
1105
|
+
|
|
1106
|
+
// Build maps for quick lookup
|
|
1107
|
+
const baselineCaseMap = new Map(baselineCases.map((c) => [c.id, c]));
|
|
1108
|
+
const currentCaseMap = new Map(currentCases.map((c) => [c.id, c]));
|
|
1109
|
+
|
|
1110
|
+
// Find all unique case IDs
|
|
1111
|
+
const allCaseIds = new Set([...baselineCaseMap.keys(), ...currentCaseMap.keys()]);
|
|
1112
|
+
|
|
1113
|
+
for (const caseId of allCaseIds) {
|
|
1114
|
+
const baselineCase = baselineCaseMap.get(caseId);
|
|
1115
|
+
const currentCase = currentCaseMap.get(caseId);
|
|
1116
|
+
|
|
1117
|
+
// Case exists only in current run (new test case)
|
|
1118
|
+
if (!baselineCase && currentCase) {
|
|
1119
|
+
addedCases.push({
|
|
1120
|
+
caseId,
|
|
1121
|
+
caseName: currentCase.name,
|
|
1122
|
+
status: currentCase.ok ? 'passed' : 'failed',
|
|
1123
|
+
});
|
|
1124
|
+
continue;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// Case exists only in baseline (removed test case)
|
|
1128
|
+
if (baselineCase && !currentCase) {
|
|
1129
|
+
removedCases.push({
|
|
1130
|
+
caseId,
|
|
1131
|
+
caseName: baselineCase.name,
|
|
1132
|
+
status: baselineCase.ok ? 'passed' : 'failed',
|
|
1133
|
+
});
|
|
1134
|
+
continue;
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
// Both exist - compare status
|
|
1138
|
+
if (baselineCase && currentCase) {
|
|
1139
|
+
const baselineStatus = baselineCase.ok ? 'passed' : 'failed';
|
|
1140
|
+
const currentStatus = currentCase.ok ? 'passed' : 'failed';
|
|
1141
|
+
|
|
1142
|
+
if (baselineStatus === currentStatus) {
|
|
1143
|
+
unchanged.push({ caseId, status: currentStatus });
|
|
1144
|
+
} else if (baselineStatus === 'passed' && currentStatus === 'failed') {
|
|
1145
|
+
newFailures.push({
|
|
1146
|
+
caseId,
|
|
1147
|
+
caseName: currentCase.name,
|
|
1148
|
+
baselineStatus,
|
|
1149
|
+
currentStatus,
|
|
1150
|
+
});
|
|
1151
|
+
} else if (baselineStatus === 'failed' && currentStatus === 'passed') {
|
|
1152
|
+
newPasses.push({
|
|
1153
|
+
caseId,
|
|
1154
|
+
caseName: currentCase.name,
|
|
1155
|
+
baselineStatus,
|
|
1156
|
+
currentStatus,
|
|
1157
|
+
});
|
|
1158
|
+
}
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
|
|
1162
|
+
return { newFailures, newPasses, unchanged, addedCases, removedCases };
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
/**
|
|
1166
|
+
* Extract cases from a manifest
|
|
1167
|
+
*/
|
|
1168
|
+
private extractCases(manifest: AnyManifest): Array<{ id: string; name?: string; ok: boolean }> {
|
|
1169
|
+
if ('cases' in manifest && Array.isArray(manifest.cases)) {
|
|
1170
|
+
return manifest.cases.map((c) => ({
|
|
1171
|
+
id: c.id,
|
|
1172
|
+
name: c.name,
|
|
1173
|
+
ok: c.ok,
|
|
1174
|
+
}));
|
|
1175
|
+
}
|
|
1176
|
+
return [];
|
|
1177
|
+
}
|
|
1178
|
+
|
|
716
1179
|
// ==========================================================================
|
|
717
1180
|
// Helper Methods
|
|
718
1181
|
// ==========================================================================
|
|
@@ -750,19 +1213,29 @@ export class ArtemisKit {
|
|
|
750
1213
|
* Build mutation instances from mutation names
|
|
751
1214
|
*/
|
|
752
1215
|
private buildMutations(mutationNames?: string[]): Mutation[] {
|
|
753
|
-
|
|
1216
|
+
// If no mutation names specified, use all available mutations
|
|
1217
|
+
if (!mutationNames || mutationNames.length === 0) {
|
|
1218
|
+
return Object.values(MUTATION_MAP).map((MutationClass) => new MutationClass());
|
|
1219
|
+
}
|
|
1220
|
+
|
|
754
1221
|
const mutations: Mutation[] = [];
|
|
1222
|
+
const unknownMutations: string[] = [];
|
|
755
1223
|
|
|
756
|
-
for (const name of
|
|
1224
|
+
for (const name of mutationNames) {
|
|
757
1225
|
const MutationClass = MUTATION_MAP[name];
|
|
758
1226
|
if (MutationClass) {
|
|
759
1227
|
mutations.push(new MutationClass());
|
|
1228
|
+
} else {
|
|
1229
|
+
unknownMutations.push(name);
|
|
760
1230
|
}
|
|
761
1231
|
}
|
|
762
1232
|
|
|
763
|
-
//
|
|
764
|
-
if (
|
|
765
|
-
|
|
1233
|
+
// Throw error for unknown mutation names instead of silently ignoring
|
|
1234
|
+
if (unknownMutations.length > 0) {
|
|
1235
|
+
const availableMutations = Object.keys(MUTATION_MAP).join(', ');
|
|
1236
|
+
throw new Error(
|
|
1237
|
+
`Unknown mutation(s): ${unknownMutations.join(', ')}. Available mutations: ${availableMutations}`
|
|
1238
|
+
);
|
|
766
1239
|
}
|
|
767
1240
|
|
|
768
1241
|
return mutations;
|