@artemiskit/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +229 -0
  2. package/README.md +44 -1
  3. package/dist/artemiskit.d.ts +149 -0
  4. package/dist/artemiskit.d.ts.map +1 -0
  5. package/dist/builders/index.d.ts +352 -0
  6. package/dist/builders/index.d.ts.map +1 -0
  7. package/dist/builders/index.js +354 -0
  8. package/dist/contracts/index.d.ts +253 -0
  9. package/dist/contracts/index.d.ts.map +1 -0
  10. package/dist/contracts/index.js +53 -0
  11. package/dist/guardian/action-validator.d.ts +92 -0
  12. package/dist/guardian/action-validator.d.ts.map +1 -0
  13. package/dist/guardian/circuit-breaker.d.ts +211 -0
  14. package/dist/guardian/circuit-breaker.d.ts.map +1 -0
  15. package/dist/guardian/guardian.d.ts +213 -0
  16. package/dist/guardian/guardian.d.ts.map +1 -0
  17. package/dist/guardian/guardrails.d.ts +107 -0
  18. package/dist/guardian/guardrails.d.ts.map +1 -0
  19. package/dist/guardian/index.d.ts +35 -0
  20. package/dist/guardian/index.d.ts.map +1 -0
  21. package/dist/guardian/intent-classifier.d.ts +81 -0
  22. package/dist/guardian/intent-classifier.d.ts.map +1 -0
  23. package/dist/guardian/interceptor.d.ts +141 -0
  24. package/dist/guardian/interceptor.d.ts.map +1 -0
  25. package/dist/guardian/policy.d.ts +52 -0
  26. package/dist/guardian/policy.d.ts.map +1 -0
  27. package/dist/guardian/semantic-validator.d.ts +61 -0
  28. package/dist/guardian/semantic-validator.d.ts.map +1 -0
  29. package/dist/guardian/types.d.ts +653 -0
  30. package/dist/guardian/types.d.ts.map +1 -0
  31. package/dist/index.d.ts +23 -0
  32. package/dist/index.d.ts.map +1 -0
  33. package/dist/index.js +44918 -28644
  34. package/dist/matchers/core.d.ts +81 -0
  35. package/dist/matchers/core.d.ts.map +1 -0
  36. package/dist/matchers/index.d.ts +6 -0
  37. package/dist/matchers/index.d.ts.map +1 -0
  38. package/dist/matchers/index.js +13 -59
  39. package/dist/matchers/jest.d.ts +46 -0
  40. package/dist/matchers/jest.d.ts.map +1 -0
  41. package/dist/matchers/jest.js +0 -46
  42. package/dist/matchers/vitest.d.ts +24 -0
  43. package/dist/matchers/vitest.d.ts.map +1 -0
  44. package/dist/matchers/vitest.js +0 -46
  45. package/dist/types-only.d.ts +31 -0
  46. package/dist/types-only.d.ts.map +1 -0
  47. package/dist/types-only.js +1 -0
  48. package/dist/types.d.ts +379 -0
  49. package/dist/types.d.ts.map +1 -0
  50. package/dist/utils/index.d.ts +112 -0
  51. package/dist/utils/index.d.ts.map +1 -0
  52. package/dist/utils/index.js +58 -0
  53. package/package.json +19 -4
  54. package/src/__tests__/builders.test.ts +450 -0
  55. package/src/__tests__/circuit-breaker.test.ts +704 -0
  56. package/src/__tests__/compare.test.ts +469 -0
  57. package/src/__tests__/contracts.test.ts +354 -0
  58. package/src/__tests__/guardian-modes.test.ts +215 -0
  59. package/src/__tests__/guardrails.test.ts +404 -0
  60. package/src/__tests__/semantic-validator.test.ts +421 -0
  61. package/src/__tests__/utils.test.ts +324 -0
  62. package/src/__tests__/validate.test.ts +310 -0
  63. package/src/artemiskit.ts +479 -5
  64. package/src/builders/index.ts +700 -0
  65. package/src/contracts/index.ts +357 -0
  66. package/src/guardian/circuit-breaker.ts +74 -28
  67. package/src/guardian/guardian.ts +307 -20
  68. package/src/guardian/guardrails.ts +143 -1
  69. package/src/guardian/index.ts +35 -1
  70. package/src/guardian/interceptor.ts +87 -11
  71. package/src/guardian/semantic-validator.ts +399 -0
  72. package/src/guardian/types.ts +413 -2
  73. package/src/index.ts +258 -14
  74. package/src/types-only.ts +276 -0
  75. package/src/types.ts +186 -0
  76. package/src/utils/index.ts +225 -0
  77. package/tsconfig.json +2 -1
  78. package/adapters/openai/dist/index.js +0 -5625
package/src/artemiskit.ts CHANGED
@@ -5,27 +5,36 @@
5
5
 
6
6
  import {
7
7
  type AdapterConfig,
8
+ type AnyManifest,
8
9
  type ModelClient,
9
10
  type RedTeamCaseResult,
10
11
  type RedTeamManifest,
11
12
  type RedTeamSeverity,
13
+ type RunManifest,
14
+ type StorageAdapter,
12
15
  type StressManifest,
13
16
  type StressMetrics,
14
17
  type StressRequestResult,
15
18
  runScenario as coreRunScenario,
16
19
  createAdapter,
20
+ createStorageAdapter,
17
21
  getGitInfo,
18
22
  parseScenarioFile,
23
+ resolveScenarioPaths,
19
24
  } from '@artemiskit/core';
20
25
  import type { Scenario } from '@artemiskit/core';
21
26
  import {
27
+ AgentConfusionMutation,
28
+ ChainManipulationMutation,
22
29
  CotInjectionMutation,
23
30
  EncodingMutation,
24
31
  InstructionFlipMutation,
32
+ MemoryPoisoningMutation,
25
33
  MultiTurnMutation,
26
34
  type Mutation,
27
35
  RedTeamGenerator,
28
36
  RoleSpoofMutation,
37
+ ToolAbuseMutation,
29
38
  TypoMutation,
30
39
  UnsafeResponseDetector,
31
40
  } from '@artemiskit/redteam';
@@ -37,6 +46,8 @@ import type {
37
46
  ArtemisKitEvents,
38
47
  CaseCompleteHandler,
39
48
  CaseStartHandler,
49
+ CompareOptions,
50
+ CompareResult,
40
51
  ProgressHandler,
41
52
  RedTeamMutationCompleteHandler,
42
53
  RedTeamMutationStartHandler,
@@ -47,18 +58,26 @@ import type {
47
58
  StressOptions,
48
59
  StressRequestCompleteHandler,
49
60
  StressResult,
61
+ ValidateOptions,
62
+ ValidationResult,
50
63
  } from './types';
51
64
 
52
65
  /**
53
66
  * Available mutation name to class mapping
54
67
  */
55
68
  const MUTATION_MAP: Record<string, new () => Mutation> = {
69
+ // Core mutations (v0.1.x - v0.2.x)
56
70
  typo: TypoMutation,
57
71
  'role-spoof': RoleSpoofMutation,
58
72
  'instruction-flip': InstructionFlipMutation,
59
73
  'cot-injection': CotInjectionMutation,
60
74
  encoding: EncodingMutation,
61
75
  'multi-turn': MultiTurnMutation,
76
+ // Agent-specific mutations (v0.3.1)
77
+ 'agent-confusion': AgentConfusionMutation,
78
+ 'tool-abuse': ToolAbuseMutation,
79
+ 'memory-poisoning': MemoryPoisoningMutation,
80
+ 'chain-manipulation': ChainManipulationMutation,
62
81
  };
63
82
 
64
83
  type AnyEventHandler = (event: unknown) => void;
@@ -83,6 +102,7 @@ type AnyEventHandler = (event: unknown) => void;
83
102
  export class ArtemisKit {
84
103
  private config: ArtemisKitConfig;
85
104
  private eventHandlers: Map<ArtemisKitEventName, Set<AnyEventHandler>> = new Map();
105
+ private storage: StorageAdapter | null = null;
86
106
 
87
107
  constructor(config: ArtemisKitConfig = {}) {
88
108
  this.config = {
@@ -94,7 +114,13 @@ export class ArtemisKit {
94
114
  timeout: config.timeout,
95
115
  retries: config.retries ?? 0,
96
116
  concurrency: config.concurrency ?? 1,
117
+ storage: config.storage,
97
118
  };
119
+
120
+ // Initialize storage adapter if configured
121
+ if (config.storage) {
122
+ this.storage = createStorageAdapter(config.storage);
123
+ }
98
124
  }
99
125
 
100
126
  // ==========================================================================
@@ -244,6 +270,7 @@ export class ArtemisKit {
244
270
  timeout: options.timeout ?? this.config.timeout,
245
271
  retries: options.retries ?? this.config.retries,
246
272
  redaction: options.redaction ?? this.config.redaction,
273
+ toolExecutor: options.toolExecutor,
247
274
  onCaseComplete: (caseResult, index, total) => {
248
275
  this.emit('caseComplete', { result: caseResult, index, total });
249
276
  this.emit('progress', {
@@ -713,6 +740,443 @@ export class ArtemisKit {
713
740
  };
714
741
  }
715
742
 
743
+ // ==========================================================================
744
+ // Validation & Comparison Methods
745
+ // ==========================================================================
746
+
747
+ /**
748
+ * Validate scenario files without execution (pre-flight checks)
749
+ *
750
+ * Use this for CI/CD pipelines to catch configuration errors before running tests.
751
+ *
752
+ * @example
753
+ * ```typescript
754
+ * const result = await kit.validate({ scenario: './scenarios/**\/*.yaml' });
755
+ * if (!result.valid) {
756
+ * console.error('Validation errors:', result.errors);
757
+ * process.exit(1);
758
+ * }
759
+ * ```
760
+ */
761
+ async validate(options: ValidateOptions): Promise<ValidationResult> {
762
+ const inputPaths = Array.isArray(options.scenario) ? options.scenario : [options.scenario];
763
+ const strict = options.strict ?? false;
764
+
765
+ const errors: Array<{
766
+ file: string;
767
+ message: string;
768
+ line?: number;
769
+ column?: number;
770
+ }> = [];
771
+
772
+ const warnings: Array<{
773
+ file: string;
774
+ message: string;
775
+ line?: number;
776
+ column?: number;
777
+ }> = [];
778
+
779
+ const scenarios: Array<{
780
+ file: string;
781
+ name: string;
782
+ caseCount: number;
783
+ valid: boolean;
784
+ }> = [];
785
+
786
+ // Resolve glob patterns and directories to actual file paths
787
+ const resolvedPaths: string[] = [];
788
+ for (const inputPath of inputPaths) {
789
+ try {
790
+ const paths = await resolveScenarioPaths(inputPath);
791
+ resolvedPaths.push(...paths);
792
+ } catch (error) {
793
+ errors.push({
794
+ file: inputPath,
795
+ message: `Failed to resolve path: ${(error as Error).message}`,
796
+ });
797
+ }
798
+ }
799
+
800
+ this.emit('progress', {
801
+ message: `Validating ${resolvedPaths.length} scenario file(s)...`,
802
+ phase: 'setup',
803
+ progress: 0,
804
+ });
805
+
806
+ for (let i = 0; i < resolvedPaths.length; i++) {
807
+ const scenarioPath = resolvedPaths[i];
808
+
809
+ try {
810
+ // Parse and validate scenario file
811
+ const scenario = await parseScenarioFile(scenarioPath);
812
+
813
+ // Check for common issues
814
+ const fileWarnings: typeof warnings = [];
815
+ const fileErrors: typeof errors = [];
816
+
817
+ // Validate cases
818
+ if (!scenario.cases || scenario.cases.length === 0) {
819
+ fileErrors.push({
820
+ file: scenarioPath,
821
+ message: 'Scenario has no test cases defined',
822
+ });
823
+ }
824
+
825
+ // Validate each case
826
+ for (let j = 0; j < (scenario.cases ?? []).length; j++) {
827
+ const testCase = scenario.cases[j];
828
+
829
+ if (!testCase.id) {
830
+ fileErrors.push({
831
+ file: scenarioPath,
832
+ message: `Case at index ${j} is missing required 'id' field`,
833
+ });
834
+ }
835
+
836
+ if (!testCase.prompt) {
837
+ fileErrors.push({
838
+ file: scenarioPath,
839
+ message: `Case '${testCase.id ?? j}' is missing required 'prompt' field`,
840
+ });
841
+ }
842
+
843
+ // Check for empty expectations (warning)
844
+ if (!testCase.expected) {
845
+ fileWarnings.push({
846
+ file: scenarioPath,
847
+ message: `Case '${testCase.id ?? j}' has no expectations defined - will only check for LLM response`,
848
+ });
849
+ }
850
+
851
+ // Validate expectation type
852
+ if (testCase.expected) {
853
+ const validTypes = [
854
+ 'contains',
855
+ 'not_contains',
856
+ 'exact',
857
+ 'regex',
858
+ 'fuzzy',
859
+ 'similarity',
860
+ 'llm_grader',
861
+ 'json_schema',
862
+ 'combined',
863
+ 'inline',
864
+ ];
865
+ if (!validTypes.includes(testCase.expected.type)) {
866
+ fileErrors.push({
867
+ file: scenarioPath,
868
+ message: `Case '${testCase.id ?? j}' has invalid expectation type '${testCase.expected.type}'`,
869
+ });
870
+ }
871
+ }
872
+ }
873
+
874
+ // Check for duplicate case IDs
875
+ const caseIds = (scenario.cases ?? []).map((c) => c.id).filter(Boolean);
876
+ const seenIds = new Set<string>();
877
+ for (const id of caseIds) {
878
+ if (seenIds.has(id)) {
879
+ fileErrors.push({
880
+ file: scenarioPath,
881
+ message: `Duplicate case ID '${id}' found in scenario`,
882
+ });
883
+ }
884
+ seenIds.add(id);
885
+ }
886
+
887
+ // Check provider/model configuration
888
+ if (!scenario.provider && !this.config.provider) {
889
+ fileWarnings.push({
890
+ file: scenarioPath,
891
+ message: 'No provider specified in scenario or ArtemisKit config',
892
+ });
893
+ }
894
+
895
+ if (!scenario.model && !this.config.model) {
896
+ fileWarnings.push({
897
+ file: scenarioPath,
898
+ message: 'No model specified in scenario or ArtemisKit config',
899
+ });
900
+ }
901
+
902
+ errors.push(...fileErrors);
903
+ warnings.push(...fileWarnings);
904
+
905
+ scenarios.push({
906
+ file: scenarioPath,
907
+ name: scenario.name ?? scenarioPath,
908
+ caseCount: scenario.cases?.length ?? 0,
909
+ valid: fileErrors.length === 0,
910
+ });
911
+ } catch (error) {
912
+ errors.push({
913
+ file: scenarioPath,
914
+ message: `Failed to parse scenario: ${(error as Error).message}`,
915
+ });
916
+
917
+ scenarios.push({
918
+ file: scenarioPath,
919
+ name: scenarioPath,
920
+ caseCount: 0,
921
+ valid: false,
922
+ });
923
+ }
924
+
925
+ this.emit('progress', {
926
+ message: `Validated ${i + 1}/${resolvedPaths.length} scenarios`,
927
+ phase: 'running',
928
+ progress: Math.round(((i + 1) / resolvedPaths.length) * 100),
929
+ });
930
+ }
931
+
932
+ const valid = errors.length === 0 && (!strict || warnings.length === 0);
933
+
934
+ this.emit('progress', {
935
+ message: `Validation ${valid ? 'passed' : 'failed'}: ${errors.length} error(s), ${warnings.length} warning(s)`,
936
+ phase: 'teardown',
937
+ progress: 100,
938
+ });
939
+
940
+ return {
941
+ valid,
942
+ scenarios,
943
+ errors,
944
+ warnings,
945
+ };
946
+ }
947
+
948
+ /**
949
+ * Compare two test runs for regression detection
950
+ *
951
+ * Requires storage configuration to be set in ArtemisKitConfig.
952
+ *
953
+ * @example
954
+ * ```typescript
955
+ * const kit = new ArtemisKit({
956
+ * storage: { type: 'local', basePath: './artemis-runs' }
957
+ * });
958
+ *
959
+ * const comparison = await kit.compare({
960
+ * baseline: 'run_abc123',
961
+ * current: 'run_def456',
962
+ * threshold: 0.05, // 5% regression threshold
963
+ * });
964
+ *
965
+ * if (comparison.hasRegression) {
966
+ * console.error('Regression detected!', comparison.regressions);
967
+ * }
968
+ * ```
969
+ *
970
+ * @since 0.3.2
971
+ */
972
+ async compare(options: CompareOptions): Promise<CompareResult> {
973
+ const { baseline, current, threshold = 0.05 } = options;
974
+
975
+ // Validate storage is configured
976
+ if (!this.storage) {
977
+ throw new Error(
978
+ 'Storage configuration required for compare(). ' +
979
+ 'Set storage in ArtemisKitConfig: { storage: { type: "local", basePath: "./artemis-runs" } }'
980
+ );
981
+ }
982
+
983
+ this.emit('progress', {
984
+ message: `Comparing runs: ${baseline} vs ${current}`,
985
+ phase: 'setup',
986
+ progress: 0,
987
+ });
988
+
989
+ // Resolve "latest" baseline if specified
990
+ let baselineRunId = baseline;
991
+ if (baseline === 'latest') {
992
+ this.emit('progress', {
993
+ message: 'Resolving latest run as baseline...',
994
+ phase: 'setup',
995
+ progress: 10,
996
+ });
997
+
998
+ // StorageAdapter.list() returns runs sorted by startTime descending (most recent first)
999
+ // so runs[0] is guaranteed to be the latest run
1000
+ const runs = await this.storage.list({ limit: 1, type: 'run' });
1001
+ if (runs.length === 0) {
1002
+ throw new Error('No runs found in storage. Cannot use "latest" as baseline.');
1003
+ }
1004
+ baselineRunId = runs[0].runId;
1005
+ }
1006
+
1007
+ this.emit('progress', {
1008
+ message: 'Loading manifests...',
1009
+ phase: 'running',
1010
+ progress: 20,
1011
+ });
1012
+
1013
+ // Load both manifests
1014
+ const [baselineManifest, currentManifest] = await Promise.all([
1015
+ this.storage.load(baselineRunId),
1016
+ this.storage.load(current),
1017
+ ]);
1018
+
1019
+ this.emit('progress', {
1020
+ message: 'Analyzing results...',
1021
+ phase: 'running',
1022
+ progress: 50,
1023
+ });
1024
+
1025
+ // Extract metrics from manifests
1026
+ const baselineMetrics = this.extractRunMetrics(baselineManifest, baselineRunId);
1027
+ const currentMetrics = this.extractRunMetrics(currentManifest, current);
1028
+
1029
+ // Calculate success rate delta
1030
+ const successRateDelta = currentMetrics.successRate - baselineMetrics.successRate;
1031
+
1032
+ this.emit('progress', {
1033
+ message: 'Comparing cases...',
1034
+ phase: 'running',
1035
+ progress: 70,
1036
+ });
1037
+
1038
+ // Compare case-by-case results
1039
+ const caseComparison = this.compareCases(baselineManifest, currentManifest);
1040
+
1041
+ // Determine if there's a regression
1042
+ // A regression is when the current success rate is lower than baseline by more than threshold
1043
+ const hasRegression = successRateDelta < -threshold;
1044
+
1045
+ this.emit('progress', {
1046
+ message: hasRegression
1047
+ ? `Regression detected: ${(successRateDelta * 100).toFixed(1)}% change`
1048
+ : `No regression: ${(successRateDelta * 100).toFixed(1)}% change`,
1049
+ phase: 'teardown',
1050
+ progress: 100,
1051
+ });
1052
+
1053
+ return {
1054
+ baseline: baselineMetrics,
1055
+ current: currentMetrics,
1056
+ comparison: {
1057
+ successRateDelta,
1058
+ ...caseComparison,
1059
+ },
1060
+ hasRegression,
1061
+ threshold,
1062
+ };
1063
+ }
1064
+
1065
+ /**
1066
+ * Extract run metrics from a manifest
1067
+ */
1068
+ private extractRunMetrics(manifest: AnyManifest, runId: string): CompareResult['baseline'] {
1069
+ // Handle RunManifest
1070
+ if ('metrics' in manifest && 'success_rate' in manifest.metrics) {
1071
+ const runManifest = manifest as RunManifest;
1072
+ return {
1073
+ runId,
1074
+ successRate: runManifest.metrics.success_rate,
1075
+ totalCases: runManifest.metrics.total_cases,
1076
+ passedCases: runManifest.metrics.passed_cases,
1077
+ failedCases: runManifest.metrics.failed_cases,
1078
+ };
1079
+ }
1080
+
1081
+ // Throw for non-RunManifest types - compare() only supports RunManifest
1082
+ throw new Error(
1083
+ `Cannot extract metrics from run "${runId}": expected RunManifest but got a different manifest type. The compare() method only supports comparing scenario evaluation runs, not stress tests or red team runs.`
1084
+ );
1085
+ }
1086
+
1087
+ /**
1088
+ * Compare cases between baseline and current manifests
1089
+ */
1090
+ private compareCases(
1091
+ baselineManifest: AnyManifest,
1092
+ currentManifest: AnyManifest
1093
+ ): Pick<
1094
+ CompareResult['comparison'],
1095
+ 'newFailures' | 'newPasses' | 'unchanged' | 'addedCases' | 'removedCases'
1096
+ > {
1097
+ const newFailures: CompareResult['comparison']['newFailures'] = [];
1098
+ const newPasses: CompareResult['comparison']['newPasses'] = [];
1099
+ const unchanged: CompareResult['comparison']['unchanged'] = [];
1100
+ const addedCases: CompareResult['comparison']['addedCases'] = [];
1101
+ const removedCases: CompareResult['comparison']['removedCases'] = [];
1102
+
1103
+ // Get cases from manifests
1104
+ const baselineCases = this.extractCases(baselineManifest);
1105
+ const currentCases = this.extractCases(currentManifest);
1106
+
1107
+ // Build maps for quick lookup
1108
+ const baselineCaseMap = new Map(baselineCases.map((c) => [c.id, c]));
1109
+ const currentCaseMap = new Map(currentCases.map((c) => [c.id, c]));
1110
+
1111
+ // Find all unique case IDs
1112
+ const allCaseIds = new Set([...baselineCaseMap.keys(), ...currentCaseMap.keys()]);
1113
+
1114
+ for (const caseId of allCaseIds) {
1115
+ const baselineCase = baselineCaseMap.get(caseId);
1116
+ const currentCase = currentCaseMap.get(caseId);
1117
+
1118
+ // Case exists only in current run (new test case)
1119
+ if (!baselineCase && currentCase) {
1120
+ addedCases.push({
1121
+ caseId,
1122
+ caseName: currentCase.name,
1123
+ status: currentCase.ok ? 'passed' : 'failed',
1124
+ });
1125
+ continue;
1126
+ }
1127
+
1128
+ // Case exists only in baseline (removed test case)
1129
+ if (baselineCase && !currentCase) {
1130
+ removedCases.push({
1131
+ caseId,
1132
+ caseName: baselineCase.name,
1133
+ status: baselineCase.ok ? 'passed' : 'failed',
1134
+ });
1135
+ continue;
1136
+ }
1137
+
1138
+ // Both exist - compare status
1139
+ if (baselineCase && currentCase) {
1140
+ const baselineStatus = baselineCase.ok ? 'passed' : 'failed';
1141
+ const currentStatus = currentCase.ok ? 'passed' : 'failed';
1142
+
1143
+ if (baselineStatus === currentStatus) {
1144
+ unchanged.push({ caseId, status: currentStatus });
1145
+ } else if (baselineStatus === 'passed' && currentStatus === 'failed') {
1146
+ newFailures.push({
1147
+ caseId,
1148
+ caseName: currentCase.name,
1149
+ baselineStatus,
1150
+ currentStatus,
1151
+ });
1152
+ } else if (baselineStatus === 'failed' && currentStatus === 'passed') {
1153
+ newPasses.push({
1154
+ caseId,
1155
+ caseName: currentCase.name,
1156
+ baselineStatus,
1157
+ currentStatus,
1158
+ });
1159
+ }
1160
+ }
1161
+ }
1162
+
1163
+ return { newFailures, newPasses, unchanged, addedCases, removedCases };
1164
+ }
1165
+
1166
+ /**
1167
+ * Extract cases from a manifest
1168
+ */
1169
+ private extractCases(manifest: AnyManifest): Array<{ id: string; name?: string; ok: boolean }> {
1170
+ if ('cases' in manifest && Array.isArray(manifest.cases)) {
1171
+ return manifest.cases.map((c) => ({
1172
+ id: c.id,
1173
+ name: c.name,
1174
+ ok: c.ok,
1175
+ }));
1176
+ }
1177
+ return [];
1178
+ }
1179
+
716
1180
  // ==========================================================================
717
1181
  // Helper Methods
718
1182
  // ==========================================================================
@@ -750,19 +1214,29 @@ export class ArtemisKit {
750
1214
  * Build mutation instances from mutation names
751
1215
  */
752
1216
  private buildMutations(mutationNames?: string[]): Mutation[] {
753
- const names = mutationNames ?? Object.keys(MUTATION_MAP);
1217
+ // If no mutation names specified, use all available mutations
1218
+ if (!mutationNames || mutationNames.length === 0) {
1219
+ return Object.values(MUTATION_MAP).map((MutationClass) => new MutationClass());
1220
+ }
1221
+
754
1222
  const mutations: Mutation[] = [];
1223
+ const unknownMutations: string[] = [];
755
1224
 
756
- for (const name of names) {
1225
+ for (const name of mutationNames) {
757
1226
  const MutationClass = MUTATION_MAP[name];
758
1227
  if (MutationClass) {
759
1228
  mutations.push(new MutationClass());
1229
+ } else {
1230
+ unknownMutations.push(name);
760
1231
  }
761
1232
  }
762
1233
 
763
- // If no valid mutations found, use all defaults
764
- if (mutations.length === 0) {
765
- return Object.values(MUTATION_MAP).map((MutationClass) => new MutationClass());
1234
+ // Throw error for unknown mutation names instead of silently ignoring
1235
+ if (unknownMutations.length > 0) {
1236
+ const availableMutations = Object.keys(MUTATION_MAP).join(', ');
1237
+ throw new Error(
1238
+ `Unknown mutation(s): ${unknownMutations.join(', ')}. Available mutations: ${availableMutations}`
1239
+ );
766
1240
  }
767
1241
 
768
1242
  return mutations;