@artemiskit/core 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/CHANGELOG.md +48 -0
  2. package/dist/adapters/factory.d.ts +23 -0
  3. package/dist/adapters/factory.d.ts.map +1 -0
  4. package/dist/adapters/index.d.ts +7 -0
  5. package/dist/adapters/index.d.ts.map +1 -0
  6. package/dist/adapters/registry.d.ts +56 -0
  7. package/dist/adapters/registry.d.ts.map +1 -0
  8. package/dist/adapters/types.d.ts +151 -0
  9. package/dist/adapters/types.d.ts.map +1 -0
  10. package/dist/artifacts/index.d.ts +6 -0
  11. package/dist/artifacts/index.d.ts.map +1 -0
  12. package/dist/artifacts/manifest.d.ts +19 -0
  13. package/dist/artifacts/manifest.d.ts.map +1 -0
  14. package/dist/artifacts/types.d.ts +368 -0
  15. package/dist/artifacts/types.d.ts.map +1 -0
  16. package/dist/evaluators/contains.d.ts +10 -0
  17. package/dist/evaluators/contains.d.ts.map +1 -0
  18. package/dist/evaluators/exact.d.ts +10 -0
  19. package/dist/evaluators/exact.d.ts.map +1 -0
  20. package/dist/evaluators/fuzzy.d.ts +10 -0
  21. package/dist/evaluators/fuzzy.d.ts.map +1 -0
  22. package/dist/evaluators/index.d.ts +24 -0
  23. package/dist/evaluators/index.d.ts.map +1 -0
  24. package/dist/evaluators/json-schema.d.ts +11 -0
  25. package/dist/evaluators/json-schema.d.ts.map +1 -0
  26. package/dist/evaluators/llm-grader.d.ts +11 -0
  27. package/dist/evaluators/llm-grader.d.ts.map +1 -0
  28. package/dist/evaluators/regex.d.ts +10 -0
  29. package/dist/evaluators/regex.d.ts.map +1 -0
  30. package/dist/evaluators/types.d.ts +29 -0
  31. package/dist/evaluators/types.d.ts.map +1 -0
  32. package/dist/index.d.ts +14 -0
  33. package/dist/index.d.ts.map +1 -0
  34. package/dist/index.js +26021 -0
  35. package/dist/provenance/environment.d.ts +12 -0
  36. package/dist/provenance/environment.d.ts.map +1 -0
  37. package/dist/provenance/git.d.ts +9 -0
  38. package/dist/provenance/git.d.ts.map +1 -0
  39. package/dist/provenance/index.d.ts +6 -0
  40. package/dist/provenance/index.d.ts.map +1 -0
  41. package/dist/redaction/index.d.ts +3 -0
  42. package/dist/redaction/index.d.ts.map +1 -0
  43. package/dist/redaction/redactor.d.ts +79 -0
  44. package/dist/redaction/redactor.d.ts.map +1 -0
  45. package/dist/redaction/types.d.ts +120 -0
  46. package/dist/redaction/types.d.ts.map +1 -0
  47. package/dist/runner/executor.d.ts +11 -0
  48. package/dist/runner/executor.d.ts.map +1 -0
  49. package/dist/runner/index.d.ts +7 -0
  50. package/dist/runner/index.d.ts.map +1 -0
  51. package/dist/runner/runner.d.ts +13 -0
  52. package/dist/runner/runner.d.ts.map +1 -0
  53. package/dist/runner/types.d.ts +57 -0
  54. package/dist/runner/types.d.ts.map +1 -0
  55. package/dist/scenario/index.d.ts +7 -0
  56. package/dist/scenario/index.d.ts.map +1 -0
  57. package/dist/scenario/parser.d.ts +17 -0
  58. package/dist/scenario/parser.d.ts.map +1 -0
  59. package/dist/scenario/schema.d.ts +945 -0
  60. package/dist/scenario/schema.d.ts.map +1 -0
  61. package/dist/scenario/variables.d.ts +19 -0
  62. package/dist/scenario/variables.d.ts.map +1 -0
  63. package/dist/storage/factory.d.ts +13 -0
  64. package/dist/storage/factory.d.ts.map +1 -0
  65. package/dist/storage/index.d.ts +8 -0
  66. package/dist/storage/index.d.ts.map +1 -0
  67. package/dist/storage/local.d.ts +20 -0
  68. package/dist/storage/local.d.ts.map +1 -0
  69. package/dist/storage/supabase.d.ts +21 -0
  70. package/dist/storage/supabase.d.ts.map +1 -0
  71. package/dist/storage/types.d.ts +86 -0
  72. package/dist/storage/types.d.ts.map +1 -0
  73. package/dist/utils/errors.d.ts +25 -0
  74. package/dist/utils/errors.d.ts.map +1 -0
  75. package/dist/utils/index.d.ts +6 -0
  76. package/dist/utils/index.d.ts.map +1 -0
  77. package/dist/utils/logger.d.ts +21 -0
  78. package/dist/utils/logger.d.ts.map +1 -0
  79. package/package.json +56 -0
  80. package/src/adapters/factory.ts +75 -0
  81. package/src/adapters/index.ts +7 -0
  82. package/src/adapters/registry.ts +143 -0
  83. package/src/adapters/types.ts +184 -0
  84. package/src/artifacts/index.ts +6 -0
  85. package/src/artifacts/manifest.test.ts +206 -0
  86. package/src/artifacts/manifest.ts +136 -0
  87. package/src/artifacts/types.ts +426 -0
  88. package/src/evaluators/contains.test.ts +58 -0
  89. package/src/evaluators/contains.ts +41 -0
  90. package/src/evaluators/exact.test.ts +48 -0
  91. package/src/evaluators/exact.ts +33 -0
  92. package/src/evaluators/fuzzy.test.ts +50 -0
  93. package/src/evaluators/fuzzy.ts +39 -0
  94. package/src/evaluators/index.ts +53 -0
  95. package/src/evaluators/json-schema.ts +98 -0
  96. package/src/evaluators/llm-grader.ts +100 -0
  97. package/src/evaluators/regex.test.ts +73 -0
  98. package/src/evaluators/regex.ts +43 -0
  99. package/src/evaluators/types.ts +37 -0
  100. package/src/index.ts +31 -0
  101. package/src/provenance/environment.ts +18 -0
  102. package/src/provenance/git.ts +48 -0
  103. package/src/provenance/index.ts +6 -0
  104. package/src/redaction/index.ts +23 -0
  105. package/src/redaction/redactor.test.ts +258 -0
  106. package/src/redaction/redactor.ts +246 -0
  107. package/src/redaction/types.ts +135 -0
  108. package/src/runner/executor.ts +251 -0
  109. package/src/runner/index.ts +7 -0
  110. package/src/runner/runner.ts +153 -0
  111. package/src/runner/types.ts +60 -0
  112. package/src/scenario/index.ts +7 -0
  113. package/src/scenario/parser.test.ts +99 -0
  114. package/src/scenario/parser.ts +108 -0
  115. package/src/scenario/schema.ts +176 -0
  116. package/src/scenario/variables.test.ts +150 -0
  117. package/src/scenario/variables.ts +60 -0
  118. package/src/storage/factory.ts +52 -0
  119. package/src/storage/index.ts +8 -0
  120. package/src/storage/local.test.ts +165 -0
  121. package/src/storage/local.ts +194 -0
  122. package/src/storage/supabase.ts +151 -0
  123. package/src/storage/types.ts +98 -0
  124. package/src/utils/errors.ts +76 -0
  125. package/src/utils/index.ts +6 -0
  126. package/src/utils/logger.ts +59 -0
  127. package/tsconfig.json +13 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/scenario/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;GAEG;AACH,eAAO,MAAM,cAAc,mIAWzB,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoBpB,CAAC;AAEd;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2CzB,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;EAG5B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,eAAe,gFAAuE,CAAC;AAEpG;;;GAGG;AACH,QAAA,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;GAAuC,CAAC;AAEnE;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAazB,6DAA6D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAE7D,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAYzB,6CAA6C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAnB7C,6DAA6D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiC7D,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAChE,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC"}
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Variable substitution for scenario templates
3
+ * Supports {{variable}} syntax in strings
4
+ */
5
+ import type { Variables } from './schema';
6
+ /**
7
+ * Substitute variables in a string using {{variable}} syntax
8
+ */
9
+ export declare function substituteString(str: string, variables: Variables): string;
10
+ /**
11
+ * Recursively substitute variables in an object
12
+ * Only substitutes in string values, preserving other types
13
+ */
14
+ export declare function substituteVariables<T>(obj: T, variables: Variables): T;
15
+ /**
16
+ * Merge variables with case-level overriding scenario-level
17
+ */
18
+ export declare function mergeVariables(scenarioVars: Variables | undefined, caseVars: Variables | undefined): Variables;
19
+ //# sourceMappingURL=variables.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"variables.d.ts","sourceRoot":"","sources":["../../src/scenario/variables.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAE1C;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,MAAM,CAQ1E;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,GAAG,CAAC,CAsBtE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,YAAY,EAAE,SAAS,GAAG,SAAS,EACnC,QAAQ,EAAE,SAAS,GAAG,SAAS,GAC9B,SAAS,CAKX"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Storage factory for creating storage adapters
3
+ */
4
+ import type { StorageAdapter, StorageConfig } from './types';
5
+ /**
6
+ * Create a storage adapter from configuration
7
+ */
8
+ export declare function createStorageAdapter(config: StorageConfig): StorageAdapter;
9
+ /**
10
+ * Create storage adapter from environment variables
11
+ */
12
+ export declare function createStorageFromEnv(): StorageAdapter;
13
+ //# sourceMappingURL=factory.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/storage/factory.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7D;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,aAAa,GAAG,cAAc,CAqB1E;AAED;;GAEG;AACH,wBAAgB,oBAAoB,IAAI,cAAc,CAarD"}
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Storage module exports
3
+ */
4
+ export * from './types';
5
+ export { createStorageAdapter, createStorageFromEnv } from './factory';
6
+ export { SupabaseStorageAdapter, type SupabaseStorageConfig } from './supabase';
7
+ export { LocalStorageAdapter } from './local';
8
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,SAAS,CAAC;AACxB,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,KAAK,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC"}
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Local filesystem storage adapter
3
+ */
4
+ import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
5
+ import type { ComparisonResult, ListOptions, RunListItem, StorageAdapter } from './types';
6
+ export declare class LocalStorageAdapter implements StorageAdapter {
7
+ private basePath;
8
+ constructor(basePath?: string);
9
+ save(manifest: AnyManifest): Promise<string>;
10
+ load(runId: string): Promise<AnyManifest>;
11
+ loadRun(runId: string): Promise<RunManifest>;
12
+ loadRedTeam(runId: string): Promise<RedTeamManifest>;
13
+ loadStress(runId: string): Promise<StressManifest>;
14
+ list(options?: ListOptions): Promise<RunListItem[]>;
15
+ delete(runId: string): Promise<void>;
16
+ compare(baselineId: string, currentId: string): Promise<ComparisonResult>;
17
+ private listDirectories;
18
+ private listFiles;
19
+ }
20
+ //# sourceMappingURL=local.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"local.d.ts","sourceRoot":"","sources":["../../src/storage/local.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpG,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAkC1F,qBAAa,mBAAoB,YAAW,cAAc;IACxD,OAAO,CAAC,QAAQ,CAAS;gBAEb,QAAQ,SAAmB;IAIjC,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAU5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAczC,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAQ5C,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAQpD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAQlD,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmDnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYpC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;YAiBjE,eAAe;YASf,SAAS;CAQxB"}
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Supabase storage adapter
3
+ */
4
+ import type { RunManifest } from '../artifacts/types';
5
+ import type { ComparisonResult, ListOptions, RunListItem, StorageAdapter } from './types';
6
+ export interface SupabaseStorageConfig {
7
+ url: string;
8
+ anonKey: string;
9
+ bucket?: string;
10
+ }
11
+ export declare class SupabaseStorageAdapter implements StorageAdapter {
12
+ private client;
13
+ private bucket;
14
+ constructor(config: SupabaseStorageConfig);
15
+ save(manifest: RunManifest): Promise<string>;
16
+ load(runId: string): Promise<RunManifest>;
17
+ list(options?: ListOptions): Promise<RunListItem[]>;
18
+ delete(runId: string): Promise<void>;
19
+ compare(baselineId: string, currentId: string): Promise<ComparisonResult>;
20
+ }
21
+ //# sourceMappingURL=supabase.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"supabase.d.ts","sourceRoot":"","sources":["../../src/storage/supabase.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE1F,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,sBAAuB,YAAW,cAAc;IAC3D,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,EAAE,qBAAqB;IAKnC,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IA4C5C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAuBzC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAiCnD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAcpC,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAahF"}
@@ -0,0 +1,86 @@
1
+ /**
2
+ * Storage types and interfaces
3
+ */
4
+ import type { AnyManifest, RedTeamManifest, RunManifest, StressManifest } from '../artifacts/types';
5
+ /**
6
+ * Run listing item
7
+ */
8
+ export interface RunListItem {
9
+ runId: string;
10
+ scenario: string;
11
+ successRate: number;
12
+ createdAt: string;
13
+ /** Type of manifest (run, redteam, stress) */
14
+ type?: 'run' | 'redteam' | 'stress';
15
+ }
16
+ /**
17
+ * Comparison result between two runs
18
+ */
19
+ export interface ComparisonResult {
20
+ baseline: RunManifest;
21
+ current: RunManifest;
22
+ delta: {
23
+ successRate: number;
24
+ latency: number;
25
+ tokens: number;
26
+ };
27
+ }
28
+ /**
29
+ * List options for filtering runs
30
+ */
31
+ export interface ListOptions {
32
+ project?: string;
33
+ scenario?: string;
34
+ limit?: number;
35
+ offset?: number;
36
+ /** Filter by manifest type */
37
+ type?: 'run' | 'redteam' | 'stress';
38
+ }
39
+ /**
40
+ * Storage adapter interface - implement to create custom storage backends
41
+ */
42
+ export interface StorageAdapter {
43
+ /**
44
+ * Save a run manifest (any type)
45
+ */
46
+ save(manifest: AnyManifest): Promise<string>;
47
+ /**
48
+ * Load a run manifest by ID
49
+ */
50
+ load(runId: string): Promise<AnyManifest>;
51
+ /**
52
+ * Load a standard run manifest by ID
53
+ */
54
+ loadRun?(runId: string): Promise<RunManifest>;
55
+ /**
56
+ * Load a red team manifest by ID
57
+ */
58
+ loadRedTeam?(runId: string): Promise<RedTeamManifest>;
59
+ /**
60
+ * Load a stress manifest by ID
61
+ */
62
+ loadStress?(runId: string): Promise<StressManifest>;
63
+ /**
64
+ * List runs with optional filters
65
+ */
66
+ list(options?: ListOptions): Promise<RunListItem[]>;
67
+ /**
68
+ * Delete a run
69
+ */
70
+ delete(runId: string): Promise<void>;
71
+ /**
72
+ * Compare two runs
73
+ */
74
+ compare?(baselineId: string, currentId: string): Promise<ComparisonResult>;
75
+ }
76
+ /**
77
+ * Storage configuration
78
+ */
79
+ export interface StorageConfig {
80
+ type: 'supabase' | 'local';
81
+ url?: string;
82
+ anonKey?: string;
83
+ bucket?: string;
84
+ basePath?: string;
85
+ }
86
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,WAAW,CAAC;IACtB,OAAO,EAAE,WAAW,CAAC;IACrB,KAAK,EAAE;QACL,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,KAAK,GAAG,SAAS,GAAG,QAAQ,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7C;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE1C;;OAEG;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAE9C;;OAEG;IACH,WAAW,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEtD;;OAEG;IACH,UAAU,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAEpD;;OAEG;IACH,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEpD;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErC;;OAEG;IACH,OAAO,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC5E;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB"}
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Error handling utilities for Artemis
3
+ */
4
+ /**
5
+ * Error codes used throughout Artemis
6
+ */
7
+ export type ArtemisErrorCode = 'UNKNOWN_PROVIDER' | 'PROVIDER_UNAVAILABLE' | 'SCENARIO_READ_ERROR' | 'SCENARIO_PARSE_ERROR' | 'SCENARIO_VALIDATION_ERROR' | 'ADAPTER_ERROR' | 'GENERATION_ERROR' | 'EVALUATION_ERROR' | 'STORAGE_ERROR' | 'CONFIG_ERROR' | 'UNKNOWN_ERROR';
8
+ /**
9
+ * Custom error class for Artemis
10
+ */
11
+ export declare class ArtemisError extends Error {
12
+ readonly code: ArtemisErrorCode;
13
+ readonly details?: Record<string, unknown>;
14
+ constructor(message: string, code?: ArtemisErrorCode, details?: Record<string, unknown>);
15
+ toJSON(): Record<string, unknown>;
16
+ }
17
+ /**
18
+ * Check if error is an ArtemisError
19
+ */
20
+ export declare function isArtemisError(error: unknown): error is ArtemisError;
21
+ /**
22
+ * Wrap unknown errors in ArtemisError
23
+ */
24
+ export declare function wrapError(error: unknown, code?: ArtemisErrorCode, context?: string): ArtemisError;
25
+ //# sourceMappingURL=errors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/utils/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,MAAM,gBAAgB,GACxB,kBAAkB,GAClB,sBAAsB,GACtB,qBAAqB,GACrB,sBAAsB,GACtB,2BAA2B,GAC3B,eAAe,GACf,kBAAkB,GAClB,kBAAkB,GAClB,eAAe,GACf,cAAc,GACd,eAAe,CAAC;AAEpB;;GAEG;AACH,qBAAa,YAAa,SAAQ,KAAK;IACrC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAGzC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,gBAAkC,EACxC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAUnC,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CASlC;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAEpE;AAED;;GAEG;AACH,wBAAgB,SAAS,CACvB,KAAK,EAAE,OAAO,EACd,IAAI,GAAE,gBAAkC,EACxC,OAAO,CAAC,EAAE,MAAM,GACf,YAAY,CAUd"}
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Utilities module exports
3
+ */
4
+ export * from './errors';
5
+ export * from './logger';
6
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC"}
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Logger utility for Artemis
3
+ */
4
+ export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
5
+ /**
6
+ * Logger class for consistent logging across Artemis
7
+ */
8
+ export declare class Logger {
9
+ private logger;
10
+ constructor(name: string);
11
+ debug(message: string, data?: Record<string, unknown>): void;
12
+ info(message: string, data?: Record<string, unknown>): void;
13
+ warn(message: string, data?: Record<string, unknown>): void;
14
+ error(message: string, error?: Error | unknown, data?: Record<string, unknown>): void;
15
+ child(bindings: Record<string, unknown>): Logger;
16
+ }
17
+ /**
18
+ * Default logger instance
19
+ */
20
+ export declare const logger: Logger;
21
+ //# sourceMappingURL=logger.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;AAY3D;;GAEG;AACH,qBAAa,MAAM;IACjB,OAAO,CAAC,MAAM,CAAc;gBAEhB,IAAI,EAAE,MAAM;IAIxB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI5D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3D,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,GAAG,OAAO,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAQrF,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM;CAKjD;AAED;;GAEG;AACH,eAAO,MAAM,MAAM,QAAwB,CAAC"}
package/package.json ADDED
@@ -0,0 +1,56 @@
1
+ {
2
+ "name": "@artemiskit/core",
3
+ "version": "0.1.2",
4
+ "description": "Core runner, evaluators, and storage for ArtemisKit LLM evaluation toolkit",
5
+ "type": "module",
6
+ "license": "Apache-2.0",
7
+ "author": "code-sensei",
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "https://github.com/code-sensei/artemiskit.git",
11
+ "directory": "packages/core"
12
+ },
13
+ "bugs": {
14
+ "url": "https://github.com/code-sensei/artemiskit/issues"
15
+ },
16
+ "keywords": [
17
+ "llm",
18
+ "testing",
19
+ "ai",
20
+ "agents",
21
+ "evaluation",
22
+ "openai",
23
+ "anthropic",
24
+ "azure",
25
+ "red-team",
26
+ "security"
27
+ ],
28
+ "main": "./dist/index.js",
29
+ "types": "./dist/index.d.ts",
30
+ "exports": {
31
+ ".": {
32
+ "import": "./dist/index.js",
33
+ "types": "./dist/index.d.ts"
34
+ }
35
+ },
36
+ "scripts": {
37
+ "build": "tsc --emitDeclarationOnly && bun build ./src/index.ts --outdir ./dist --target bun",
38
+ "typecheck": "tsc --noEmit",
39
+ "clean": "rm -rf dist",
40
+ "test": "bun test"
41
+ },
42
+ "dependencies": {
43
+ "@supabase/supabase-js": "^2.39.0",
44
+ "zod": "^3.22.0",
45
+ "yaml": "^2.3.0",
46
+ "fastest-levenshtein": "^1.0.16",
47
+ "nanoid": "^5.0.0",
48
+ "pino": "^8.17.0",
49
+ "prom-client": "^15.1.0"
50
+ },
51
+ "devDependencies": {
52
+ "@types/bun": "^1.1.0",
53
+ "pino-pretty": "^10.3.0",
54
+ "typescript": "^5.3.0"
55
+ }
56
+ }
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Adapter factory for creating model clients
3
+ */
4
+
5
+ import { ArtemisError } from '../utils/errors';
6
+ import { adapterRegistry, registerBuiltInAdapters } from './registry';
7
+ import type { AdapterConfig, ModelClient, ProviderType } from './types';
8
+
9
+ let initialized = false;
10
+
11
+ /**
12
+ * Create a model adapter from configuration
13
+ */
14
+ export async function createAdapter(config: AdapterConfig): Promise<ModelClient> {
15
+ // Only try to register built-in adapters if none are registered yet
16
+ // This allows CLI or other consumers to register adapters before calling createAdapter
17
+ if (!initialized && adapterRegistry.list().length === 0) {
18
+ try {
19
+ await registerBuiltInAdapters();
20
+ } catch {
21
+ // Ignore errors - adapters may be registered externally
22
+ }
23
+ initialized = true;
24
+ }
25
+
26
+ if (adapterRegistry.isUnavailable(config.provider)) {
27
+ const reason = adapterRegistry.getUnavailableReason(config.provider);
28
+ throw new ArtemisError(
29
+ `Provider '${config.provider}' is not yet available. ${reason} ` +
30
+ `Available providers: ${adapterRegistry.list().join(', ')}`,
31
+ 'PROVIDER_UNAVAILABLE'
32
+ );
33
+ }
34
+
35
+ const factory = adapterRegistry.get(config.provider);
36
+
37
+ if (!factory) {
38
+ const available = adapterRegistry.list().join(', ');
39
+ const unavailable = adapterRegistry
40
+ .listUnavailable()
41
+ .map((u) => u.provider)
42
+ .join(', ');
43
+ throw new ArtemisError(
44
+ `Unknown provider: ${config.provider}. ` +
45
+ `Available: ${available || 'none'}. ` +
46
+ `Coming soon: ${unavailable || 'none'}`,
47
+ 'UNKNOWN_PROVIDER'
48
+ );
49
+ }
50
+
51
+ return factory(config);
52
+ }
53
+
54
+ /**
55
+ * Register a custom adapter
56
+ */
57
+ export function registerAdapter(
58
+ provider: ProviderType | string,
59
+ factory: (config: AdapterConfig) => Promise<ModelClient>
60
+ ): void {
61
+ adapterRegistry.register(provider as ProviderType, factory);
62
+ }
63
+
64
+ /**
65
+ * List available adapters
66
+ */
67
+ export function listAdapters(): {
68
+ available: string[];
69
+ unavailable: { provider: string; reason: string }[];
70
+ } {
71
+ return {
72
+ available: adapterRegistry.list(),
73
+ unavailable: adapterRegistry.listUnavailable(),
74
+ };
75
+ }
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Adapter module exports
3
+ */
4
+
5
+ export * from './types';
6
+ export * from './factory';
7
+ export * from './registry';
@@ -0,0 +1,143 @@
1
+ /**
2
+ * Adapter registry for dynamic adapter management
3
+ */
4
+
5
+ import type { AdapterConfig, ModelClient, ProviderType } from './types';
6
+
7
+ type AdapterFactory = (config: AdapterConfig) => Promise<ModelClient>;
8
+
9
+ interface UnavailableInfo {
10
+ provider: ProviderType;
11
+ reason: string;
12
+ }
13
+
14
+ /**
15
+ * Registry for adapter factories
16
+ */
17
+ class AdapterRegistry {
18
+ private adapters: Map<ProviderType, AdapterFactory> = new Map();
19
+ private unavailableProviders: Map<ProviderType, string> = new Map();
20
+
21
+ /**
22
+ * Register an adapter factory for a provider type
23
+ */
24
+ register(provider: ProviderType, factory: AdapterFactory): void {
25
+ this.adapters.set(provider, factory);
26
+ this.unavailableProviders.delete(provider);
27
+ }
28
+
29
+ /**
30
+ * Mark a provider as unavailable (post-MVP)
31
+ */
32
+ markUnavailable(provider: ProviderType, reason: string): void {
33
+ this.unavailableProviders.set(provider, reason);
34
+ }
35
+
36
+ /**
37
+ * Get an adapter factory by provider type
38
+ */
39
+ get(provider: ProviderType): AdapterFactory | undefined {
40
+ return this.adapters.get(provider);
41
+ }
42
+
43
+ /**
44
+ * Check if a provider is registered
45
+ */
46
+ has(provider: ProviderType): boolean {
47
+ return this.adapters.has(provider);
48
+ }
49
+
50
+ /**
51
+ * Check if a provider is marked as unavailable
52
+ */
53
+ isUnavailable(provider: ProviderType): boolean {
54
+ return this.unavailableProviders.has(provider);
55
+ }
56
+
57
+ /**
58
+ * Get the reason why a provider is unavailable
59
+ */
60
+ getUnavailableReason(provider: ProviderType): string | undefined {
61
+ return this.unavailableProviders.get(provider);
62
+ }
63
+
64
+ /**
65
+ * List all registered providers
66
+ */
67
+ list(): ProviderType[] {
68
+ return Array.from(this.adapters.keys());
69
+ }
70
+
71
+ /**
72
+ * List unavailable providers with reasons
73
+ */
74
+ listUnavailable(): UnavailableInfo[] {
75
+ return Array.from(this.unavailableProviders.entries()).map(([provider, reason]) => ({
76
+ provider,
77
+ reason,
78
+ }));
79
+ }
80
+ }
81
+
82
+ export const adapterRegistry = new AdapterRegistry();
83
+
84
+ /**
85
+ * Dynamically import a module with error handling
86
+ */
87
+ async function tryImport<T>(moduleName: string): Promise<T> {
88
+ try {
89
+ return await import(/* @vite-ignore */ moduleName);
90
+ } catch (error) {
91
+ throw new Error(
92
+ `Failed to load adapter module '${moduleName}'. ` +
93
+ `Make sure it's installed: bun add ${moduleName}`
94
+ );
95
+ }
96
+ }
97
+
98
+ /**
99
+ * Register built-in adapters
100
+ * Adapters are loaded lazily when first requested
101
+ */
102
+ export async function registerBuiltInAdapters(): Promise<void> {
103
+ // ============================================
104
+ // MVP ADAPTERS - Fully implemented
105
+ // ============================================
106
+
107
+ adapterRegistry.register('openai', async (config) => {
108
+ const mod = await tryImport<{ OpenAIAdapter: new (c: AdapterConfig) => ModelClient }>(
109
+ '@artemiskit/adapter-openai'
110
+ );
111
+ return new mod.OpenAIAdapter(config);
112
+ });
113
+
114
+ adapterRegistry.register('azure-openai', async (config) => {
115
+ const mod = await tryImport<{ OpenAIAdapter: new (c: AdapterConfig) => ModelClient }>(
116
+ '@artemiskit/adapter-openai'
117
+ );
118
+ return new mod.OpenAIAdapter(config);
119
+ });
120
+
121
+ adapterRegistry.register('vercel-ai', async (config) => {
122
+ const mod = await tryImport<{ VercelAIAdapter: new (c: AdapterConfig) => ModelClient }>(
123
+ '@artemiskit/adapter-vercel-ai'
124
+ );
125
+ return new mod.VercelAIAdapter(config);
126
+ });
127
+
128
+ adapterRegistry.register('anthropic', async (config) => {
129
+ const mod = await tryImport<{ AnthropicAdapter: new (c: AdapterConfig) => ModelClient }>(
130
+ '@artemiskit/adapter-anthropic'
131
+ );
132
+ return new mod.AnthropicAdapter(config);
133
+ });
134
+
135
+ // ============================================
136
+ // POST-MVP ADAPTERS - Not yet available
137
+ // ============================================
138
+ adapterRegistry.markUnavailable('google', 'Google adapter coming in v0.3.0');
139
+ adapterRegistry.markUnavailable('mistral', 'Mistral adapter coming in v0.3.0');
140
+ adapterRegistry.markUnavailable('ollama', 'Ollama adapter coming in v0.3.0');
141
+ adapterRegistry.markUnavailable('huggingface', 'Hugging Face adapter coming in v0.4.0');
142
+ adapterRegistry.markUnavailable('cohere', 'Cohere adapter coming in v0.4.0');
143
+ }