@artemiskit/cli 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/artemis-runs/my-project/-sEsU7KtJ7VE.json +188 -0
- package/bin/artemis.ts +13 -0
- package/dist/bin/artemis.d.ts +6 -0
- package/dist/bin/artemis.d.ts.map +1 -0
- package/dist/index.js +51297 -0
- package/dist/src/adapters.d.ts +6 -0
- package/dist/src/adapters.d.ts.map +1 -0
- package/dist/src/cli.d.ts +6 -0
- package/dist/src/cli.d.ts.map +1 -0
- package/dist/src/commands/compare.d.ts +6 -0
- package/dist/src/commands/compare.d.ts.map +1 -0
- package/dist/src/commands/history.d.ts +6 -0
- package/dist/src/commands/history.d.ts.map +1 -0
- package/dist/src/commands/index.d.ts +8 -0
- package/dist/src/commands/index.d.ts.map +1 -0
- package/dist/src/commands/init.d.ts +6 -0
- package/dist/src/commands/init.d.ts.map +1 -0
- package/dist/src/commands/redteam.d.ts +6 -0
- package/dist/src/commands/redteam.d.ts.map +1 -0
- package/dist/src/commands/report.d.ts +6 -0
- package/dist/src/commands/report.d.ts.map +1 -0
- package/dist/src/commands/run.d.ts +6 -0
- package/dist/src/commands/run.d.ts.map +1 -0
- package/dist/src/commands/stress.d.ts +6 -0
- package/dist/src/commands/stress.d.ts.map +1 -0
- package/dist/src/config/index.d.ts +6 -0
- package/dist/src/config/index.d.ts.map +1 -0
- package/dist/src/config/loader.d.ts +13 -0
- package/dist/src/config/loader.d.ts.map +1 -0
- package/dist/src/config/schema.d.ts +215 -0
- package/dist/src/config/schema.d.ts.map +1 -0
- package/dist/src/index.d.ts +6 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/utils/adapter.d.ts +71 -0
- package/dist/src/utils/adapter.d.ts.map +1 -0
- package/dist/src/utils/storage.d.ts +22 -0
- package/dist/src/utils/storage.d.ts.map +1 -0
- package/package.json +65 -0
- package/src/adapters.ts +33 -0
- package/src/cli.ts +34 -0
- package/src/commands/compare.ts +104 -0
- package/src/commands/history.ts +80 -0
- package/src/commands/index.ts +8 -0
- package/src/commands/init.ts +111 -0
- package/src/commands/redteam.ts +511 -0
- package/src/commands/report.ts +126 -0
- package/src/commands/run.ts +233 -0
- package/src/commands/stress.ts +501 -0
- package/src/config/index.ts +6 -0
- package/src/config/loader.ts +112 -0
- package/src/config/schema.ts +56 -0
- package/src/index.ts +6 -0
- package/src/utils/adapter.ts +542 -0
- package/src/utils/storage.ts +67 -0
- package/tsconfig.json +13 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../../src/adapters.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAuBtD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/cli.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,wBAAgB,SAAS,IAAI,OAAO,CAmBnC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../../src/commands/compare.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AASpC,wBAAgB,cAAc,IAAI,OAAO,CAwFxC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"history.d.ts","sourceRoot":"","sources":["../../../src/commands/history.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAWpC,wBAAgB,cAAc,IAAI,OAAO,CA8DxC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/commands/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../../src/commands/init.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgEpC,wBAAgB,WAAW,IAAI,OAAO,CAuCrC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/commands/redteam.ts"],"names":[],"mappings":"AAAA;;GAEG;AA+BH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAwBpC,wBAAgB,cAAc,IAAI,OAAO,CAmUxC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/commands/report.ts"],"names":[],"mappings":"AAAA;;GAEG;AAYH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAoDpC,wBAAgB,aAAa,IAAI,OAAO,CA2DvC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/commands/run.ts"],"names":[],"mappings":"AAAA;;GAEG;AAUH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAyBpC,wBAAgB,UAAU,IAAI,OAAO,CA6IpC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stress.d.ts","sourceRoot":"","sources":["../../../src/commands/stress.ts"],"names":[],"mappings":"AAAA;;GAEG;AAkBH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA0BpC,wBAAgB,aAAa,IAAI,OAAO,CAuOvC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/config/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration file loader
|
|
3
|
+
*/
|
|
4
|
+
import { type ArtemisConfig } from './schema';
|
|
5
|
+
/**
|
|
6
|
+
* Find and load the configuration file
|
|
7
|
+
*/
|
|
8
|
+
export declare function loadConfig(configPath?: string): Promise<ArtemisConfig | null>;
|
|
9
|
+
/**
|
|
10
|
+
* Get a merged config with CLI options taking precedence
|
|
11
|
+
*/
|
|
12
|
+
export declare function mergeConfig(fileConfig: ArtemisConfig | null, cliOptions: Partial<ArtemisConfig>): ArtemisConfig;
|
|
13
|
+
//# sourceMappingURL=loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,EAAE,KAAK,aAAa,EAAuB,MAAM,UAAU,CAAC;AAInE;;GAEG;AACH,wBAAsB,UAAU,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CA8BnF;AAiDD;;GAEG;AACH,wBAAgB,WAAW,CACzB,UAAU,EAAE,aAAa,GAAG,IAAI,EAChC,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,GACjC,aAAa,CAWf"}
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration schema for artemis.config.yaml
|
|
3
|
+
*/
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
declare const ProviderConfigSchema: z.ZodObject<{
|
|
6
|
+
apiKey: z.ZodOptional<z.ZodString>;
|
|
7
|
+
baseUrl: z.ZodOptional<z.ZodString>;
|
|
8
|
+
defaultModel: z.ZodOptional<z.ZodString>;
|
|
9
|
+
timeout: z.ZodOptional<z.ZodNumber>;
|
|
10
|
+
maxRetries: z.ZodOptional<z.ZodNumber>;
|
|
11
|
+
organization: z.ZodOptional<z.ZodString>;
|
|
12
|
+
resourceName: z.ZodOptional<z.ZodString>;
|
|
13
|
+
deploymentName: z.ZodOptional<z.ZodString>;
|
|
14
|
+
apiVersion: z.ZodOptional<z.ZodString>;
|
|
15
|
+
underlyingProvider: z.ZodOptional<z.ZodEnum<["openai", "azure", "anthropic", "google", "mistral"]>>;
|
|
16
|
+
}, "strip", z.ZodTypeAny, {
|
|
17
|
+
apiKey?: string | undefined;
|
|
18
|
+
baseUrl?: string | undefined;
|
|
19
|
+
defaultModel?: string | undefined;
|
|
20
|
+
timeout?: number | undefined;
|
|
21
|
+
maxRetries?: number | undefined;
|
|
22
|
+
organization?: string | undefined;
|
|
23
|
+
resourceName?: string | undefined;
|
|
24
|
+
deploymentName?: string | undefined;
|
|
25
|
+
apiVersion?: string | undefined;
|
|
26
|
+
underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
|
|
27
|
+
}, {
|
|
28
|
+
apiKey?: string | undefined;
|
|
29
|
+
baseUrl?: string | undefined;
|
|
30
|
+
defaultModel?: string | undefined;
|
|
31
|
+
timeout?: number | undefined;
|
|
32
|
+
maxRetries?: number | undefined;
|
|
33
|
+
organization?: string | undefined;
|
|
34
|
+
resourceName?: string | undefined;
|
|
35
|
+
deploymentName?: string | undefined;
|
|
36
|
+
apiVersion?: string | undefined;
|
|
37
|
+
underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
|
|
38
|
+
}>;
|
|
39
|
+
declare const StorageConfigSchema: z.ZodObject<{
|
|
40
|
+
type: z.ZodDefault<z.ZodEnum<["supabase", "local"]>>;
|
|
41
|
+
url: z.ZodOptional<z.ZodString>;
|
|
42
|
+
anonKey: z.ZodOptional<z.ZodString>;
|
|
43
|
+
bucket: z.ZodOptional<z.ZodString>;
|
|
44
|
+
basePath: z.ZodOptional<z.ZodString>;
|
|
45
|
+
}, "strip", z.ZodTypeAny, {
|
|
46
|
+
type: "supabase" | "local";
|
|
47
|
+
url?: string | undefined;
|
|
48
|
+
anonKey?: string | undefined;
|
|
49
|
+
bucket?: string | undefined;
|
|
50
|
+
basePath?: string | undefined;
|
|
51
|
+
}, {
|
|
52
|
+
type?: "supabase" | "local" | undefined;
|
|
53
|
+
url?: string | undefined;
|
|
54
|
+
anonKey?: string | undefined;
|
|
55
|
+
bucket?: string | undefined;
|
|
56
|
+
basePath?: string | undefined;
|
|
57
|
+
}>;
|
|
58
|
+
export declare const ArtemisConfigSchema: z.ZodObject<{
|
|
59
|
+
project: z.ZodDefault<z.ZodString>;
|
|
60
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
61
|
+
model: z.ZodOptional<z.ZodString>;
|
|
62
|
+
providers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
63
|
+
apiKey: z.ZodOptional<z.ZodString>;
|
|
64
|
+
baseUrl: z.ZodOptional<z.ZodString>;
|
|
65
|
+
defaultModel: z.ZodOptional<z.ZodString>;
|
|
66
|
+
timeout: z.ZodOptional<z.ZodNumber>;
|
|
67
|
+
maxRetries: z.ZodOptional<z.ZodNumber>;
|
|
68
|
+
organization: z.ZodOptional<z.ZodString>;
|
|
69
|
+
resourceName: z.ZodOptional<z.ZodString>;
|
|
70
|
+
deploymentName: z.ZodOptional<z.ZodString>;
|
|
71
|
+
apiVersion: z.ZodOptional<z.ZodString>;
|
|
72
|
+
underlyingProvider: z.ZodOptional<z.ZodEnum<["openai", "azure", "anthropic", "google", "mistral"]>>;
|
|
73
|
+
}, "strip", z.ZodTypeAny, {
|
|
74
|
+
apiKey?: string | undefined;
|
|
75
|
+
baseUrl?: string | undefined;
|
|
76
|
+
defaultModel?: string | undefined;
|
|
77
|
+
timeout?: number | undefined;
|
|
78
|
+
maxRetries?: number | undefined;
|
|
79
|
+
organization?: string | undefined;
|
|
80
|
+
resourceName?: string | undefined;
|
|
81
|
+
deploymentName?: string | undefined;
|
|
82
|
+
apiVersion?: string | undefined;
|
|
83
|
+
underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
|
|
84
|
+
}, {
|
|
85
|
+
apiKey?: string | undefined;
|
|
86
|
+
baseUrl?: string | undefined;
|
|
87
|
+
defaultModel?: string | undefined;
|
|
88
|
+
timeout?: number | undefined;
|
|
89
|
+
maxRetries?: number | undefined;
|
|
90
|
+
organization?: string | undefined;
|
|
91
|
+
resourceName?: string | undefined;
|
|
92
|
+
deploymentName?: string | undefined;
|
|
93
|
+
apiVersion?: string | undefined;
|
|
94
|
+
underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
|
|
95
|
+
}>>>;
|
|
96
|
+
storage: z.ZodOptional<z.ZodObject<{
|
|
97
|
+
type: z.ZodDefault<z.ZodEnum<["supabase", "local"]>>;
|
|
98
|
+
url: z.ZodOptional<z.ZodString>;
|
|
99
|
+
anonKey: z.ZodOptional<z.ZodString>;
|
|
100
|
+
bucket: z.ZodOptional<z.ZodString>;
|
|
101
|
+
basePath: z.ZodOptional<z.ZodString>;
|
|
102
|
+
}, "strip", z.ZodTypeAny, {
|
|
103
|
+
type: "supabase" | "local";
|
|
104
|
+
url?: string | undefined;
|
|
105
|
+
anonKey?: string | undefined;
|
|
106
|
+
bucket?: string | undefined;
|
|
107
|
+
basePath?: string | undefined;
|
|
108
|
+
}, {
|
|
109
|
+
type?: "supabase" | "local" | undefined;
|
|
110
|
+
url?: string | undefined;
|
|
111
|
+
anonKey?: string | undefined;
|
|
112
|
+
bucket?: string | undefined;
|
|
113
|
+
basePath?: string | undefined;
|
|
114
|
+
}>>;
|
|
115
|
+
scenariosDir: z.ZodDefault<z.ZodString>;
|
|
116
|
+
output: z.ZodOptional<z.ZodObject<{
|
|
117
|
+
format: z.ZodDefault<z.ZodEnum<["json", "html", "both"]>>;
|
|
118
|
+
dir: z.ZodDefault<z.ZodString>;
|
|
119
|
+
}, "strip", z.ZodTypeAny, {
|
|
120
|
+
format: "json" | "html" | "both";
|
|
121
|
+
dir: string;
|
|
122
|
+
}, {
|
|
123
|
+
format?: "json" | "html" | "both" | undefined;
|
|
124
|
+
dir?: string | undefined;
|
|
125
|
+
}>>;
|
|
126
|
+
ci: z.ZodOptional<z.ZodObject<{
|
|
127
|
+
failOnRegression: z.ZodDefault<z.ZodBoolean>;
|
|
128
|
+
regressionThreshold: z.ZodDefault<z.ZodNumber>;
|
|
129
|
+
baselineStrategy: z.ZodDefault<z.ZodEnum<["latest", "tagged", "specific"]>>;
|
|
130
|
+
baselineRunId: z.ZodOptional<z.ZodString>;
|
|
131
|
+
}, "strip", z.ZodTypeAny, {
|
|
132
|
+
failOnRegression: boolean;
|
|
133
|
+
regressionThreshold: number;
|
|
134
|
+
baselineStrategy: "latest" | "tagged" | "specific";
|
|
135
|
+
baselineRunId?: string | undefined;
|
|
136
|
+
}, {
|
|
137
|
+
failOnRegression?: boolean | undefined;
|
|
138
|
+
regressionThreshold?: number | undefined;
|
|
139
|
+
baselineStrategy?: "latest" | "tagged" | "specific" | undefined;
|
|
140
|
+
baselineRunId?: string | undefined;
|
|
141
|
+
}>>;
|
|
142
|
+
}, "strip", z.ZodTypeAny, {
|
|
143
|
+
project: string;
|
|
144
|
+
scenariosDir: string;
|
|
145
|
+
provider?: string | undefined;
|
|
146
|
+
model?: string | undefined;
|
|
147
|
+
providers?: Record<string, {
|
|
148
|
+
apiKey?: string | undefined;
|
|
149
|
+
baseUrl?: string | undefined;
|
|
150
|
+
defaultModel?: string | undefined;
|
|
151
|
+
timeout?: number | undefined;
|
|
152
|
+
maxRetries?: number | undefined;
|
|
153
|
+
organization?: string | undefined;
|
|
154
|
+
resourceName?: string | undefined;
|
|
155
|
+
deploymentName?: string | undefined;
|
|
156
|
+
apiVersion?: string | undefined;
|
|
157
|
+
underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
|
|
158
|
+
}> | undefined;
|
|
159
|
+
storage?: {
|
|
160
|
+
type: "supabase" | "local";
|
|
161
|
+
url?: string | undefined;
|
|
162
|
+
anonKey?: string | undefined;
|
|
163
|
+
bucket?: string | undefined;
|
|
164
|
+
basePath?: string | undefined;
|
|
165
|
+
} | undefined;
|
|
166
|
+
output?: {
|
|
167
|
+
format: "json" | "html" | "both";
|
|
168
|
+
dir: string;
|
|
169
|
+
} | undefined;
|
|
170
|
+
ci?: {
|
|
171
|
+
failOnRegression: boolean;
|
|
172
|
+
regressionThreshold: number;
|
|
173
|
+
baselineStrategy: "latest" | "tagged" | "specific";
|
|
174
|
+
baselineRunId?: string | undefined;
|
|
175
|
+
} | undefined;
|
|
176
|
+
}, {
|
|
177
|
+
project?: string | undefined;
|
|
178
|
+
provider?: string | undefined;
|
|
179
|
+
model?: string | undefined;
|
|
180
|
+
providers?: Record<string, {
|
|
181
|
+
apiKey?: string | undefined;
|
|
182
|
+
baseUrl?: string | undefined;
|
|
183
|
+
defaultModel?: string | undefined;
|
|
184
|
+
timeout?: number | undefined;
|
|
185
|
+
maxRetries?: number | undefined;
|
|
186
|
+
organization?: string | undefined;
|
|
187
|
+
resourceName?: string | undefined;
|
|
188
|
+
deploymentName?: string | undefined;
|
|
189
|
+
apiVersion?: string | undefined;
|
|
190
|
+
underlyingProvider?: "openai" | "anthropic" | "google" | "mistral" | "azure" | undefined;
|
|
191
|
+
}> | undefined;
|
|
192
|
+
storage?: {
|
|
193
|
+
type?: "supabase" | "local" | undefined;
|
|
194
|
+
url?: string | undefined;
|
|
195
|
+
anonKey?: string | undefined;
|
|
196
|
+
bucket?: string | undefined;
|
|
197
|
+
basePath?: string | undefined;
|
|
198
|
+
} | undefined;
|
|
199
|
+
scenariosDir?: string | undefined;
|
|
200
|
+
output?: {
|
|
201
|
+
format?: "json" | "html" | "both" | undefined;
|
|
202
|
+
dir?: string | undefined;
|
|
203
|
+
} | undefined;
|
|
204
|
+
ci?: {
|
|
205
|
+
failOnRegression?: boolean | undefined;
|
|
206
|
+
regressionThreshold?: number | undefined;
|
|
207
|
+
baselineStrategy?: "latest" | "tagged" | "specific" | undefined;
|
|
208
|
+
baselineRunId?: string | undefined;
|
|
209
|
+
} | undefined;
|
|
210
|
+
}>;
|
|
211
|
+
export type ArtemisConfig = z.infer<typeof ArtemisConfigSchema>;
|
|
212
|
+
export type ProviderConfig = z.infer<typeof ProviderConfigSchema>;
|
|
213
|
+
export type StorageConfig = z.infer<typeof StorageConfigSchema>;
|
|
214
|
+
export {};
|
|
215
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/config/schema.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAcxB,CAAC;AAEH,QAAA,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;EAMvB,CAAC;AAcH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAS9B,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared adapter configuration builder
|
|
3
|
+
*
|
|
4
|
+
* Config precedence (top wins):
|
|
5
|
+
* 1. CLI options (--provider, --model, etc.)
|
|
6
|
+
* 2. Scenario providerConfig
|
|
7
|
+
* 3. artemis.config.yaml providers.<provider>
|
|
8
|
+
* 4. Environment variables
|
|
9
|
+
* 5. Defaults
|
|
10
|
+
*/
|
|
11
|
+
import type { AdapterConfig } from '@artemiskit/core';
|
|
12
|
+
import type { ConfigSource, ResolvedConfig } from '@artemiskit/core';
|
|
13
|
+
import type { ProviderConfig } from '@artemiskit/core';
|
|
14
|
+
import type { ArtemisConfig } from '../config/schema';
|
|
15
|
+
export interface AdapterConfigOptions {
|
|
16
|
+
/** Provider name from CLI or scenario */
|
|
17
|
+
provider: string;
|
|
18
|
+
/** Model name from CLI or scenario (display/identifier) */
|
|
19
|
+
model?: string;
|
|
20
|
+
/** Temperature from CLI */
|
|
21
|
+
temperature?: number;
|
|
22
|
+
/** Max tokens from CLI */
|
|
23
|
+
maxTokens?: number;
|
|
24
|
+
/** Provider config from scenario file */
|
|
25
|
+
scenarioConfig?: ProviderConfig;
|
|
26
|
+
/** Config from artemis.config.yaml */
|
|
27
|
+
fileConfig?: ArtemisConfig | null;
|
|
28
|
+
/** Source of provider (for tracking) */
|
|
29
|
+
providerSource?: ConfigSource;
|
|
30
|
+
/** Source of model (for tracking) */
|
|
31
|
+
modelSource?: ConfigSource;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Result of building adapter configuration
|
|
35
|
+
*/
|
|
36
|
+
export interface AdapterConfigResult {
|
|
37
|
+
/** Adapter configuration for creating the client */
|
|
38
|
+
adapterConfig: AdapterConfig;
|
|
39
|
+
/** Resolved configuration with source tracking for manifest */
|
|
40
|
+
resolvedConfig: ResolvedConfig;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Build adapter configuration with proper precedence and source tracking
|
|
44
|
+
*
|
|
45
|
+
* Resolution order for each field:
|
|
46
|
+
* CLI > scenarioConfig > fileConfig.providers[provider] > environment variable > default
|
|
47
|
+
*/
|
|
48
|
+
export declare function buildAdapterConfig(options: AdapterConfigOptions): AdapterConfigResult;
|
|
49
|
+
/**
|
|
50
|
+
* Get the effective provider from various sources with source tracking
|
|
51
|
+
*/
|
|
52
|
+
export declare function resolveProviderWithSource(cliProvider?: string, scenarioProvider?: string, configProvider?: string): {
|
|
53
|
+
provider: string;
|
|
54
|
+
source: ConfigSource;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Get the effective model from various sources with source tracking
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveModelWithSource(cliModel?: string, scenarioModel?: string, configModel?: string): {
|
|
60
|
+
model: string | undefined;
|
|
61
|
+
source: ConfigSource | undefined;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* Get the effective provider from various sources (legacy - without source tracking)
|
|
65
|
+
*/
|
|
66
|
+
export declare function resolveProvider(cliProvider?: string, scenarioProvider?: string, configProvider?: string): string;
|
|
67
|
+
/**
|
|
68
|
+
* Get the effective model from various sources (legacy - without source tracking)
|
|
69
|
+
*/
|
|
70
|
+
export declare function resolveModel(cliModel?: string, scenarioModel?: string, configModel?: string): string | undefined;
|
|
71
|
+
//# sourceMappingURL=adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../src/utils/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,MAAM,WAAW,oBAAoB;IACnC,yCAAyC;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,sCAAsC;IACtC,UAAU,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IAClC,wCAAwC;IACxC,cAAc,CAAC,EAAE,YAAY,CAAC;IAC9B,qCAAqC;IACrC,WAAW,CAAC,EAAE,YAAY,CAAC;CAC5B;AAOD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oDAAoD;IACpD,aAAa,EAAE,aAAa,CAAC;IAC7B,+DAA+D;IAC/D,cAAc,EAAE,cAAc,CAAC;CAChC;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,oBAAoB,GAAG,mBAAmB,CA8ErF;AAuWD;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,MAAM,EACzB,cAAc,CAAC,EAAE,MAAM,GACtB;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAA;CAAE,CAK5C;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,CAAC,EAAE,MAAM,EACjB,aAAa,CAAC,EAAE,MAAM,EACtB,WAAW,CAAC,EAAE,MAAM,GACnB;IAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,MAAM,EAAE,YAAY,GAAG,SAAS,CAAA;CAAE,CAKjE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAC7B,WAAW,CAAC,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,MAAM,EACzB,cAAc,CAAC,EAAE,MAAM,GACtB,MAAM,CAER;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,CAAC,EAAE,MAAM,EACjB,aAAa,CAAC,EAAE,MAAM,EACtB,WAAW,CAAC,EAAE,MAAM,GACnB,MAAM,GAAG,SAAS,CAEpB"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared storage configuration builder
|
|
3
|
+
*
|
|
4
|
+
* Config precedence (top wins):
|
|
5
|
+
* 1. artemis.config.yaml storage section
|
|
6
|
+
* 2. Environment variables
|
|
7
|
+
* 3. Defaults
|
|
8
|
+
*/
|
|
9
|
+
import { type StorageAdapter } from '@artemiskit/core';
|
|
10
|
+
import type { ArtemisConfig } from '../config/schema';
|
|
11
|
+
export interface StorageOptions {
|
|
12
|
+
/** Config from artemis.config.yaml */
|
|
13
|
+
fileConfig?: ArtemisConfig | null;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Create storage adapter with config file support
|
|
17
|
+
*
|
|
18
|
+
* Resolution order:
|
|
19
|
+
* fileConfig.storage > environment variables > default (local)
|
|
20
|
+
*/
|
|
21
|
+
export declare function createStorage(options?: StorageOptions): StorageAdapter;
|
|
22
|
+
//# sourceMappingURL=storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../../../src/utils/storage.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,cAAc,EAAwB,MAAM,kBAAkB,CAAC;AAC7E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,MAAM,WAAW,cAAc;IAC7B,sCAAsC;IACtC,UAAU,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;CACnC;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,OAAO,GAAE,cAAmB,GAAG,cAAc,CA2C1E"}
|
package/package.json
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@artemiskit/cli",
|
|
3
|
+
"version": "0.1.2",
|
|
4
|
+
"description": "Command-line interface for ArtemisKit LLM evaluation toolkit",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "Apache-2.0",
|
|
7
|
+
"author": "code-sensei",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/code-sensei/artemiskit.git",
|
|
11
|
+
"directory": "packages/cli"
|
|
12
|
+
},
|
|
13
|
+
"bugs": {
|
|
14
|
+
"url": "https://github.com/code-sensei/artemiskit/issues"
|
|
15
|
+
},
|
|
16
|
+
"keywords": [
|
|
17
|
+
"llm",
|
|
18
|
+
"testing",
|
|
19
|
+
"cli",
|
|
20
|
+
"ai",
|
|
21
|
+
"agents",
|
|
22
|
+
"evaluation",
|
|
23
|
+
"openai",
|
|
24
|
+
"anthropic",
|
|
25
|
+
"red-team"
|
|
26
|
+
],
|
|
27
|
+
"bin": {
|
|
28
|
+
"artemiskit": "./bin/artemis.ts",
|
|
29
|
+
"akit": "./bin/artemis.ts"
|
|
30
|
+
},
|
|
31
|
+
"main": "./dist/index.js",
|
|
32
|
+
"types": "./dist/index.d.ts",
|
|
33
|
+
"exports": {
|
|
34
|
+
".": {
|
|
35
|
+
"import": "./dist/index.js",
|
|
36
|
+
"types": "./dist/index.d.ts"
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
"scripts": {
|
|
40
|
+
"build": "tsc && bun build ./src/index.ts --outdir ./dist --target bun",
|
|
41
|
+
"dev": "bun run ./bin/artemis.ts",
|
|
42
|
+
"typecheck": "tsc --noEmit",
|
|
43
|
+
"clean": "rm -rf dist",
|
|
44
|
+
"test": "bun test"
|
|
45
|
+
},
|
|
46
|
+
"dependencies": {
|
|
47
|
+
"@artemiskit/adapter-openai": "workspace:*",
|
|
48
|
+
"@artemiskit/adapter-vercel-ai": "workspace:*",
|
|
49
|
+
"@artemiskit/core": "workspace:*",
|
|
50
|
+
"@artemiskit/redteam": "workspace:*",
|
|
51
|
+
"@artemiskit/reports": "workspace:*",
|
|
52
|
+
"chalk": "^5.3.0",
|
|
53
|
+
"cli-table3": "^0.6.3",
|
|
54
|
+
"commander": "^12.0.0",
|
|
55
|
+
"inquirer": "^9.2.0",
|
|
56
|
+
"nanoid": "5.1.6",
|
|
57
|
+
"ora": "^8.0.0",
|
|
58
|
+
"yaml": "^2.3.4",
|
|
59
|
+
"zod": "^3.22.0"
|
|
60
|
+
},
|
|
61
|
+
"devDependencies": {
|
|
62
|
+
"@types/bun": "^1.1.0",
|
|
63
|
+
"typescript": "^5.3.0"
|
|
64
|
+
}
|
|
65
|
+
}
|
package/src/adapters.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Register adapters for CLI usage
|
|
3
|
+
* This imports adapters directly to avoid dynamic import issues in bundled code
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { OpenAIAdapter } from '@artemiskit/adapter-openai';
|
|
7
|
+
import { VercelAIAdapter } from '@artemiskit/adapter-vercel-ai';
|
|
8
|
+
import { type AdapterConfig, type ModelClient, adapterRegistry } from '@artemiskit/core';
|
|
9
|
+
|
|
10
|
+
export async function registerAdapters(): Promise<void> {
|
|
11
|
+
// OpenAI adapter
|
|
12
|
+
adapterRegistry.register('openai', async (config: AdapterConfig): Promise<ModelClient> => {
|
|
13
|
+
return new OpenAIAdapter(config);
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
// Azure OpenAI uses the same adapter
|
|
17
|
+
adapterRegistry.register('azure-openai', async (config: AdapterConfig): Promise<ModelClient> => {
|
|
18
|
+
return new OpenAIAdapter(config);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
// Vercel AI SDK adapter
|
|
22
|
+
adapterRegistry.register('vercel-ai', async (config: AdapterConfig): Promise<ModelClient> => {
|
|
23
|
+
return new VercelAIAdapter(config);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
// Mark post-MVP adapters as unavailable
|
|
27
|
+
adapterRegistry.markUnavailable('anthropic', 'Anthropic adapter coming in v0.2.0');
|
|
28
|
+
adapterRegistry.markUnavailable('google', 'Google adapter coming in v0.3.0');
|
|
29
|
+
adapterRegistry.markUnavailable('mistral', 'Mistral adapter coming in v0.3.0');
|
|
30
|
+
adapterRegistry.markUnavailable('ollama', 'Ollama adapter coming in v0.3.0');
|
|
31
|
+
adapterRegistry.markUnavailable('huggingface', 'Hugging Face adapter coming in v0.4.0');
|
|
32
|
+
adapterRegistry.markUnavailable('cohere', 'Cohere adapter coming in v0.4.0');
|
|
33
|
+
}
|
package/src/cli.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ArtemisKit CLI - Main entry point
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { Command } from 'commander';
|
|
6
|
+
import { version } from '../package.json';
|
|
7
|
+
import { compareCommand } from './commands/compare';
|
|
8
|
+
import { historyCommand } from './commands/history';
|
|
9
|
+
import { initCommand } from './commands/init';
|
|
10
|
+
import { redteamCommand } from './commands/redteam';
|
|
11
|
+
import { reportCommand } from './commands/report';
|
|
12
|
+
import { runCommand } from './commands/run';
|
|
13
|
+
import { stressCommand } from './commands/stress';
|
|
14
|
+
|
|
15
|
+
export function createCLI(): Command {
|
|
16
|
+
const program = new Command();
|
|
17
|
+
|
|
18
|
+
program
|
|
19
|
+
.name('artemiskit')
|
|
20
|
+
.description(
|
|
21
|
+
'ArtemisKit - Open-source Agent Reliability Toolkit - Test, validate, audit and evaluate LLMs and LLM-driven agents'
|
|
22
|
+
)
|
|
23
|
+
.version(version);
|
|
24
|
+
|
|
25
|
+
program.addCommand(initCommand());
|
|
26
|
+
program.addCommand(runCommand());
|
|
27
|
+
program.addCommand(compareCommand());
|
|
28
|
+
program.addCommand(historyCommand());
|
|
29
|
+
program.addCommand(reportCommand());
|
|
30
|
+
program.addCommand(redteamCommand());
|
|
31
|
+
program.addCommand(stressCommand());
|
|
32
|
+
|
|
33
|
+
return program;
|
|
34
|
+
}
|