@artemiskit/cli 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +106 -0
- package/bin/artemis.ts +0 -0
- package/dist/index.js +70954 -35881
- package/dist/src/cli.d.ts.map +1 -1
- package/dist/src/commands/compare.d.ts.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/redteam.d.ts.map +1 -1
- package/dist/src/commands/run.d.ts.map +1 -1
- package/dist/src/commands/stress.d.ts.map +1 -1
- package/dist/src/config/loader.d.ts +3 -1
- package/dist/src/config/loader.d.ts.map +1 -1
- package/dist/src/config/schema.d.ts +8 -0
- package/dist/src/config/schema.d.ts.map +1 -1
- package/dist/src/ui/index.d.ts +3 -1
- package/dist/src/ui/index.d.ts.map +1 -1
- package/dist/src/ui/panels.d.ts +21 -0
- package/dist/src/ui/panels.d.ts.map +1 -1
- package/dist/src/ui/prompts.d.ts +92 -0
- package/dist/src/ui/prompts.d.ts.map +1 -0
- package/dist/src/utils/adapter.d.ts.map +1 -1
- package/dist/src/utils/update-checker.d.ts +31 -0
- package/dist/src/utils/update-checker.d.ts.map +1 -0
- package/package.json +6 -6
- package/src/cli.ts +22 -1
- package/src/commands/compare.ts +25 -0
- package/src/commands/init.ts +221 -77
- package/src/commands/redteam.ts +63 -10
- package/src/commands/run.ts +542 -137
- package/src/commands/stress.ts +76 -3
- package/src/config/loader.ts +5 -2
- package/src/config/schema.ts +1 -0
- package/src/ui/index.ts +19 -0
- package/src/ui/panels.ts +153 -5
- package/src/ui/prompts.ts +749 -0
- package/src/utils/adapter.ts +8 -0
- package/src/utils/update-checker.ts +121 -0
|
@@ -0,0 +1,749 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Interactive prompts module
|
|
3
|
+
* Provides Inquirer-based user prompts for CLI interactivity
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import chalk from 'chalk';
|
|
7
|
+
import inquirer from 'inquirer';
|
|
8
|
+
import { Spinner } from './live-status.js';
|
|
9
|
+
import { isTTY } from './utils.js';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Check if interactive mode is available
|
|
13
|
+
*/
|
|
14
|
+
export function isInteractive(): boolean {
|
|
15
|
+
return isTTY && process.stdin.isTTY === true;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Provider options for selection
|
|
20
|
+
* Note: Only providers with implemented adapters are included
|
|
21
|
+
*/
|
|
22
|
+
export const PROVIDER_CHOICES = [
|
|
23
|
+
{ name: 'OpenAI (GPT-4o, GPT-4.1, o3, etc.)', value: 'openai' },
|
|
24
|
+
{ name: 'Azure OpenAI', value: 'azure-openai' },
|
|
25
|
+
{ name: 'Vercel AI SDK', value: 'vercel-ai' },
|
|
26
|
+
{ name: 'Anthropic (Claude)', value: 'anthropic' },
|
|
27
|
+
// { name: 'Google AI (Gemini)', value: 'google' },
|
|
28
|
+
// { name: 'Mistral AI', value: 'mistral' },
|
|
29
|
+
// { name: 'Ollama (Local)', value: 'ollama' },
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Known models by provider - used for validation
|
|
34
|
+
* Updated January 2026
|
|
35
|
+
*/
|
|
36
|
+
export const KNOWN_MODELS: Record<string, string[]> = {
|
|
37
|
+
openai: [
|
|
38
|
+
// GPT-5 series
|
|
39
|
+
'gpt-5.2',
|
|
40
|
+
'gpt-5.1',
|
|
41
|
+
'gpt-5',
|
|
42
|
+
// GPT-4.1 series
|
|
43
|
+
'gpt-4.1',
|
|
44
|
+
'gpt-4.1-mini',
|
|
45
|
+
'gpt-4.1-nano',
|
|
46
|
+
// GPT-4o series
|
|
47
|
+
'gpt-4o',
|
|
48
|
+
'gpt-4o-mini',
|
|
49
|
+
'gpt-4o-audio-preview',
|
|
50
|
+
// GPT-4 series
|
|
51
|
+
'gpt-4-turbo',
|
|
52
|
+
'gpt-4-turbo-preview',
|
|
53
|
+
'gpt-4',
|
|
54
|
+
// o-series reasoning models
|
|
55
|
+
'o4-mini',
|
|
56
|
+
'o3',
|
|
57
|
+
'o3-mini',
|
|
58
|
+
'o1',
|
|
59
|
+
'o1-mini',
|
|
60
|
+
'o1-preview',
|
|
61
|
+
// Legacy
|
|
62
|
+
'gpt-3.5-turbo',
|
|
63
|
+
'gpt-3.5-turbo-16k',
|
|
64
|
+
],
|
|
65
|
+
'azure-openai': [
|
|
66
|
+
// Azure uses deployment names, so we can't validate strictly
|
|
67
|
+
// Common deployment patterns
|
|
68
|
+
],
|
|
69
|
+
'vercel-ai': [
|
|
70
|
+
// Vercel AI SDK uses provider/model format or direct model names
|
|
71
|
+
// It wraps other providers, so validation is provider-dependent
|
|
72
|
+
],
|
|
73
|
+
anthropic: [
|
|
74
|
+
// Claude 4.5 series (latest)
|
|
75
|
+
'claude-opus-4-5-20250901',
|
|
76
|
+
'claude-sonnet-4-5-20250929',
|
|
77
|
+
'claude-haiku-4-5-20251015',
|
|
78
|
+
// Claude 4.1 series
|
|
79
|
+
'claude-opus-4-1-20250805',
|
|
80
|
+
// Claude 4 series
|
|
81
|
+
'claude-opus-4-20250514',
|
|
82
|
+
'claude-sonnet-4-20250514',
|
|
83
|
+
// Claude 3.5 series
|
|
84
|
+
'claude-3-5-sonnet-20241022',
|
|
85
|
+
'claude-3-5-sonnet-20240620',
|
|
86
|
+
'claude-3-5-haiku-20241022',
|
|
87
|
+
// Claude 3 series (some deprecated)
|
|
88
|
+
'claude-3-opus-20240229',
|
|
89
|
+
'claude-3-sonnet-20240229',
|
|
90
|
+
'claude-3-haiku-20240307',
|
|
91
|
+
],
|
|
92
|
+
google: [
|
|
93
|
+
// Gemini 3.0 series (Latest)
|
|
94
|
+
'gemini-3.0-ultra',
|
|
95
|
+
'gemini-3.0-pro',
|
|
96
|
+
'gemini-3.0-flash',
|
|
97
|
+
// Gemini 2.5 series
|
|
98
|
+
'gemini-2.5-pro',
|
|
99
|
+
'gemini-2.5-flash',
|
|
100
|
+
'gemini-2.5-flash-preview-04-17',
|
|
101
|
+
// Gemini 2.0 series
|
|
102
|
+
'gemini-2.0-flash',
|
|
103
|
+
'gemini-2.0-flash-lite',
|
|
104
|
+
'gemini-2.0-flash-thinking-exp',
|
|
105
|
+
'gemini-2.0-pro',
|
|
106
|
+
'gemini-2.0-pro-exp',
|
|
107
|
+
// Gemini 1.5 series
|
|
108
|
+
'gemini-1.5-pro',
|
|
109
|
+
'gemini-1.5-pro-latest',
|
|
110
|
+
'gemini-1.5-flash',
|
|
111
|
+
'gemini-1.5-flash-latest',
|
|
112
|
+
// Legacy
|
|
113
|
+
'gemini-pro',
|
|
114
|
+
'gemini-pro-vision',
|
|
115
|
+
],
|
|
116
|
+
mistral: [
|
|
117
|
+
// Latest models
|
|
118
|
+
'mistral-large-latest',
|
|
119
|
+
'mistral-large-2411',
|
|
120
|
+
'mistral-medium-latest',
|
|
121
|
+
'mistral-small-latest',
|
|
122
|
+
'mistral-small-2409',
|
|
123
|
+
// Specialized
|
|
124
|
+
'codestral-latest',
|
|
125
|
+
'codestral-2405',
|
|
126
|
+
'ministral-8b-latest',
|
|
127
|
+
'ministral-3b-latest',
|
|
128
|
+
// Open models
|
|
129
|
+
'open-mistral-nemo',
|
|
130
|
+
'open-mixtral-8x22b',
|
|
131
|
+
'open-mixtral-8x7b',
|
|
132
|
+
],
|
|
133
|
+
ollama: [
|
|
134
|
+
// Popular local models
|
|
135
|
+
'llama3.3',
|
|
136
|
+
'llama3.2',
|
|
137
|
+
'llama3.1',
|
|
138
|
+
'llama3',
|
|
139
|
+
'llama2',
|
|
140
|
+
'mistral',
|
|
141
|
+
'mixtral',
|
|
142
|
+
'codellama',
|
|
143
|
+
'phi3',
|
|
144
|
+
'gemma2',
|
|
145
|
+
'qwen2.5',
|
|
146
|
+
'deepseek-coder-v2',
|
|
147
|
+
],
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Model choices displayed in the prompt
|
|
152
|
+
* Updated January 2026
|
|
153
|
+
*/
|
|
154
|
+
export const MODEL_CHOICES: Record<string, { name: string; value: string }[]> = {
|
|
155
|
+
openai: [
|
|
156
|
+
{ name: 'gpt-4.1 (Latest, best for coding)', value: 'gpt-4.1' },
|
|
157
|
+
{ name: 'gpt-4.1-mini (Fast, cost-effective)', value: 'gpt-4.1-mini' },
|
|
158
|
+
{ name: 'gpt-4o (Multimodal, audio support)', value: 'gpt-4o' },
|
|
159
|
+
{ name: 'gpt-4o-mini (Fast multimodal)', value: 'gpt-4o-mini' },
|
|
160
|
+
{ name: 'o3 (Advanced reasoning)', value: 'o3' },
|
|
161
|
+
{ name: 'o3-mini (Fast reasoning)', value: 'o3-mini' },
|
|
162
|
+
{ name: 'gpt-3.5-turbo (Legacy, budget)', value: 'gpt-3.5-turbo' },
|
|
163
|
+
{ name: 'Other (specify)', value: '__custom__' },
|
|
164
|
+
],
|
|
165
|
+
'azure-openai': [
|
|
166
|
+
{ name: 'Use deployment name from config', value: '' },
|
|
167
|
+
{ name: 'Enter deployment name', value: '__custom__' },
|
|
168
|
+
],
|
|
169
|
+
'vercel-ai': [
|
|
170
|
+
{ name: 'gpt-4.1 (via OpenAI)', value: 'gpt-4.1' },
|
|
171
|
+
{ name: 'gpt-4o (via OpenAI)', value: 'gpt-4o' },
|
|
172
|
+
{ name: 'claude-sonnet-4-5 (via Anthropic)', value: 'claude-sonnet-4-5-20250929' },
|
|
173
|
+
{ name: 'gemini-2.0-flash (via Google)', value: 'gemini-2.0-flash' },
|
|
174
|
+
{ name: 'Other (specify)', value: '__custom__' },
|
|
175
|
+
],
|
|
176
|
+
anthropic: [
|
|
177
|
+
{ name: 'claude-sonnet-4-5 (Latest, recommended)', value: 'claude-sonnet-4-5-20250929' },
|
|
178
|
+
{ name: 'claude-opus-4-5 (Most capable)', value: 'claude-opus-4-5-20250901' },
|
|
179
|
+
{ name: 'claude-haiku-4-5 (Fast, efficient)', value: 'claude-haiku-4-5-20251015' },
|
|
180
|
+
{ name: 'claude-opus-4-1 (Agentic tasks)', value: 'claude-opus-4-1-20250805' },
|
|
181
|
+
{ name: 'claude-sonnet-4 (Previous gen)', value: 'claude-sonnet-4-20250514' },
|
|
182
|
+
{ name: 'claude-3-5-sonnet (Legacy)', value: 'claude-3-5-sonnet-20241022' },
|
|
183
|
+
{ name: 'Other (specify)', value: '__custom__' },
|
|
184
|
+
],
|
|
185
|
+
google: [
|
|
186
|
+
{ name: 'gemini-3.0-pro (Latest, most capable)', value: 'gemini-3.0-pro' },
|
|
187
|
+
{ name: 'gemini-3.0-flash (Latest, fast)', value: 'gemini-3.0-flash' },
|
|
188
|
+
{ name: 'gemini-2.5-pro (Previous gen, capable)', value: 'gemini-2.5-pro' },
|
|
189
|
+
{ name: 'gemini-2.5-flash (Previous gen, fast)', value: 'gemini-2.5-flash' },
|
|
190
|
+
{ name: 'gemini-2.0-flash (Fast multimodal)', value: 'gemini-2.0-flash' },
|
|
191
|
+
{ name: 'gemini-1.5-pro (Long context)', value: 'gemini-1.5-pro' },
|
|
192
|
+
{ name: 'Other (specify)', value: '__custom__' },
|
|
193
|
+
],
|
|
194
|
+
mistral: [
|
|
195
|
+
{ name: 'mistral-large-latest (Most capable)', value: 'mistral-large-latest' },
|
|
196
|
+
{ name: 'mistral-small-latest (Fast, efficient)', value: 'mistral-small-latest' },
|
|
197
|
+
{ name: 'codestral-latest (Code generation)', value: 'codestral-latest' },
|
|
198
|
+
{ name: 'ministral-8b-latest (Lightweight)', value: 'ministral-8b-latest' },
|
|
199
|
+
{ name: 'Other (specify)', value: '__custom__' },
|
|
200
|
+
],
|
|
201
|
+
ollama: [
|
|
202
|
+
{ name: 'llama3.3 (Latest Llama)', value: 'llama3.3' },
|
|
203
|
+
{ name: 'llama3.2 (Multimodal)', value: 'llama3.2' },
|
|
204
|
+
{ name: 'mistral (7B, fast)', value: 'mistral' },
|
|
205
|
+
{ name: 'codellama (Code generation)', value: 'codellama' },
|
|
206
|
+
{ name: 'phi3 (Microsoft, efficient)', value: 'phi3' },
|
|
207
|
+
{ name: 'qwen2.5 (Alibaba)', value: 'qwen2.5' },
|
|
208
|
+
{ name: 'Other (specify)', value: '__custom__' },
|
|
209
|
+
],
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Prompt user to select a provider
|
|
214
|
+
*/
|
|
215
|
+
export async function promptProvider(message = 'Select a provider:'): Promise<string> {
|
|
216
|
+
const { provider } = await inquirer.prompt([
|
|
217
|
+
{
|
|
218
|
+
type: 'list',
|
|
219
|
+
name: 'provider',
|
|
220
|
+
message,
|
|
221
|
+
choices: PROVIDER_CHOICES,
|
|
222
|
+
},
|
|
223
|
+
]);
|
|
224
|
+
return provider;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Vercel AI SDK provider detection patterns
|
|
229
|
+
* Maps model name prefixes to their underlying provider
|
|
230
|
+
* Based on Vercel AI SDK documentation (January 2026)
|
|
231
|
+
*/
|
|
232
|
+
const VERCEL_AI_PROVIDER_PATTERNS: { pattern: RegExp; provider: string; description: string }[] = [
|
|
233
|
+
// OpenAI patterns
|
|
234
|
+
{ pattern: /^gpt-/i, provider: 'openai', description: 'OpenAI GPT models' },
|
|
235
|
+
{ pattern: /^o[134]-/i, provider: 'openai', description: 'OpenAI o-series reasoning models' },
|
|
236
|
+
{ pattern: /^chatgpt-/i, provider: 'openai', description: 'OpenAI ChatGPT models' },
|
|
237
|
+
{ pattern: /^davinci/i, provider: 'openai', description: 'OpenAI Davinci models' },
|
|
238
|
+
|
|
239
|
+
// Anthropic patterns
|
|
240
|
+
{ pattern: /^claude-/i, provider: 'anthropic', description: 'Anthropic Claude models' },
|
|
241
|
+
|
|
242
|
+
// Google patterns
|
|
243
|
+
{ pattern: /^gemini-/i, provider: 'google', description: 'Google Gemini models' },
|
|
244
|
+
{ pattern: /^models\/gemini/i, provider: 'google', description: 'Google Gemini (full path)' },
|
|
245
|
+
|
|
246
|
+
// Mistral patterns
|
|
247
|
+
{ pattern: /^mistral-/i, provider: 'mistral', description: 'Mistral AI models' },
|
|
248
|
+
{ pattern: /^pixtral-/i, provider: 'mistral', description: 'Mistral Pixtral vision models' },
|
|
249
|
+
{ pattern: /^ministral-/i, provider: 'mistral', description: 'Mistral Ministral models' },
|
|
250
|
+
{ pattern: /^magistral-/i, provider: 'mistral', description: 'Mistral Magistral models' },
|
|
251
|
+
{ pattern: /^codestral/i, provider: 'mistral', description: 'Mistral Codestral models' },
|
|
252
|
+
{ pattern: /^open-mistral-/i, provider: 'mistral', description: 'Mistral open models' },
|
|
253
|
+
{ pattern: /^open-mixtral-/i, provider: 'mistral', description: 'Mistral Mixtral models' },
|
|
254
|
+
|
|
255
|
+
// xAI patterns
|
|
256
|
+
{ pattern: /^grok-/i, provider: 'xai', description: 'xAI Grok models' },
|
|
257
|
+
|
|
258
|
+
// DeepSeek patterns
|
|
259
|
+
{ pattern: /^deepseek-/i, provider: 'deepseek', description: 'DeepSeek models' },
|
|
260
|
+
|
|
261
|
+
// Cohere patterns
|
|
262
|
+
{ pattern: /^command-/i, provider: 'cohere', description: 'Cohere Command models' },
|
|
263
|
+
{ pattern: /^c4ai-/i, provider: 'cohere', description: 'Cohere C4AI models' },
|
|
264
|
+
|
|
265
|
+
// Meta/Llama patterns (could be Groq, Together, Fireworks, etc.)
|
|
266
|
+
{ pattern: /^llama-/i, provider: 'meta', description: 'Meta Llama models (various providers)' },
|
|
267
|
+
{ pattern: /^meta-llama/i, provider: 'meta', description: 'Meta Llama models (full name)' },
|
|
268
|
+
|
|
269
|
+
// Groq patterns
|
|
270
|
+
{ pattern: /^groq\//i, provider: 'groq', description: 'Groq provider prefix' },
|
|
271
|
+
|
|
272
|
+
// Amazon Bedrock patterns
|
|
273
|
+
{ pattern: /^amazon\./i, provider: 'amazon-bedrock', description: 'Amazon Bedrock models' },
|
|
274
|
+
{ pattern: /^anthropic\./i, provider: 'amazon-bedrock', description: 'Anthropic via Bedrock' },
|
|
275
|
+
|
|
276
|
+
// Azure patterns
|
|
277
|
+
{ pattern: /^azure\//i, provider: 'azure-openai', description: 'Azure OpenAI deployment' },
|
|
278
|
+
];
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Detect the underlying provider for a Vercel AI SDK model
|
|
282
|
+
*/
|
|
283
|
+
function detectVercelAIProvider(model: string): { provider: string; description: string } | null {
|
|
284
|
+
for (const { pattern, provider, description } of VERCEL_AI_PROVIDER_PATTERNS) {
|
|
285
|
+
if (pattern.test(model)) {
|
|
286
|
+
return { provider, description };
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return null;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Check if a model is in the known models list for a provider
|
|
294
|
+
*/
|
|
295
|
+
function isKnownModel(provider: string, model: string): boolean {
|
|
296
|
+
const knownModels = KNOWN_MODELS[provider];
|
|
297
|
+
if (!knownModels || knownModels.length === 0) {
|
|
298
|
+
// Providers like azure-openai and vercel-ai don't have strict model lists
|
|
299
|
+
return true;
|
|
300
|
+
}
|
|
301
|
+
return knownModels.includes(model);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Validate a custom model with the user
|
|
306
|
+
* Implements hybrid validation: static check + optional API validation
|
|
307
|
+
*/
|
|
308
|
+
async function validateCustomModel(provider: string, model: string): Promise<string | null> {
|
|
309
|
+
// Handle Azure OpenAI - deployment names are user-defined
|
|
310
|
+
if (provider === 'azure-openai') {
|
|
311
|
+
console.log(chalk.yellow('\n ⚠ Azure OpenAI uses deployment names, not model names.\n'));
|
|
312
|
+
console.log(
|
|
313
|
+
chalk.dim(
|
|
314
|
+
' Ensure your deployment exists in your Azure OpenAI resource.\n' +
|
|
315
|
+
' Common deployment names: gpt-4o, gpt-4-turbo, gpt-35-turbo\n'
|
|
316
|
+
)
|
|
317
|
+
);
|
|
318
|
+
|
|
319
|
+
const { azureAction } = await inquirer.prompt([
|
|
320
|
+
{
|
|
321
|
+
type: 'list',
|
|
322
|
+
name: 'azureAction',
|
|
323
|
+
message: 'How would you like to proceed?',
|
|
324
|
+
choices: [
|
|
325
|
+
{ name: `Continue with "${model}" (I know this deployment exists)`, value: 'continue' },
|
|
326
|
+
{ name: 'Test the deployment with a quick API call', value: 'validate' },
|
|
327
|
+
{ name: 'Enter a different deployment name', value: 'retry' },
|
|
328
|
+
],
|
|
329
|
+
},
|
|
330
|
+
]);
|
|
331
|
+
|
|
332
|
+
if (azureAction === 'continue') {
|
|
333
|
+
console.log(chalk.dim(`\n Using deployment "${model}".\n`));
|
|
334
|
+
return model;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (azureAction === 'retry') {
|
|
338
|
+
return null;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
if (azureAction === 'validate') {
|
|
342
|
+
return await performApiValidation(provider, model);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
return model;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Handle Vercel AI SDK - detect underlying provider and validate
|
|
349
|
+
if (provider === 'vercel-ai') {
|
|
350
|
+
const detected = detectVercelAIProvider(model);
|
|
351
|
+
|
|
352
|
+
if (detected) {
|
|
353
|
+
console.log(chalk.cyan(`\n ✓ Detected: ${detected.description} (${detected.provider})\n`));
|
|
354
|
+
|
|
355
|
+
// Check if the model is known for the underlying provider
|
|
356
|
+
if (isKnownModel(detected.provider, model)) {
|
|
357
|
+
console.log(chalk.dim(` Model "${model}" is recognized.\n`));
|
|
358
|
+
return model;
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Model not known - offer validation
|
|
362
|
+
console.log(
|
|
363
|
+
chalk.yellow(` ⚠ "${model}" is not in our known models list for ${detected.provider}.\n`)
|
|
364
|
+
);
|
|
365
|
+
} else {
|
|
366
|
+
console.log(
|
|
367
|
+
chalk.yellow(
|
|
368
|
+
`\n ⚠ Could not detect provider for "${model}".\n This might be a custom model or provider-specific format.\n`
|
|
369
|
+
)
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
const { vercelAction } = await inquirer.prompt([
|
|
374
|
+
{
|
|
375
|
+
type: 'list',
|
|
376
|
+
name: 'vercelAction',
|
|
377
|
+
message: 'How would you like to proceed?',
|
|
378
|
+
choices: [
|
|
379
|
+
{ name: `Continue with "${model}"`, value: 'continue' },
|
|
380
|
+
{ name: 'Test the model with a quick API call', value: 'validate' },
|
|
381
|
+
{ name: 'Enter a different model', value: 'retry' },
|
|
382
|
+
],
|
|
383
|
+
},
|
|
384
|
+
]);
|
|
385
|
+
|
|
386
|
+
if (vercelAction === 'continue') {
|
|
387
|
+
console.log(chalk.dim(`\n Using model "${model}".\n`));
|
|
388
|
+
return model;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
if (vercelAction === 'retry') {
|
|
392
|
+
return null;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
if (vercelAction === 'validate') {
|
|
396
|
+
return await performApiValidation(provider, model);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
return model;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
// Standard provider validation
|
|
403
|
+
// Check if model is in known list
|
|
404
|
+
if (isKnownModel(provider, model)) {
|
|
405
|
+
return model;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Model not in known list - prompt user
|
|
409
|
+
console.log(chalk.yellow(`\n ⚠ "${model}" is not in our known models list for ${provider}.\n`));
|
|
410
|
+
console.log(chalk.dim(' This could be a fine-tuned model, new release, or a typo.\n'));
|
|
411
|
+
|
|
412
|
+
const { action } = await inquirer.prompt([
|
|
413
|
+
{
|
|
414
|
+
type: 'list',
|
|
415
|
+
name: 'action',
|
|
416
|
+
message: 'How would you like to proceed?',
|
|
417
|
+
choices: [
|
|
418
|
+
{ name: 'Continue anyway (I know this model exists)', value: 'continue' },
|
|
419
|
+
{ name: 'Test the model with a quick API call', value: 'validate' },
|
|
420
|
+
{ name: 'Enter a different model', value: 'retry' },
|
|
421
|
+
],
|
|
422
|
+
},
|
|
423
|
+
]);
|
|
424
|
+
|
|
425
|
+
if (action === 'continue') {
|
|
426
|
+
console.log(chalk.dim(`\n Proceeding with model "${model}".\n`));
|
|
427
|
+
return model;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
if (action === 'retry') {
|
|
431
|
+
return null;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
if (action === 'validate') {
|
|
435
|
+
return await performApiValidation(provider, model);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
return model;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Perform API validation for a model
|
|
443
|
+
*/
|
|
444
|
+
async function performApiValidation(provider: string, model: string): Promise<string | null> {
|
|
445
|
+
const spinner = new Spinner();
|
|
446
|
+
spinner.start(`Validating model "${model}" with ${provider}...`);
|
|
447
|
+
|
|
448
|
+
try {
|
|
449
|
+
// Dynamic import to avoid circular dependencies
|
|
450
|
+
const { createAdapter } = await import('@artemiskit/core');
|
|
451
|
+
|
|
452
|
+
const client = await createAdapter({
|
|
453
|
+
provider: provider as 'openai' | 'anthropic' | 'azure-openai' | 'vercel-ai',
|
|
454
|
+
defaultModel: model,
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
// Make minimal API call using generate
|
|
458
|
+
await client.generate({
|
|
459
|
+
prompt: [{ role: 'user', content: 'hi' }],
|
|
460
|
+
maxTokens: 1,
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
spinner.succeed(`Model "${model}" validated successfully`);
|
|
464
|
+
return model;
|
|
465
|
+
} catch (error) {
|
|
466
|
+
spinner.fail(`Model validation failed: ${(error as Error).message}`);
|
|
467
|
+
|
|
468
|
+
const { retryAfterFail } = await inquirer.prompt([
|
|
469
|
+
{
|
|
470
|
+
type: 'confirm',
|
|
471
|
+
name: 'retryAfterFail',
|
|
472
|
+
message: 'Would you like to enter a different model?',
|
|
473
|
+
default: true,
|
|
474
|
+
},
|
|
475
|
+
]);
|
|
476
|
+
|
|
477
|
+
if (retryAfterFail) {
|
|
478
|
+
return null;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// User wants to proceed anyway despite failure
|
|
482
|
+
const { forceUse } = await inquirer.prompt([
|
|
483
|
+
{
|
|
484
|
+
type: 'confirm',
|
|
485
|
+
name: 'forceUse',
|
|
486
|
+
message: `Use "${model}" anyway? (API call failed but you might have different credentials at runtime)`,
|
|
487
|
+
default: false,
|
|
488
|
+
},
|
|
489
|
+
]);
|
|
490
|
+
|
|
491
|
+
return forceUse ? model : null;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* Prompt user to select a model for a given provider
|
|
497
|
+
* Includes hybrid validation for custom models
|
|
498
|
+
*/
|
|
499
|
+
export async function promptModel(provider: string, message = 'Select a model:'): Promise<string> {
|
|
500
|
+
const choices = MODEL_CHOICES[provider] || [{ name: 'Enter model name', value: '__custom__' }];
|
|
501
|
+
|
|
502
|
+
const { model } = await inquirer.prompt([
|
|
503
|
+
{
|
|
504
|
+
type: 'list',
|
|
505
|
+
name: 'model',
|
|
506
|
+
message,
|
|
507
|
+
choices,
|
|
508
|
+
},
|
|
509
|
+
]);
|
|
510
|
+
|
|
511
|
+
if (model === '__custom__') {
|
|
512
|
+
const { customModel } = await inquirer.prompt([
|
|
513
|
+
{
|
|
514
|
+
type: 'input',
|
|
515
|
+
name: 'customModel',
|
|
516
|
+
message: 'Enter model name:',
|
|
517
|
+
validate: (input: string) => input.trim().length > 0 || 'Model name is required',
|
|
518
|
+
},
|
|
519
|
+
]);
|
|
520
|
+
|
|
521
|
+
// Validate the custom model
|
|
522
|
+
const validatedModel = await validateCustomModel(provider, customModel.trim());
|
|
523
|
+
|
|
524
|
+
if (validatedModel === null) {
|
|
525
|
+
// User wants to retry - recursively call promptModel
|
|
526
|
+
return promptModel(provider, message);
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
return validatedModel;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
return model;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/**
|
|
536
|
+
* Prompt user to select scenarios from a list
|
|
537
|
+
*/
|
|
538
|
+
export async function promptScenarios(
|
|
539
|
+
scenarios: { path: string; name: string }[],
|
|
540
|
+
message = 'Select scenarios to run:'
|
|
541
|
+
): Promise<string[]> {
|
|
542
|
+
if (scenarios.length === 0) {
|
|
543
|
+
return [];
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
const { selected } = await inquirer.prompt([
|
|
547
|
+
{
|
|
548
|
+
type: 'checkbox',
|
|
549
|
+
name: 'selected',
|
|
550
|
+
message,
|
|
551
|
+
choices: scenarios.map((s) => ({
|
|
552
|
+
name: `${s.name} ${chalk.dim(`(${s.path})`)}`,
|
|
553
|
+
value: s.path,
|
|
554
|
+
checked: true,
|
|
555
|
+
})),
|
|
556
|
+
validate: (input: string[]) => input.length > 0 || 'Please select at least one scenario',
|
|
557
|
+
},
|
|
558
|
+
]);
|
|
559
|
+
|
|
560
|
+
return selected;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Prompt for confirmation
|
|
565
|
+
*/
|
|
566
|
+
export async function promptConfirm(message: string, defaultValue = true): Promise<boolean> {
|
|
567
|
+
const { confirmed } = await inquirer.prompt([
|
|
568
|
+
{
|
|
569
|
+
type: 'confirm',
|
|
570
|
+
name: 'confirmed',
|
|
571
|
+
message,
|
|
572
|
+
default: defaultValue,
|
|
573
|
+
},
|
|
574
|
+
]);
|
|
575
|
+
return confirmed;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Prompt for text input
|
|
580
|
+
*/
|
|
581
|
+
export async function promptInput(
|
|
582
|
+
message: string,
|
|
583
|
+
options: {
|
|
584
|
+
default?: string;
|
|
585
|
+
validate?: (input: string) => boolean | string;
|
|
586
|
+
} = {}
|
|
587
|
+
): Promise<string> {
|
|
588
|
+
const { value } = await inquirer.prompt([
|
|
589
|
+
{
|
|
590
|
+
type: 'input',
|
|
591
|
+
name: 'value',
|
|
592
|
+
message,
|
|
593
|
+
default: options.default,
|
|
594
|
+
validate: options.validate,
|
|
595
|
+
},
|
|
596
|
+
]);
|
|
597
|
+
return value;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/**
|
|
601
|
+
* Prompt for a password/secret (hidden input)
|
|
602
|
+
*/
|
|
603
|
+
export async function promptPassword(
|
|
604
|
+
message: string,
|
|
605
|
+
options: {
|
|
606
|
+
validate?: (input: string) => boolean | string;
|
|
607
|
+
} = {}
|
|
608
|
+
): Promise<string> {
|
|
609
|
+
const { value } = await inquirer.prompt([
|
|
610
|
+
{
|
|
611
|
+
type: 'password',
|
|
612
|
+
name: 'value',
|
|
613
|
+
message,
|
|
614
|
+
mask: '*',
|
|
615
|
+
validate: options.validate,
|
|
616
|
+
},
|
|
617
|
+
]);
|
|
618
|
+
return value;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Prompt for selection from a list
|
|
623
|
+
*/
|
|
624
|
+
export async function promptSelect<T extends string>(
|
|
625
|
+
message: string,
|
|
626
|
+
choices: { name: string; value: T }[]
|
|
627
|
+
): Promise<T> {
|
|
628
|
+
const { selected } = await inquirer.prompt([
|
|
629
|
+
{
|
|
630
|
+
type: 'list',
|
|
631
|
+
name: 'selected',
|
|
632
|
+
message,
|
|
633
|
+
choices,
|
|
634
|
+
},
|
|
635
|
+
]);
|
|
636
|
+
return selected;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* Interactive init wizard configuration
|
|
641
|
+
*/
|
|
642
|
+
export interface InitWizardResult {
|
|
643
|
+
projectName: string;
|
|
644
|
+
provider: string;
|
|
645
|
+
model: string;
|
|
646
|
+
storageType: 'local' | 'supabase';
|
|
647
|
+
createExample: boolean;
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Run the interactive init wizard
|
|
652
|
+
*/
|
|
653
|
+
export async function runInitWizard(): Promise<InitWizardResult> {
|
|
654
|
+
console.log(chalk.cyan("\n Let's set up ArtemisKit for your project!\n"));
|
|
655
|
+
|
|
656
|
+
// Project name
|
|
657
|
+
const { projectName } = await inquirer.prompt([
|
|
658
|
+
{
|
|
659
|
+
type: 'input',
|
|
660
|
+
name: 'projectName',
|
|
661
|
+
message: 'Project name:',
|
|
662
|
+
default: 'my-project',
|
|
663
|
+
validate: (input: string) =>
|
|
664
|
+
/^[a-z0-9-]+$/i.test(input) || 'Project name must be alphanumeric with dashes',
|
|
665
|
+
},
|
|
666
|
+
]);
|
|
667
|
+
|
|
668
|
+
// Provider selection
|
|
669
|
+
const provider = await promptProvider('Select your primary provider:');
|
|
670
|
+
|
|
671
|
+
// Model selection (with validation)
|
|
672
|
+
const model = await promptModel(provider, 'Select your default model:');
|
|
673
|
+
|
|
674
|
+
// Storage type
|
|
675
|
+
const { storageType } = await inquirer.prompt([
|
|
676
|
+
{
|
|
677
|
+
type: 'list',
|
|
678
|
+
name: 'storageType',
|
|
679
|
+
message: 'Where should test results be stored?',
|
|
680
|
+
choices: [
|
|
681
|
+
{ name: 'Local filesystem (./artemis-runs)', value: 'local' },
|
|
682
|
+
{ name: 'Supabase (cloud storage)', value: 'supabase' },
|
|
683
|
+
],
|
|
684
|
+
},
|
|
685
|
+
]);
|
|
686
|
+
|
|
687
|
+
// Create example scenario
|
|
688
|
+
const createExample = await promptConfirm('Create an example scenario?', true);
|
|
689
|
+
|
|
690
|
+
return {
|
|
691
|
+
projectName,
|
|
692
|
+
provider,
|
|
693
|
+
model,
|
|
694
|
+
storageType,
|
|
695
|
+
createExample,
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* Prompt for API key if not set
|
|
701
|
+
*/
|
|
702
|
+
export async function promptApiKeyIfNeeded(
|
|
703
|
+
provider: string,
|
|
704
|
+
envVarName: string
|
|
705
|
+
): Promise<string | null> {
|
|
706
|
+
const existingKey = process.env[envVarName];
|
|
707
|
+
if (existingKey) {
|
|
708
|
+
return null; // Already set
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
console.log(chalk.yellow(`\n ${chalk.bold(envVarName)} is not set in your environment.\n`));
|
|
712
|
+
|
|
713
|
+
const { action } = await inquirer.prompt([
|
|
714
|
+
{
|
|
715
|
+
type: 'list',
|
|
716
|
+
name: 'action',
|
|
717
|
+
message: `How would you like to provide the ${provider} API key?`,
|
|
718
|
+
choices: [
|
|
719
|
+
{ name: 'Enter it now (for this session only)', value: 'enter' },
|
|
720
|
+
{ name: "Skip (I'll set it later)", value: 'skip' },
|
|
721
|
+
],
|
|
722
|
+
},
|
|
723
|
+
]);
|
|
724
|
+
|
|
725
|
+
if (action === 'skip') {
|
|
726
|
+
return null;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
const apiKey = await promptPassword(`Enter your ${provider} API key:`, {
|
|
730
|
+
validate: (input) => input.trim().length > 0 || 'API key is required',
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
return apiKey;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/**
|
|
737
|
+
* Get the environment variable name for a provider's API key
|
|
738
|
+
*/
|
|
739
|
+
export function getApiKeyEnvVar(provider: string): string {
|
|
740
|
+
const envVars: Record<string, string> = {
|
|
741
|
+
openai: 'OPENAI_API_KEY',
|
|
742
|
+
'azure-openai': 'AZURE_OPENAI_API_KEY',
|
|
743
|
+
'vercel-ai': 'OPENAI_API_KEY', // Vercel AI typically uses underlying provider keys
|
|
744
|
+
anthropic: 'ANTHROPIC_API_KEY',
|
|
745
|
+
google: 'GOOGLE_AI_API_KEY',
|
|
746
|
+
mistral: 'MISTRAL_API_KEY',
|
|
747
|
+
};
|
|
748
|
+
return envVars[provider] || `${provider.toUpperCase().replace(/-/g, '_')}_API_KEY`;
|
|
749
|
+
}
|