jbai-cli 1.9.2 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -4
- package/bin/jbai-claude-opus.js +6 -0
- package/bin/jbai-claude-sonnet.js +6 -0
- package/bin/jbai-claude.js +16 -9
- package/bin/jbai-codex-5.2.js +6 -0
- package/bin/jbai-codex-5.3.js +6 -0
- package/bin/jbai-codex-rockhopper.js +6 -0
- package/bin/jbai-codex.js +12 -39
- package/bin/jbai-continue.js +27 -43
- package/bin/jbai-council.js +665 -0
- package/bin/jbai-gemini-3.1.js +6 -0
- package/bin/jbai-gemini-supernova.js +6 -0
- package/bin/jbai-gemini.js +17 -6
- package/bin/jbai-goose.js +11 -39
- package/bin/jbai-opencode-deepseek.js +6 -0
- package/bin/jbai-opencode-grok.js +6 -0
- package/bin/jbai-opencode-rockhopper.js +6 -0
- package/bin/jbai-opencode.js +122 -20
- package/bin/jbai-proxy.js +1110 -66
- package/bin/jbai.js +99 -42
- package/bin/test-cli-tictactoe.js +279 -0
- package/bin/test-clients.js +38 -6
- package/bin/test-model-lists.js +100 -0
- package/lib/completions.js +258 -0
- package/lib/config.js +46 -8
- package/lib/model-list.js +117 -0
- package/lib/postinstall.js +3 -0
- package/lib/proxy.js +46 -0
- package/lib/shortcut.js +47 -0
- package/package.json +13 -2
package/bin/jbai.js
CHANGED
|
@@ -3,8 +3,13 @@
|
|
|
3
3
|
const { spawn, execSync } = require('child_process');
|
|
4
4
|
const readline = require('readline');
|
|
5
5
|
const https = require('https');
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const os = require('os');
|
|
6
9
|
const config = require('../lib/config');
|
|
7
10
|
const { createHandoff } = require('../lib/handoff');
|
|
11
|
+
const { getGroupsForTool, showModelsForTool } = require('../lib/model-list');
|
|
12
|
+
const completions = require('../lib/completions');
|
|
8
13
|
|
|
9
14
|
const TOOLS = {
|
|
10
15
|
claude: {
|
|
@@ -53,14 +58,18 @@ jbai-cli v${VERSION} - JetBrains AI Platform CLI Tools
|
|
|
53
58
|
COMMANDS:
|
|
54
59
|
jbai token Show token status
|
|
55
60
|
jbai token set Set token interactively
|
|
56
|
-
jbai token refresh
|
|
61
|
+
jbai token refresh Auto-refresh token via API
|
|
62
|
+
jbai token refresh <token> Set new token (saves to ~/.jbai/token + ~/.zshrc)
|
|
57
63
|
jbai test Test API endpoints (incl. Codex /responses)
|
|
58
64
|
jbai handoff Continue task in Orca Lab
|
|
59
65
|
jbai env [staging|production] Switch environment
|
|
60
|
-
jbai models
|
|
66
|
+
jbai models [tool] List Grazie models (all|claude|codex|gemini|opencode|goose|continue)
|
|
61
67
|
jbai install Install all AI tools (claude, codex, gemini, opencode, goose, continue)
|
|
62
68
|
jbai install claude Install specific tool
|
|
63
69
|
jbai doctor Check which tools are installed
|
|
70
|
+
jbai completions Print zsh completions to stdout
|
|
71
|
+
jbai completions --install Add completions to ~/.zshrc
|
|
72
|
+
jbai completions --bash Print bash completions
|
|
64
73
|
jbai help Show this help
|
|
65
74
|
|
|
66
75
|
PROXY (for Codex Desktop, Cursor, etc.):
|
|
@@ -74,18 +83,35 @@ TOOL WRAPPERS:
|
|
|
74
83
|
jbai-codex Launch Codex CLI with JetBrains AI
|
|
75
84
|
jbai-gemini Launch Gemini CLI with JetBrains AI
|
|
76
85
|
jbai-opencode Launch OpenCode with JetBrains AI
|
|
86
|
+
jbai-council Launch Claude + Codex + OpenCode in tmux council mode
|
|
87
|
+
|
|
88
|
+
MODEL SHORTCUTS (super mode by default):
|
|
89
|
+
jbai-claude-opus Claude Code + Opus 4.6
|
|
90
|
+
jbai-claude-sonnet Claude Code + Sonnet 4.6
|
|
91
|
+
jbai-codex-5.3 Codex + GPT-5.3
|
|
92
|
+
jbai-codex-5.2 Codex + GPT-5.2
|
|
93
|
+
jbai-codex-rockhopper Codex + Rockhopper Alpha (OpenAI EAP)
|
|
94
|
+
jbai-gemini-supernova Gemini + Supernova (Google EAP)
|
|
95
|
+
jbai-gemini-3.1 Gemini + 3.1 Pro Preview
|
|
96
|
+
jbai-opencode-rockhopper OpenCode + Rockhopper Alpha
|
|
97
|
+
jbai-opencode-grok OpenCode + Grok 4 (xAI)
|
|
98
|
+
jbai-opencode-deepseek OpenCode + DeepSeek R1
|
|
77
99
|
|
|
78
100
|
SUPER MODE:
|
|
79
101
|
Add --super (or --yolo or -s) to skip confirmations:
|
|
80
102
|
jbai-claude --super # Skip permission prompts
|
|
81
103
|
jbai-codex --super # Full auto mode
|
|
82
104
|
jbai-gemini --super # Auto-confirm changes
|
|
105
|
+
(All model shortcuts above run in super mode by default)
|
|
83
106
|
|
|
84
107
|
EXAMPLES:
|
|
85
108
|
jbai token set # Set your token
|
|
86
109
|
jbai-claude # Start Claude Code
|
|
87
110
|
jbai-codex exec "explain code" # Run Codex task
|
|
88
|
-
jbai-
|
|
111
|
+
jbai-codex-rockhopper # Codex with Rockhopper (super mode)
|
|
112
|
+
jbai-gemini-supernova # Gemini with Supernova (super mode)
|
|
113
|
+
jbai-council # Launch all 3 agents in tmux
|
|
114
|
+
jbai-council --super # All agents in super mode
|
|
89
115
|
jbai handoff --task "fix lint" # Handoff task to Orca Lab
|
|
90
116
|
|
|
91
117
|
TOKEN:
|
|
@@ -157,7 +183,47 @@ async function setToken() {
|
|
|
157
183
|
});
|
|
158
184
|
}
|
|
159
185
|
|
|
160
|
-
|
|
186
|
+
function updateZshrcToken(newToken) {
|
|
187
|
+
const zshrc = path.join(os.homedir(), '.zshrc');
|
|
188
|
+
if (!fs.existsSync(zshrc)) return false;
|
|
189
|
+
|
|
190
|
+
const content = fs.readFileSync(zshrc, 'utf-8');
|
|
191
|
+
const pattern = /^(export JBAI_PROXY_KEY=").*(")/m;
|
|
192
|
+
if (!pattern.test(content)) return false;
|
|
193
|
+
|
|
194
|
+
const updated = content.replace(pattern, `$1${newToken}$2`);
|
|
195
|
+
fs.writeFileSync(zshrc, updated);
|
|
196
|
+
return true;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function saveTokenEverywhere(newToken) {
|
|
200
|
+
config.setToken(newToken);
|
|
201
|
+
console.log(' ✅ Saved to ~/.jbai/token');
|
|
202
|
+
|
|
203
|
+
if (updateZshrcToken(newToken)) {
|
|
204
|
+
console.log(' ✅ Updated JBAI_PROXY_KEY in ~/.zshrc');
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async function refreshTokenCommand(providedToken) {
|
|
209
|
+
// If a token was passed as argument, save it directly (skip API refresh)
|
|
210
|
+
if (providedToken && providedToken.includes('.')) {
|
|
211
|
+
const expiry = config.getTokenExpiry(providedToken);
|
|
212
|
+
if (!expiry) {
|
|
213
|
+
console.log('❌ Invalid token format (could not parse JWT)');
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
if (config.isTokenExpired(providedToken)) {
|
|
217
|
+
console.log('❌ Provided token is already expired');
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
console.log('🔄 Saving provided token...');
|
|
221
|
+
saveTokenEverywhere(providedToken);
|
|
222
|
+
console.log('✅ Token updated!');
|
|
223
|
+
showTokenStatus();
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
|
|
161
227
|
const token = config.getToken();
|
|
162
228
|
if (!token) {
|
|
163
229
|
console.log('❌ No token found. Run: jbai token set');
|
|
@@ -179,12 +245,13 @@ async function refreshTokenCommand() {
|
|
|
179
245
|
try {
|
|
180
246
|
console.log('🔄 Refreshing token via API...');
|
|
181
247
|
const newToken = await config.refreshTokenApi(token);
|
|
182
|
-
|
|
248
|
+
saveTokenEverywhere(newToken);
|
|
183
249
|
console.log('✅ Token refreshed successfully!');
|
|
184
250
|
showTokenStatus();
|
|
185
251
|
} catch (e) {
|
|
186
|
-
console.log(`❌
|
|
187
|
-
console.log(`
|
|
252
|
+
console.log(`❌ API refresh failed: ${e.message}`);
|
|
253
|
+
console.log(` Paste token manually: jbai token refresh <token>`);
|
|
254
|
+
console.log(` Get a new token: ${config.getEndpoints().tokenUrl}`);
|
|
188
255
|
}
|
|
189
256
|
}
|
|
190
257
|
|
|
@@ -303,41 +370,21 @@ function httpPost(url, body, headers) {
|
|
|
303
370
|
});
|
|
304
371
|
}
|
|
305
372
|
|
|
306
|
-
function showModels() {
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
const def = m === config.MODELS.claude.default ? ' (default)' : '';
|
|
312
|
-
console.log(` - ${m}${def}`);
|
|
313
|
-
});
|
|
314
|
-
|
|
315
|
-
console.log('\nGPT (OpenAI Chat) - jbai-opencode:');
|
|
316
|
-
config.MODELS.openai.available.forEach((m) => {
|
|
317
|
-
const def = m === config.MODELS.openai.default ? ' (default)' : '';
|
|
318
|
-
console.log(` - ${m}${def}`);
|
|
319
|
-
});
|
|
320
|
-
|
|
321
|
-
console.log('\nCodex (OpenAI Responses) - jbai-codex:');
|
|
322
|
-
config.MODELS.codex.available.forEach((m) => {
|
|
323
|
-
const def = m === config.MODELS.codex.default ? ' (default)' : '';
|
|
324
|
-
console.log(` - ${m}${def}`);
|
|
325
|
-
});
|
|
373
|
+
function showModels(filter) {
|
|
374
|
+
const tool = filter || 'all';
|
|
375
|
+
const heading = tool === 'all'
|
|
376
|
+
? 'Available Models via JetBrains AI Platform (Grazie):'
|
|
377
|
+
: `Available Models via JetBrains AI Platform (Grazie) for ${tool}:`;
|
|
326
378
|
|
|
327
|
-
|
|
328
|
-
config.MODELS.gemini.available.forEach((m) => {
|
|
329
|
-
const def = m === config.MODELS.gemini.default ? ' (default)' : '';
|
|
330
|
-
console.log(` - ${m}${def}`);
|
|
331
|
-
});
|
|
379
|
+
showModelsForTool(tool, heading);
|
|
332
380
|
|
|
333
|
-
|
|
334
|
-
const total =
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
console.log('via Grazie native API but not via OpenAI-compatible CLI tools.');
|
|
381
|
+
const groups = getGroupsForTool(tool);
|
|
382
|
+
const total = groups.reduce((sum, group) => sum + group.models.length, 0);
|
|
383
|
+
console.log(`Total: ${total} models`);
|
|
384
|
+
if (tool === 'all') {
|
|
385
|
+
console.log('\nNote: Other providers (DeepSeek, Mistral, Qwen, XAI, Meta) are available');
|
|
386
|
+
console.log('via Grazie native API but not via OpenAI-compatible CLI tools.');
|
|
387
|
+
}
|
|
341
388
|
}
|
|
342
389
|
|
|
343
390
|
function setEnvironment(env) {
|
|
@@ -621,7 +668,7 @@ switch (command) {
|
|
|
621
668
|
if (args[0] === 'set') {
|
|
622
669
|
setToken();
|
|
623
670
|
} else if (args[0] === 'refresh') {
|
|
624
|
-
refreshTokenCommand();
|
|
671
|
+
refreshTokenCommand(args[1]);
|
|
625
672
|
} else {
|
|
626
673
|
showTokenStatus();
|
|
627
674
|
}
|
|
@@ -633,7 +680,14 @@ switch (command) {
|
|
|
633
680
|
handoffToOrca(args);
|
|
634
681
|
break;
|
|
635
682
|
case 'models':
|
|
636
|
-
|
|
683
|
+
if (args[0]) {
|
|
684
|
+
const allowed = new Set(['all', 'claude', 'codex', 'gemini', 'opencode', 'goose', 'continue']);
|
|
685
|
+
if (!allowed.has(args[0])) {
|
|
686
|
+
console.log('Usage: jbai models [all|claude|codex|gemini|opencode|goose|continue]');
|
|
687
|
+
break;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
showModels(args[0]);
|
|
637
691
|
break;
|
|
638
692
|
case 'env':
|
|
639
693
|
setEnvironment(args[0]);
|
|
@@ -652,6 +706,9 @@ switch (command) {
|
|
|
652
706
|
proxyMod.main();
|
|
653
707
|
break;
|
|
654
708
|
}
|
|
709
|
+
case 'completions':
|
|
710
|
+
completions.run(args);
|
|
711
|
+
break;
|
|
655
712
|
case 'help':
|
|
656
713
|
case '--help':
|
|
657
714
|
case '-h':
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* E2E CLI Task Testing Script
|
|
5
|
+
* Runs each jbai CLI across all configured models and verifies
|
|
6
|
+
* tic-tac-toe sources are created in a temp directory.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const { spawnSync } = require('child_process');
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const os = require('os');
|
|
13
|
+
const config = require('../lib/config');
|
|
14
|
+
|
|
15
|
+
const PROMPT = [
|
|
16
|
+
'Create a minimal tic tac toe web app in the current directory.',
|
|
17
|
+
'Write index.html, styles.css, and app.js.',
|
|
18
|
+
'Use vanilla JS only. Keep the implementation small.',
|
|
19
|
+
'After writing files, reply with exactly: JBAI_DONE.'
|
|
20
|
+
].join(' ');
|
|
21
|
+
|
|
22
|
+
const BASE_DIR = path.join(os.tmpdir(), `jbai-cli-e2e-${Date.now()}`);
|
|
23
|
+
const DEFAULT_TIMEOUT_MS = 3 * 60 * 1000;
|
|
24
|
+
const FAST_TIMEOUT_MS = 60 * 1000;
|
|
25
|
+
const BIN_DIR = __dirname;
|
|
26
|
+
|
|
27
|
+
const REQUIRED_FILES = ['index.html', 'styles.css', 'app.js'];
|
|
28
|
+
|
|
29
|
+
function ensureDir(dir) {
|
|
30
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function hasRequiredFiles(dir) {
|
|
34
|
+
return REQUIRED_FILES.every((file) => {
|
|
35
|
+
const full = path.join(dir, file);
|
|
36
|
+
return fs.existsSync(full) && fs.statSync(full).size > 0;
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function runCommand({ cmd, args, cwd, timeoutMs }) {
|
|
41
|
+
const result = spawnSync(cmd, args, {
|
|
42
|
+
cwd,
|
|
43
|
+
encoding: 'utf-8',
|
|
44
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
45
|
+
timeout: timeoutMs || DEFAULT_TIMEOUT_MS,
|
|
46
|
+
});
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function formatResult(ok, reason) {
|
|
51
|
+
return ok ? { ok: true } : { ok: false, reason: reason || 'unknown' };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function detectToolLimitation(output) {
|
|
55
|
+
const text = (output || '').toLowerCase();
|
|
56
|
+
if (text.includes('tool') && text.includes('not found in registry')) return true;
|
|
57
|
+
if (text.includes('no direct file writing tools')) return true;
|
|
58
|
+
if (text.includes('cannot proceed') && text.includes('files')) return true;
|
|
59
|
+
if (text.includes('apps/create_app') && text.includes('do not have direct')) return true;
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function runTest(test) {
|
|
64
|
+
const dir = path.join(BASE_DIR, test.tool, test.modelId.replace(/[^a-zA-Z0-9._-]/g, '_'));
|
|
65
|
+
ensureDir(dir);
|
|
66
|
+
|
|
67
|
+
const result = runCommand({ cmd: test.cmd, args: test.args, cwd: dir, timeoutMs: test.timeoutMs });
|
|
68
|
+
const combinedOutput = `${result.stdout || ''}\n${result.stderr || ''}`;
|
|
69
|
+
const ok = result.status === 0 && hasRequiredFiles(dir);
|
|
70
|
+
const logPath = path.join(dir, 'run.log');
|
|
71
|
+
try {
|
|
72
|
+
fs.writeFileSync(logPath, combinedOutput);
|
|
73
|
+
} catch {
|
|
74
|
+
// ignore log write errors
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
ok,
|
|
79
|
+
status: result.status,
|
|
80
|
+
signal: result.signal,
|
|
81
|
+
timedOut: result.error && result.error.code === 'ETIMEDOUT',
|
|
82
|
+
output: combinedOutput,
|
|
83
|
+
dir,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function buildTests() {
|
|
88
|
+
const tests = [];
|
|
89
|
+
|
|
90
|
+
// Claude Code
|
|
91
|
+
for (const model of config.MODELS.claude.available) {
|
|
92
|
+
tests.push({
|
|
93
|
+
tool: 'jbai-claude',
|
|
94
|
+
modelId: model,
|
|
95
|
+
cmd: process.execPath,
|
|
96
|
+
args: [path.join(BIN_DIR, 'jbai-claude.js'), '--super', '--model', model, '-p', PROMPT],
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Codex CLI
|
|
101
|
+
for (const model of config.MODELS.codex.available) {
|
|
102
|
+
tests.push({
|
|
103
|
+
tool: 'jbai-codex',
|
|
104
|
+
modelId: model,
|
|
105
|
+
cmd: process.execPath,
|
|
106
|
+
args: [path.join(BIN_DIR, 'jbai-codex.js'), '--super', '--model', model, 'exec', '--skip-git-repo-check', PROMPT],
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// OpenCode (OpenAI models)
|
|
111
|
+
const environment = config.getEnvironment();
|
|
112
|
+
const openaiProvider = environment === 'staging' ? 'jbai-staging' : 'jbai';
|
|
113
|
+
const anthropicProvider = environment === 'staging' ? 'jbai-anthropic-staging' : 'jbai-anthropic';
|
|
114
|
+
|
|
115
|
+
for (const model of config.MODELS.openai.available) {
|
|
116
|
+
tests.push({
|
|
117
|
+
tool: 'jbai-opencode',
|
|
118
|
+
modelId: model,
|
|
119
|
+
cmd: process.execPath,
|
|
120
|
+
args: [path.join(BIN_DIR, 'jbai-opencode.js'), '--super', 'run', '--model', `${openaiProvider}/${model}`, PROMPT],
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
for (const model of config.MODELS.claude.available) {
|
|
124
|
+
tests.push({
|
|
125
|
+
tool: 'jbai-opencode',
|
|
126
|
+
modelId: model,
|
|
127
|
+
cmd: process.execPath,
|
|
128
|
+
args: [path.join(BIN_DIR, 'jbai-opencode.js'), '--super', 'run', '--model', `${anthropicProvider}/${model}`, PROMPT],
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Gemini CLI
|
|
133
|
+
for (const model of config.MODELS.gemini.available) {
|
|
134
|
+
tests.push({
|
|
135
|
+
tool: 'jbai-gemini',
|
|
136
|
+
modelId: model,
|
|
137
|
+
cmd: process.execPath,
|
|
138
|
+
args: [path.join(BIN_DIR, 'jbai-gemini.js'), '--super', '--model', model, '-p', PROMPT],
|
|
139
|
+
timeoutMs: FAST_TIMEOUT_MS,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Goose (OpenAI + Claude)
|
|
144
|
+
for (const model of config.MODELS.openai.available) {
|
|
145
|
+
tests.push({
|
|
146
|
+
tool: 'jbai-goose',
|
|
147
|
+
modelId: model,
|
|
148
|
+
cmd: process.execPath,
|
|
149
|
+
args: [path.join(BIN_DIR, 'jbai-goose.js'), '--super', 'run', '--provider', 'openai', '--model', model, '--with-builtin', 'code_execution', '-t', PROMPT],
|
|
150
|
+
timeoutMs: FAST_TIMEOUT_MS,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
for (const model of config.MODELS.claude.available) {
|
|
154
|
+
tests.push({
|
|
155
|
+
tool: 'jbai-goose',
|
|
156
|
+
modelId: model,
|
|
157
|
+
cmd: process.execPath,
|
|
158
|
+
args: [path.join(BIN_DIR, 'jbai-goose.js'), '--super', 'run', '--provider', 'anthropic', '--model', model, '--with-builtin', 'code_execution', '-t', PROMPT],
|
|
159
|
+
timeoutMs: FAST_TIMEOUT_MS,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Continue CLI
|
|
164
|
+
for (const model of config.MODELS.claude.available) {
|
|
165
|
+
tests.push({
|
|
166
|
+
tool: 'jbai-continue',
|
|
167
|
+
modelId: model,
|
|
168
|
+
cmd: process.execPath,
|
|
169
|
+
args: [path.join(BIN_DIR, 'jbai-continue.js'), '--super', '--model', model, '-p', PROMPT],
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
for (const model of config.MODELS.openai.available) {
|
|
173
|
+
tests.push({
|
|
174
|
+
tool: 'jbai-continue',
|
|
175
|
+
modelId: model,
|
|
176
|
+
cmd: process.execPath,
|
|
177
|
+
args: [path.join(BIN_DIR, 'jbai-continue.js'), '--super', '--model', model, '-p', PROMPT],
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
return tests;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
function main() {
|
|
185
|
+
ensureDir(BASE_DIR);
|
|
186
|
+
|
|
187
|
+
const tests = buildTests();
|
|
188
|
+
const results = [];
|
|
189
|
+
const skippedTools = new Map();
|
|
190
|
+
|
|
191
|
+
console.log(`Running ${tests.length} CLI task tests...`);
|
|
192
|
+
console.log(`Artifacts: ${BASE_DIR}`);
|
|
193
|
+
console.log('');
|
|
194
|
+
|
|
195
|
+
for (const test of tests) {
|
|
196
|
+
if (skippedTools.has(test.tool)) {
|
|
197
|
+
results.push({
|
|
198
|
+
tool: test.tool,
|
|
199
|
+
model: test.modelId,
|
|
200
|
+
ok: false,
|
|
201
|
+
skipped: true,
|
|
202
|
+
reason: skippedTools.get(test.tool),
|
|
203
|
+
});
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const label = `${test.tool} :: ${test.modelId}`;
|
|
208
|
+
process.stdout.write(`${label} ... `);
|
|
209
|
+
|
|
210
|
+
const result = runTest(test);
|
|
211
|
+
const output = result.output || '';
|
|
212
|
+
|
|
213
|
+
if (result.ok) {
|
|
214
|
+
console.log('OK');
|
|
215
|
+
results.push({ tool: test.tool, model: test.modelId, ok: true, dir: result.dir });
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
let reason = 'unknown';
|
|
220
|
+
if (result.timedOut) {
|
|
221
|
+
reason = 'timeout';
|
|
222
|
+
} else if (result.status !== 0) {
|
|
223
|
+
reason = `exit ${result.status}`;
|
|
224
|
+
} else if (!hasRequiredFiles(result.dir)) {
|
|
225
|
+
reason = 'files missing';
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
console.log(`FAIL (${reason})`);
|
|
229
|
+
results.push({ tool: test.tool, model: test.modelId, ok: false, reason, output, dir: result.dir });
|
|
230
|
+
|
|
231
|
+
if (detectToolLimitation(output)) {
|
|
232
|
+
skippedTools.set(test.tool, 'missing file-write tooling');
|
|
233
|
+
} else if (test.tool === 'jbai-goose' && reason === 'files missing') {
|
|
234
|
+
skippedTools.set(test.tool, 'missing file-write tooling');
|
|
235
|
+
} else if (test.tool === 'jbai-goose' && reason === 'timeout') {
|
|
236
|
+
skippedTools.set(test.tool, 'likely missing file-write tooling (timeout)');
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const summary = results.reduce((acc, r) => {
|
|
241
|
+
acc.total += 1;
|
|
242
|
+
if (r.ok) acc.passed += 1;
|
|
243
|
+
else if (r.skipped) acc.skipped += 1;
|
|
244
|
+
else acc.failed += 1;
|
|
245
|
+
return acc;
|
|
246
|
+
}, { total: 0, passed: 0, failed: 0, skipped: 0 });
|
|
247
|
+
|
|
248
|
+
console.log('\nSummary');
|
|
249
|
+
console.log(`Total: ${summary.total}`);
|
|
250
|
+
console.log(`Passed: ${summary.passed}`);
|
|
251
|
+
console.log(`Failed: ${summary.failed}`);
|
|
252
|
+
console.log(`Skipped: ${summary.skipped}`);
|
|
253
|
+
|
|
254
|
+
const failures = results.filter(r => !r.ok && !r.skipped);
|
|
255
|
+
if (failures.length > 0) {
|
|
256
|
+
console.log('\nFailures:');
|
|
257
|
+
for (const fail of failures) {
|
|
258
|
+
console.log(`- ${fail.tool} :: ${fail.model} (${fail.reason})`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const skipped = results.filter(r => r.skipped);
|
|
263
|
+
if (skipped.length > 0) {
|
|
264
|
+
const byTool = skipped.reduce((acc, r) => {
|
|
265
|
+
if (!acc[r.tool]) acc[r.tool] = r.reason || 'skipped';
|
|
266
|
+
return acc;
|
|
267
|
+
}, {});
|
|
268
|
+
console.log('\nSkipped tools:');
|
|
269
|
+
for (const [tool, reason] of Object.entries(byTool)) {
|
|
270
|
+
console.log(`- ${tool}: ${reason}`);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
if (summary.failed > 0) {
|
|
275
|
+
process.exit(1);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
main();
|
package/bin/test-clients.js
CHANGED
|
@@ -98,11 +98,9 @@ async function testClaude(model) {
|
|
|
98
98
|
}
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
// Test
|
|
102
|
-
async function
|
|
101
|
+
// Test OpenAI Chat (used by OpenCode)
|
|
102
|
+
async function testOpenAIChat(model) {
|
|
103
103
|
try {
|
|
104
|
-
// Codex uses the OpenAI "responses" API format
|
|
105
|
-
// For testing, we use chat/completions which is what the proxy supports
|
|
106
104
|
const needsCompletionTokens = model.startsWith('gpt-5') || model.startsWith('o1') ||
|
|
107
105
|
model.startsWith('o3') || model.startsWith('o4');
|
|
108
106
|
|
|
@@ -136,10 +134,44 @@ async function testCodex(model) {
|
|
|
136
134
|
}
|
|
137
135
|
}
|
|
138
136
|
|
|
137
|
+
// Test Codex (OpenAI Responses API - used by codex CLI)
|
|
138
|
+
async function testCodex(model) {
|
|
139
|
+
try {
|
|
140
|
+
const result = await httpPost(
|
|
141
|
+
`${endpoints.openai}/responses`,
|
|
142
|
+
{
|
|
143
|
+
model: model,
|
|
144
|
+
input: 'Reply with exactly: JBAI_OK',
|
|
145
|
+
max_output_tokens: 64
|
|
146
|
+
},
|
|
147
|
+
{ 'Grazie-Authenticate-JWT': token }
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
if (result.status === 200) {
|
|
151
|
+
const outputs = Array.isArray(result.data?.output) ? result.data.output : [];
|
|
152
|
+
const text = outputs
|
|
153
|
+
.flatMap(item => Array.isArray(item?.content) ? item.content : [])
|
|
154
|
+
.map(part => part?.text)
|
|
155
|
+
.filter(Boolean)
|
|
156
|
+
.join('\n');
|
|
157
|
+
|
|
158
|
+
if (text.includes('JBAI_OK')) {
|
|
159
|
+
return { success: true, message: 'OK', response: text };
|
|
160
|
+
}
|
|
161
|
+
return { success: true, message: 'OK (response varied)', response: text };
|
|
162
|
+
} else if (result.status === 429) {
|
|
163
|
+
return { success: true, message: 'Rate limited (model works)', error: 'Rate limit' };
|
|
164
|
+
} else {
|
|
165
|
+
return { success: false, message: `Status ${result.status}`, error: result.data.error?.message || JSON.stringify(result.data).substring(0, 100) };
|
|
166
|
+
}
|
|
167
|
+
} catch (e) {
|
|
168
|
+
return { success: false, message: 'Error', error: e.message };
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
139
172
|
// Test OpenCode (OpenAI Chat Completions API)
|
|
140
173
|
async function testOpenCode(model) {
|
|
141
|
-
|
|
142
|
-
return testCodex(model);
|
|
174
|
+
return testOpenAIChat(model);
|
|
143
175
|
}
|
|
144
176
|
|
|
145
177
|
// Test Gemini (Vertex AI API)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* E2E Model List Testing Script
|
|
5
|
+
* Verifies each jbai client exposes only Grazie models in its model list output.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const { execFileSync } = require('child_process');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
const { getGroupsForTool } = require('../lib/model-list');
|
|
11
|
+
|
|
12
|
+
const tools = [
|
|
13
|
+
{ key: 'claude', script: 'jbai-claude.js' },
|
|
14
|
+
{ key: 'codex', script: 'jbai-codex.js' },
|
|
15
|
+
{ key: 'gemini', script: 'jbai-gemini.js' },
|
|
16
|
+
{ key: 'opencode', script: 'jbai-opencode.js' },
|
|
17
|
+
{ key: 'goose', script: 'jbai-goose.js' },
|
|
18
|
+
{ key: 'continue', script: 'jbai-continue.js' },
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
const colors = {
|
|
22
|
+
reset: '\x1b[0m',
|
|
23
|
+
green: '\x1b[32m',
|
|
24
|
+
red: '\x1b[31m',
|
|
25
|
+
cyan: '\x1b[36m',
|
|
26
|
+
dim: '\x1b[2m',
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
function parseModels(output) {
|
|
30
|
+
return output
|
|
31
|
+
.split('\n')
|
|
32
|
+
.map(line => line.trim())
|
|
33
|
+
.filter(line => line.startsWith('- '))
|
|
34
|
+
.map(line => line.slice(2).replace(/\s+\(default\)$/, ''))
|
|
35
|
+
.filter(Boolean);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function expectedModels(toolKey) {
|
|
39
|
+
const groups = getGroupsForTool(toolKey);
|
|
40
|
+
return groups.flatMap(group => group.models);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function runTool(script) {
|
|
44
|
+
const scriptPath = path.join(__dirname, script);
|
|
45
|
+
return execFileSync(process.execPath, [scriptPath, '--models'], { encoding: 'utf-8' });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function diffModels(expected, actual) {
|
|
49
|
+
const expectedSet = new Set(expected);
|
|
50
|
+
const actualSet = new Set(actual);
|
|
51
|
+
const missing = expected.filter(m => !actualSet.has(m));
|
|
52
|
+
const extra = actual.filter(m => !expectedSet.has(m));
|
|
53
|
+
return { missing, extra };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function run() {
|
|
57
|
+
console.log(`\n${'='.repeat(70)}`);
|
|
58
|
+
console.log(`${colors.cyan}JBAI-CLI MODEL LIST E2E${colors.reset}`);
|
|
59
|
+
console.log(`${'='.repeat(70)}\n`);
|
|
60
|
+
|
|
61
|
+
let failures = 0;
|
|
62
|
+
|
|
63
|
+
for (const tool of tools) {
|
|
64
|
+
const expected = expectedModels(tool.key);
|
|
65
|
+
process.stdout.write(`Testing jbai-${tool.key} models... `);
|
|
66
|
+
try {
|
|
67
|
+
const output = runTool(tool.script);
|
|
68
|
+
const actual = parseModels(output);
|
|
69
|
+
const { missing, extra } = diffModels(expected, actual);
|
|
70
|
+
if (missing.length === 0 && extra.length === 0) {
|
|
71
|
+
console.log(`${colors.green}✓ OK${colors.reset}`);
|
|
72
|
+
} else {
|
|
73
|
+
failures += 1;
|
|
74
|
+
console.log(`${colors.red}✗ Mismatch${colors.reset}`);
|
|
75
|
+
if (missing.length > 0) {
|
|
76
|
+
console.log(` ${colors.dim}Missing:${colors.reset} ${missing.join(', ')}`);
|
|
77
|
+
}
|
|
78
|
+
if (extra.length > 0) {
|
|
79
|
+
console.log(` ${colors.dim}Extra:${colors.reset} ${extra.join(', ')}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
} catch (err) {
|
|
83
|
+
failures += 1;
|
|
84
|
+
console.log(`${colors.red}✗ Error${colors.reset}`);
|
|
85
|
+
const msg = err && err.message ? err.message : String(err);
|
|
86
|
+
console.log(` ${colors.dim}${msg}${colors.reset}`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
console.log(`\n${'='.repeat(70)}\n`);
|
|
91
|
+
|
|
92
|
+
if (failures > 0) {
|
|
93
|
+
process.exit(1);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
run().catch((err) => {
|
|
98
|
+
console.error(err);
|
|
99
|
+
process.exit(1);
|
|
100
|
+
});
|