@exreve/exk 1.0.44 → 1.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agentSession.js +121 -10
- package/dist/app-child.js +163 -1674
- package/dist/appHandlers.js +142 -0
- package/dist/appManager.js +5 -5
- package/dist/appRunner.js +2 -2
- package/dist/benchmark-startup.js +347 -0
- package/dist/cloudflaredHandlers.js +279 -0
- package/dist/containerHandlers.js +193 -0
- package/dist/fsHandlers.js +86 -0
- package/dist/githubHandlers.js +521 -0
- package/dist/index.js +164 -1743
- package/dist/projectAnalyzer.js +10 -3
- package/dist/projectManager.js +1 -1
- package/dist/runnerGenerator.js +2 -3
- package/dist/sessionHandlers.js +271 -0
- package/dist/ttc-cli.tar.gz +0 -0
- package/dist/updateHandlers.js +82 -0
- package/dist/updater.js +2 -5
- package/package.json +3 -3
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* App Control Handlers Module
|
|
3
|
+
*
|
|
4
|
+
* Handles app:start, app:stop, app:restart, app:status, app:logs.
|
|
5
|
+
*/
|
|
6
|
+
import { getProjectConfig } from './projectAnalyzer.js';
|
|
7
|
+
import { startApp, stopApp, restartApp, getAppStatuses, getAppLogs } from './appManager.js';
|
|
8
|
+
export function registerAppHandlers(socket, foreground) {
|
|
9
|
+
socket.on('app:start', async (data, callback) => {
|
|
10
|
+
try {
|
|
11
|
+
const { projectId, projectPath, appName } = data;
|
|
12
|
+
const config = await getProjectConfig(projectPath);
|
|
13
|
+
if (!config) {
|
|
14
|
+
callback?.({ success: false, error: 'Project config not found' });
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
const app = config.apps.find(a => a.name === appName);
|
|
18
|
+
if (!app) {
|
|
19
|
+
callback?.({ success: false, error: `App "${appName}" not found in project config` });
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const result = await startApp(projectPath, projectId, app);
|
|
23
|
+
if (result.success) {
|
|
24
|
+
if (foreground)
|
|
25
|
+
console.log(`✓ Started app: ${appName} (PID: ${result.pid})`);
|
|
26
|
+
socket.emit('app:started', {
|
|
27
|
+
projectId,
|
|
28
|
+
appName,
|
|
29
|
+
processId: result.processId,
|
|
30
|
+
pid: result.pid,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
callback?.(result);
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
if (foreground)
|
|
37
|
+
console.error(`✗ Error starting app: ${error.message}`);
|
|
38
|
+
callback?.({ success: false, error: error.message });
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
socket.on('app:stop', async (data, callback) => {
|
|
42
|
+
try {
|
|
43
|
+
const { projectId, appName } = data;
|
|
44
|
+
const config = await getProjectConfig(data.projectPath);
|
|
45
|
+
const app = config?.apps.find(a => a.name === appName);
|
|
46
|
+
const result = await stopApp(projectId, appName, app);
|
|
47
|
+
if (result.success) {
|
|
48
|
+
if (foreground)
|
|
49
|
+
console.log(`✓ Stopped app: ${appName}`);
|
|
50
|
+
socket.emit('app:stopped', {
|
|
51
|
+
projectId,
|
|
52
|
+
appName,
|
|
53
|
+
appControlId: data.appControlId,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
callback?.(result);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (foreground)
|
|
60
|
+
console.error(`✗ Error stopping app: ${error.message}`);
|
|
61
|
+
callback?.({ success: false, error: error.message });
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
socket.on('app:restart', async (data, callback) => {
|
|
65
|
+
try {
|
|
66
|
+
const { projectId, projectPath, appName } = data;
|
|
67
|
+
const config = await getProjectConfig(projectPath);
|
|
68
|
+
if (!config) {
|
|
69
|
+
callback?.({ success: false, error: 'Project config not found' });
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
const app = config.apps.find(a => a.name === appName);
|
|
73
|
+
if (!app) {
|
|
74
|
+
callback?.({ success: false, error: `App "${appName}" not found in project config` });
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const result = await restartApp(projectPath, projectId, app);
|
|
78
|
+
if (result.success) {
|
|
79
|
+
if (foreground)
|
|
80
|
+
console.log(`✓ Restarted app: ${appName} (PID: ${result.pid})`);
|
|
81
|
+
socket.emit('app:restarted', {
|
|
82
|
+
projectId,
|
|
83
|
+
appName,
|
|
84
|
+
processId: result.processId,
|
|
85
|
+
pid: result.pid,
|
|
86
|
+
appControlId: data.appControlId,
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
callback?.(result);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
if (foreground)
|
|
93
|
+
console.error(`✗ Error restarting app: ${error.message}`);
|
|
94
|
+
callback?.({ success: false, error: error.message });
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
socket.on('app:status', async (data, callback) => {
|
|
98
|
+
try {
|
|
99
|
+
const { projectId, projectPath, appName } = data;
|
|
100
|
+
const config = await getProjectConfig(projectPath);
|
|
101
|
+
if (!config) {
|
|
102
|
+
callback?.({ success: false, error: 'Project config not found' });
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
const appsToCheck = appName
|
|
106
|
+
? config.apps.filter(a => a.name === appName)
|
|
107
|
+
: config.apps;
|
|
108
|
+
const statuses = getAppStatuses(projectId, appsToCheck);
|
|
109
|
+
if (data.appControlId) {
|
|
110
|
+
socket.emit('app:control:response', {
|
|
111
|
+
appControlId: data.appControlId,
|
|
112
|
+
success: true,
|
|
113
|
+
apps: statuses,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
callback?.({ success: true, apps: statuses });
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
if (foreground)
|
|
120
|
+
console.error(`✗ Error getting app status: ${error.message}`);
|
|
121
|
+
callback?.({ success: false, error: error.message });
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
socket.on('app:logs', async (data, callback) => {
|
|
125
|
+
try {
|
|
126
|
+
const { projectId, appName } = data;
|
|
127
|
+
const result = await getAppLogs(projectId, appName, 100);
|
|
128
|
+
if (data.appControlId) {
|
|
129
|
+
socket.emit('app:control:response', {
|
|
130
|
+
appControlId: data.appControlId,
|
|
131
|
+
...result,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
callback?.(result);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
if (foreground)
|
|
138
|
+
console.error(`✗ Error getting app logs: ${error.message}`);
|
|
139
|
+
callback?.({ success: false, error: error.message });
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
}
|
package/dist/appManager.js
CHANGED
|
@@ -46,16 +46,16 @@ export async function startApp(projectPath, projectId, app) {
|
|
|
46
46
|
const logFile = getLogFilePath(projectId, app.name);
|
|
47
47
|
// Create runner based on app type
|
|
48
48
|
const runner = createRunner(app, projectPath, projectId, {
|
|
49
|
-
onOutput: async (
|
|
49
|
+
onOutput: async (_output) => {
|
|
50
50
|
// Logs are handled by the runner itself
|
|
51
51
|
},
|
|
52
|
-
onError: (
|
|
52
|
+
onError: (_error) => {
|
|
53
53
|
// Errors are logged by runner
|
|
54
54
|
},
|
|
55
|
-
onExit: (
|
|
55
|
+
onExit: (_code) => {
|
|
56
56
|
removeAppFromRunning(projectId, app.name);
|
|
57
57
|
},
|
|
58
|
-
onStats: (
|
|
58
|
+
onStats: (_stats) => {
|
|
59
59
|
// Stats updates can be used for monitoring
|
|
60
60
|
},
|
|
61
61
|
});
|
|
@@ -90,7 +90,7 @@ export async function startApp(projectPath, projectId, app) {
|
|
|
90
90
|
/**
|
|
91
91
|
* Stop an app using wrapper runner
|
|
92
92
|
*/
|
|
93
|
-
export async function stopApp(projectId, appName,
|
|
93
|
+
export async function stopApp(projectId, appName, _app) {
|
|
94
94
|
try {
|
|
95
95
|
const running = getRunningApp(projectId, appName);
|
|
96
96
|
if (!running) {
|
package/dist/appRunner.js
CHANGED
|
@@ -140,7 +140,7 @@ export class StaticFrontendRunner extends BaseRunner {
|
|
|
140
140
|
// Create a mock process for compatibility
|
|
141
141
|
this.process = {
|
|
142
142
|
pid: process.pid,
|
|
143
|
-
kill: (
|
|
143
|
+
kill: (_signal) => {
|
|
144
144
|
this.stop();
|
|
145
145
|
},
|
|
146
146
|
};
|
|
@@ -291,7 +291,7 @@ export class BackendRunner extends BaseRunner {
|
|
|
291
291
|
? path.join(this.projectPath, this.app.directory)
|
|
292
292
|
: this.projectPath;
|
|
293
293
|
const { shell: stopShell, args: stopArgs } = shellSpawnOpts(this.app.stopCommand);
|
|
294
|
-
|
|
294
|
+
spawn(stopShell, stopArgs, {
|
|
295
295
|
cwd: workingDir,
|
|
296
296
|
stdio: 'ignore',
|
|
297
297
|
});
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Benchmark: cold query() vs warm startup() + query()
|
|
3
|
+
*
|
|
4
|
+
* Measures time-to-first-token for:
|
|
5
|
+
* 1. Cold query() (current behavior - spawns subprocess each time)
|
|
6
|
+
* 2. Warm startup() -> query() (pre-warmed subprocess)
|
|
7
|
+
*
|
|
8
|
+
* Usage: cd cli && npx tsx benchmark-startup.ts
|
|
9
|
+
*/
|
|
10
|
+
import { query, startup } from '@anthropic-ai/claude-agent-sdk';
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import os from 'os';
|
|
14
|
+
import { execSync } from 'child_process';
|
|
15
|
+
import { createRequire } from 'module';
|
|
16
|
+
// Resolve Claude Code executable (mirrors agentSession.ts logic)
|
|
17
|
+
function resolveClaudeExe() {
|
|
18
|
+
// Try glibc native binary first (most Linux distros)
|
|
19
|
+
for (const pkg of [
|
|
20
|
+
'@anthropic-ai/claude-agent-sdk-linux-x64',
|
|
21
|
+
'@anthropic-ai/claude-agent-sdk-linux-x64-musl',
|
|
22
|
+
]) {
|
|
23
|
+
try {
|
|
24
|
+
const req = createRequire(import.meta.url);
|
|
25
|
+
const nativePkgPath = req.resolve(`${pkg}/package.json`);
|
|
26
|
+
const nativePath = path.join(path.dirname(nativePkgPath), 'claude');
|
|
27
|
+
if (fs.existsSync(nativePath)) {
|
|
28
|
+
// Test if it's actually executable
|
|
29
|
+
try {
|
|
30
|
+
execSync(`${nativePath} --version`, { stdio: 'pipe', timeout: 5000 });
|
|
31
|
+
return nativePath;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// Binary can't execute (e.g. musl on glibc), skip
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
catch { }
|
|
39
|
+
}
|
|
40
|
+
// Fallback: cli.js (Node.js-based)
|
|
41
|
+
try {
|
|
42
|
+
const req = createRequire(import.meta.url);
|
|
43
|
+
const pkgPath = req.resolve('@anthropic-ai/claude-agent-sdk/package.json');
|
|
44
|
+
const cliPath = path.join(path.dirname(pkgPath), 'cli.js');
|
|
45
|
+
if (fs.existsSync(cliPath))
|
|
46
|
+
return cliPath;
|
|
47
|
+
}
|
|
48
|
+
catch { }
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
const CLAUDE_EXE = resolveClaudeExe();
|
|
52
|
+
// ── Config ──────────────────────────────────────────────────────────
|
|
53
|
+
const AI_CONFIG_PATH = path.join(os.homedir(), '.talk-to-code', 'ai-config.json');
|
|
54
|
+
function loadConfig() {
|
|
55
|
+
try {
|
|
56
|
+
return JSON.parse(fs.readFileSync(AI_CONFIG_PATH, 'utf-8'));
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return {};
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// ── Helpers ─────────────────────────────────────────────────────────
|
|
63
|
+
const GREEN = '\x1b[32m';
|
|
64
|
+
const RED = '\x1b[31m';
|
|
65
|
+
const CYAN = '\x1b[36m';
|
|
66
|
+
const YELLOW = '\x1b[33m';
|
|
67
|
+
const BOLD = '\x1b[1m';
|
|
68
|
+
const RESET = '\x1b[0m';
|
|
69
|
+
function hr() {
|
|
70
|
+
console.log(`${'─'.repeat(60)}`);
|
|
71
|
+
}
|
|
72
|
+
async function measureColdQuery(apiKey, baseUrl, model, env, settingsEnv, label) {
|
|
73
|
+
const queryStart = Date.now();
|
|
74
|
+
let firstTokenMs = 0;
|
|
75
|
+
let initMs = 0;
|
|
76
|
+
let totalMs = 0;
|
|
77
|
+
try {
|
|
78
|
+
const q = query({
|
|
79
|
+
prompt: 'Say exactly: "Hello, I am ready." Nothing else.',
|
|
80
|
+
options: {
|
|
81
|
+
apiKey,
|
|
82
|
+
model,
|
|
83
|
+
cwd: '/tmp',
|
|
84
|
+
permissionMode: 'bypassPermissions',
|
|
85
|
+
allowDangerouslySkipPermissions: true,
|
|
86
|
+
maxTurns: 1,
|
|
87
|
+
env,
|
|
88
|
+
settings: { env: settingsEnv },
|
|
89
|
+
...(CLAUDE_EXE ? { pathToClaudeCodeExecutable: CLAUDE_EXE } : {}),
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
for await (const event of q) {
|
|
93
|
+
const now = Date.now();
|
|
94
|
+
if (event.type === 'system' && event.subtype === 'init') {
|
|
95
|
+
initMs = now - queryStart;
|
|
96
|
+
}
|
|
97
|
+
if (event.type === 'assistant') {
|
|
98
|
+
if (firstTokenMs === 0) {
|
|
99
|
+
firstTokenMs = now - queryStart;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (event.type === 'result') {
|
|
103
|
+
totalMs = now - queryStart;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
return {
|
|
109
|
+
firstTokenMs: firstTokenMs || -1,
|
|
110
|
+
totalMs: Date.now() - queryStart,
|
|
111
|
+
startupMs: initMs || -1,
|
|
112
|
+
error: err.message,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
console.log(` ${label}: init=${initMs}ms, first_token=${firstTokenMs}ms, total=${totalMs}ms`);
|
|
116
|
+
return { firstTokenMs, totalMs, startupMs: initMs };
|
|
117
|
+
}
|
|
118
|
+
async function measureWarmQuery(apiKey, baseUrl, model, env, settingsEnv, label) {
|
|
119
|
+
// Phase 1: Pre-warm
|
|
120
|
+
const prewarmStart = Date.now();
|
|
121
|
+
let warmQuery;
|
|
122
|
+
try {
|
|
123
|
+
warmQuery = await startup({
|
|
124
|
+
options: {
|
|
125
|
+
apiKey,
|
|
126
|
+
model,
|
|
127
|
+
cwd: '/tmp',
|
|
128
|
+
permissionMode: 'bypassPermissions',
|
|
129
|
+
allowDangerouslySkipPermissions: true,
|
|
130
|
+
maxTurns: 1,
|
|
131
|
+
env,
|
|
132
|
+
settings: { env: settingsEnv },
|
|
133
|
+
...(CLAUDE_EXE ? { pathToClaudeCodeExecutable: CLAUDE_EXE } : {}),
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
catch (err) {
|
|
138
|
+
return {
|
|
139
|
+
prewarmMs: Date.now() - prewarmStart,
|
|
140
|
+
firstTokenMs: -1,
|
|
141
|
+
totalMs: Date.now() - prewarmStart,
|
|
142
|
+
error: `startup() failed: ${err.message}`,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
const prewarmMs = Date.now() - prewarmStart;
|
|
146
|
+
console.log(` ${label} prewarm: ${prewarmMs}ms`);
|
|
147
|
+
// Phase 2: Query on warm subprocess
|
|
148
|
+
const queryStart = Date.now();
|
|
149
|
+
let firstTokenMs = 0;
|
|
150
|
+
let totalMs = 0;
|
|
151
|
+
try {
|
|
152
|
+
const q = warmQuery.query('Say exactly: "Hello, I am ready." Nothing else.');
|
|
153
|
+
for await (const event of q) {
|
|
154
|
+
const now = Date.now();
|
|
155
|
+
if (event.type === 'assistant') {
|
|
156
|
+
if (firstTokenMs === 0) {
|
|
157
|
+
firstTokenMs = now - queryStart;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (event.type === 'result') {
|
|
161
|
+
totalMs = now - queryStart;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
catch (err) {
|
|
166
|
+
return {
|
|
167
|
+
prewarmMs,
|
|
168
|
+
firstTokenMs: firstTokenMs || -1,
|
|
169
|
+
totalMs: Date.now() - queryStart,
|
|
170
|
+
error: `warm query() failed: ${err.message}`,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
console.log(` ${label} query: first_token=${firstTokenMs}ms, total=${totalMs}ms`);
|
|
174
|
+
return { prewarmMs, firstTokenMs, totalMs };
|
|
175
|
+
}
|
|
176
|
+
// ── Main ────────────────────────────────────────────────────────────
|
|
177
|
+
async function main() {
|
|
178
|
+
console.log(`\n${BOLD}╔══════════════════════════════════════════════════════════╗${RESET}`);
|
|
179
|
+
console.log(`${BOLD}║ startup() Benchmark: Cold vs Warm Time-to-First-Token ║${RESET}`);
|
|
180
|
+
console.log(`${BOLD}╚══════════════════════════════════════════════════════════╝${RESET}\n`);
|
|
181
|
+
const config = loadConfig();
|
|
182
|
+
const minimaxKey = config.minimaxApiKey || process.env.MINIMAX_API_KEY || '';
|
|
183
|
+
if (!minimaxKey) {
|
|
184
|
+
console.log(`${RED}ERROR: No MiniMax API key found. Set minimaxApiKey in ai-config.json${RESET}`);
|
|
185
|
+
process.exit(1);
|
|
186
|
+
}
|
|
187
|
+
const MINIMAX_BASE_URL = 'https://api.minimax.io/anthropic';
|
|
188
|
+
const MINIMAX_MODEL = 'MiniMax-M2.7-highspeed';
|
|
189
|
+
const env = {
|
|
190
|
+
...process.env,
|
|
191
|
+
ANTHROPIC_API_KEY: minimaxKey,
|
|
192
|
+
ANTHROPIC_BASE_URL: MINIMAX_BASE_URL,
|
|
193
|
+
ANTHROPIC_MODEL: MINIMAX_MODEL,
|
|
194
|
+
ANTHROPIC_DEFAULT_SONNET_MODEL: MINIMAX_MODEL,
|
|
195
|
+
ANTHROPIC_DEFAULT_OPUS_MODEL: MINIMAX_MODEL,
|
|
196
|
+
ANTHROPIC_DEFAULT_HAIKU_MODEL: MINIMAX_MODEL,
|
|
197
|
+
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
|
|
198
|
+
};
|
|
199
|
+
const settingsEnv = {
|
|
200
|
+
ANTHROPIC_API_KEY: minimaxKey,
|
|
201
|
+
ANTHROPIC_BASE_URL: MINIMAX_BASE_URL,
|
|
202
|
+
ANTHROPIC_MODEL: MINIMAX_MODEL,
|
|
203
|
+
ANTHROPIC_DEFAULT_SONNET_MODEL: MINIMAX_MODEL,
|
|
204
|
+
ANTHROPIC_DEFAULT_OPUS_MODEL: MINIMAX_MODEL,
|
|
205
|
+
ANTHROPIC_DEFAULT_HAIKU_MODEL: MINIMAX_MODEL,
|
|
206
|
+
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
|
|
207
|
+
};
|
|
208
|
+
console.log(`${CYAN}Provider:${RESET} MiniMax`);
|
|
209
|
+
console.log(`${CYAN}Model:${RESET} ${MINIMAX_MODEL}`);
|
|
210
|
+
console.log(`${CYAN}Base URL:${RESET} ${MINIMAX_BASE_URL}`);
|
|
211
|
+
console.log(`${CYAN}Key:${RESET} ${minimaxKey.slice(0, 8)}...`);
|
|
212
|
+
console.log(`${CYAN}Exe:${RESET} ${CLAUDE_EXE || '(not found)'}\n`);
|
|
213
|
+
if (!CLAUDE_EXE) {
|
|
214
|
+
console.log(`${RED}ERROR: Could not resolve Claude Code executable${RESET}`);
|
|
215
|
+
process.exit(1);
|
|
216
|
+
}
|
|
217
|
+
// ============== TEST 1: Cold Query ==============
|
|
218
|
+
console.log(`${BOLD}TEST 1: Cold query() — current behavior${RESET}`);
|
|
219
|
+
hr();
|
|
220
|
+
const coldResults = [];
|
|
221
|
+
for (let i = 0; i < 3; i++) {
|
|
222
|
+
const result = await measureColdQuery(minimaxKey, MINIMAX_BASE_URL, MINIMAX_MODEL, env, settingsEnv, ` Run ${i + 1}`);
|
|
223
|
+
coldResults.push(result);
|
|
224
|
+
if (result.error) {
|
|
225
|
+
console.log(` ${RED}Error: ${result.error}${RESET}`);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
// ============== TEST 2: Warm startup() + query() ==============
|
|
229
|
+
console.log(`\n${BOLD}TEST 2: Warm startup() → query() — pre-warmed subprocess${RESET}`);
|
|
230
|
+
hr();
|
|
231
|
+
const warmResults = [];
|
|
232
|
+
for (let i = 0; i < 3; i++) {
|
|
233
|
+
const result = await measureWarmQuery(minimaxKey, MINIMAX_BASE_URL, MINIMAX_MODEL, env, settingsEnv, ` Run ${i + 1}`);
|
|
234
|
+
warmResults.push(result);
|
|
235
|
+
if (result.error) {
|
|
236
|
+
console.log(` ${RED}Error: ${result.error}${RESET}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
// ============== Summary ==============
|
|
240
|
+
console.log(`\n${BOLD}══════════════════════════════════════════════════════════${RESET}`);
|
|
241
|
+
console.log(`${BOLD} SUMMARY${RESET}`);
|
|
242
|
+
console.log(`${BOLD}══════════════════════════════════════════════════════════${RESET}\n`);
|
|
243
|
+
const validCold = coldResults.filter(r => !r.error && r.firstTokenMs > 0);
|
|
244
|
+
const validWarm = warmResults.filter(r => !r.error && r.firstTokenMs > 0);
|
|
245
|
+
if (validCold.length > 0) {
|
|
246
|
+
const avgColdFirst = Math.round(validCold.reduce((s, r) => s + r.firstTokenMs, 0) / validCold.length);
|
|
247
|
+
const avgColdTotal = Math.round(validCold.reduce((s, r) => s + r.totalMs, 0) / validCold.length);
|
|
248
|
+
const avgColdInit = Math.round(validCold.reduce((s, r) => s + r.startupMs, 0) / validCold.length);
|
|
249
|
+
console.log(` ${YELLOW}Cold query() (avg of ${validCold.length}):${RESET}`);
|
|
250
|
+
console.log(` Init/subprocess spawn: ${avgColdInit}ms`);
|
|
251
|
+
console.log(` Time to first token: ${avgColdFirst}ms`);
|
|
252
|
+
console.log(` Total query time: ${avgColdTotal}ms`);
|
|
253
|
+
}
|
|
254
|
+
else {
|
|
255
|
+
console.log(` ${RED}Cold query: ALL RUNS FAILED${RESET}`);
|
|
256
|
+
coldResults.forEach((r, i) => r.error && console.log(` Run ${i + 1}: ${r.error}`));
|
|
257
|
+
}
|
|
258
|
+
console.log();
|
|
259
|
+
if (validWarm.length > 0) {
|
|
260
|
+
const avgWarmPrem = Math.round(validWarm.reduce((s, r) => s + r.prewarmMs, 0) / validWarm.length);
|
|
261
|
+
const avgWarmFirst = Math.round(validWarm.reduce((s, r) => s + r.firstTokenMs, 0) / validWarm.length);
|
|
262
|
+
const avgWarmTotal = Math.round(validWarm.reduce((s, r) => s + r.totalMs, 0) / validWarm.length);
|
|
263
|
+
console.log(` ${GREEN}Warm startup() → query() (avg of ${validWarm.length}):${RESET}`);
|
|
264
|
+
console.log(` Prewarm (startup()): ${avgWarmPrem}ms`);
|
|
265
|
+
console.log(` Time to first token: ${avgWarmFirst}ms`);
|
|
266
|
+
console.log(` Total query time: ${avgWarmTotal}ms`);
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
console.log(` ${RED}Warm query: ALL RUNS FAILED${RESET}`);
|
|
270
|
+
warmResults.forEach((r, i) => r.error && console.log(` Run ${i + 1}: ${r.error}`));
|
|
271
|
+
}
|
|
272
|
+
console.log();
|
|
273
|
+
if (validCold.length > 0 && validWarm.length > 0) {
|
|
274
|
+
const avgColdFirst = Math.round(validCold.reduce((s, r) => s + r.firstTokenMs, 0) / validCold.length);
|
|
275
|
+
const avgWarmFirst = Math.round(validWarm.reduce((s, r) => s + r.firstTokenMs, 0) / validWarm.length);
|
|
276
|
+
const speedup = avgColdFirst > 0 ? (avgColdFirst / avgWarmFirst).toFixed(1) : 'N/A';
|
|
277
|
+
const saved = avgColdFirst - avgWarmFirst;
|
|
278
|
+
console.log(` ${BOLD}Speedup (first token):${RESET} ${speedup}x faster with startup()`);
|
|
279
|
+
console.log(` ${BOLD}Time saved:${RESET} ${saved > 0 ? saved : 0}ms per query`);
|
|
280
|
+
console.log();
|
|
281
|
+
if (parseFloat(speedup) >= 5) {
|
|
282
|
+
console.log(` ${GREEN}${BOLD}✓ Significant improvement! startup() is highly beneficial here.${RESET}`);
|
|
283
|
+
}
|
|
284
|
+
else if (parseFloat(speedup) >= 2) {
|
|
285
|
+
console.log(` ${YELLOW}! Moderate improvement. startup() helps but not 20x as advertised.${RESET}`);
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
console.log(` ${RED}✗ Minimal improvement. startup() overhead may not be worth it here.${RESET}`);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
console.log(`\n${BOLD}══════════════════════════════════════════════════════════${RESET}\n`);
|
|
292
|
+
// ============== Bonus: Can we reuse startup()? ==============
|
|
293
|
+
console.log(`${BOLD}TEST 3: Can one WarmQuery be reused for multiple queries?${RESET}`);
|
|
294
|
+
hr();
|
|
295
|
+
try {
|
|
296
|
+
const warmHandle = await startup({
|
|
297
|
+
options: {
|
|
298
|
+
apiKey: minimaxKey,
|
|
299
|
+
model: MINIMAX_MODEL,
|
|
300
|
+
cwd: '/tmp',
|
|
301
|
+
permissionMode: 'bypassPermissions',
|
|
302
|
+
allowDangerouslySkipPermissions: true,
|
|
303
|
+
maxTurns: 1,
|
|
304
|
+
env,
|
|
305
|
+
settings: { env: settingsEnv },
|
|
306
|
+
...(CLAUDE_EXE ? { pathToClaudeCodeExecutable: CLAUDE_EXE } : {}),
|
|
307
|
+
},
|
|
308
|
+
});
|
|
309
|
+
console.log(' startup() succeeded, attempting first query()...');
|
|
310
|
+
const q1Start = Date.now();
|
|
311
|
+
let q1FirstToken = 0;
|
|
312
|
+
const q1 = warmHandle.query('Say "one"');
|
|
313
|
+
for await (const event of q1) {
|
|
314
|
+
if (event.type === 'assistant' && q1FirstToken === 0) {
|
|
315
|
+
q1FirstToken = Date.now() - q1Start;
|
|
316
|
+
}
|
|
317
|
+
if (event.type === 'result') {
|
|
318
|
+
console.log(` Query 1: first_token=${q1FirstToken}ms, total=${Date.now() - q1Start}ms`);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
console.log(' Attempting second query() on same WarmQuery...');
|
|
322
|
+
try {
|
|
323
|
+
const q2 = warmHandle.query('Say "two"');
|
|
324
|
+
for await (const _event of q2) {
|
|
325
|
+
// drain
|
|
326
|
+
}
|
|
327
|
+
console.log(` ${GREEN}Second query succeeded — WarmQuery CAN be reused!${RESET}`);
|
|
328
|
+
}
|
|
329
|
+
catch (reuseErr) {
|
|
330
|
+
console.log(` ${RED}Second query FAILED: ${reuseErr.message}${RESET}`);
|
|
331
|
+
console.log(` ${YELLOW}→ WarmQuery is single-use, must call startup() again for each query.${RESET}`);
|
|
332
|
+
}
|
|
333
|
+
try {
|
|
334
|
+
warmHandle.close();
|
|
335
|
+
}
|
|
336
|
+
catch { }
|
|
337
|
+
}
|
|
338
|
+
catch (err) {
|
|
339
|
+
console.log(` ${RED}startup() failed: ${err.message}${RESET}`);
|
|
340
|
+
}
|
|
341
|
+
console.log();
|
|
342
|
+
process.exit(0);
|
|
343
|
+
}
|
|
344
|
+
main().catch((err) => {
|
|
345
|
+
console.error('Unhandled error:', err);
|
|
346
|
+
process.exit(1);
|
|
347
|
+
});
|