@supen-ai/cli 1.4.3 → 1.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/daemon/dist/agent-sdk/drivers/codex-app-server-driver.d.ts +2 -0
- package/daemon/dist/agent-sdk/drivers/codex-app-server-driver.d.ts.map +1 -1
- package/daemon/dist/agent-sdk/drivers/codex-app-server-driver.js +33 -35
- package/daemon/dist/agent-sdk/drivers/codex-app-server-driver.js.map +1 -1
- package/daemon/dist/autonomy/{session-autonomy.d.ts → thread-autonomy.d.ts} +1 -1
- package/daemon/dist/autonomy/thread-autonomy.d.ts.map +1 -0
- package/daemon/dist/autonomy/{session-autonomy.js → thread-autonomy.js} +1 -1
- package/daemon/dist/autonomy/thread-autonomy.js.map +1 -0
- package/daemon/dist/channels/http-routes.d.ts.map +1 -1
- package/daemon/dist/channels/http-routes.js +1 -7
- package/daemon/dist/channels/http-routes.js.map +1 -1
- package/daemon/dist/commands/catalog.js +1 -1
- package/daemon/dist/commands/catalog.js.map +1 -1
- package/daemon/dist/core/codex-subscription.d.ts.map +1 -1
- package/daemon/dist/core/codex-subscription.js +5 -27
- package/daemon/dist/core/codex-subscription.js.map +1 -1
- package/daemon/dist/core/direct-http-access.d.ts +2 -0
- package/daemon/dist/core/direct-http-access.d.ts.map +1 -0
- package/daemon/dist/core/direct-http-access.js +6 -0
- package/daemon/dist/core/direct-http-access.js.map +1 -0
- package/daemon/dist/core/gateway-config.d.ts.map +1 -1
- package/daemon/dist/core/gateway-config.js +8 -0
- package/daemon/dist/core/gateway-config.js.map +1 -1
- package/daemon/dist/core/gateway.d.ts +4 -0
- package/daemon/dist/core/gateway.d.ts.map +1 -1
- package/daemon/dist/core/gateway.js +60 -11
- package/daemon/dist/core/gateway.js.map +1 -1
- package/daemon/dist/core/logger.d.ts +1 -0
- package/daemon/dist/core/logger.d.ts.map +1 -1
- package/daemon/dist/core/logger.js +18 -0
- package/daemon/dist/core/logger.js.map +1 -1
- package/daemon/dist/core/os-info.d.ts +12 -1
- package/daemon/dist/core/os-info.d.ts.map +1 -1
- package/daemon/dist/core/os-info.js +29 -0
- package/daemon/dist/core/os-info.js.map +1 -1
- package/daemon/dist/core/store.js +1 -8
- package/daemon/dist/core/store.js.map +1 -1
- package/daemon/dist/core/thread-context.d.ts +6 -0
- package/daemon/dist/core/thread-context.d.ts.map +1 -1
- package/daemon/dist/core/thread-context.js +33 -0
- package/daemon/dist/core/thread-context.js.map +1 -1
- package/daemon/dist/core/thread-runtime-state.d.ts.map +1 -1
- package/daemon/dist/core/thread-runtime-state.js +8 -0
- package/daemon/dist/core/thread-runtime-state.js.map +1 -1
- package/daemon/dist/core/tool-lookup.d.ts +4 -0
- package/daemon/dist/core/tool-lookup.d.ts.map +1 -0
- package/daemon/dist/core/tool-lookup.js +51 -0
- package/daemon/dist/core/tool-lookup.js.map +1 -0
- package/daemon/dist/http/context.d.ts.map +1 -1
- package/daemon/dist/http/context.js +20 -1
- package/daemon/dist/http/context.js.map +1 -1
- package/daemon/dist/http/routes/agents.js +4 -4
- package/daemon/dist/http/routes/agents.js.map +1 -1
- package/daemon/dist/http/routes/automations.d.ts.map +1 -1
- package/daemon/dist/http/routes/automations.js +0 -8
- package/daemon/dist/http/routes/automations.js.map +1 -1
- package/daemon/dist/http/routes/rpc.d.ts.map +1 -1
- package/daemon/dist/http/routes/rpc.js +18 -6
- package/daemon/dist/http/routes/rpc.js.map +1 -1
- package/daemon/dist/http/routes/system.d.ts.map +1 -1
- package/daemon/dist/http/routes/system.js +193 -52
- package/daemon/dist/http/routes/system.js.map +1 -1
- package/daemon/dist/http/routes/threads.d.ts.map +1 -1
- package/daemon/dist/http/routes/threads.js +196 -6
- package/daemon/dist/http/routes/threads.js.map +1 -1
- package/daemon/dist/http/thread-stream.d.ts +1 -1
- package/daemon/dist/http/thread-stream.d.ts.map +1 -1
- package/daemon/dist/http/thread-stream.js +12 -2
- package/daemon/dist/http/thread-stream.js.map +1 -1
- package/daemon/dist/mcp/client.d.ts +3 -8
- package/daemon/dist/mcp/client.d.ts.map +1 -1
- package/daemon/dist/mcp/client.js +3 -16
- package/daemon/dist/mcp/client.js.map +1 -1
- package/daemon/dist/task-executor.d.ts +7 -5
- package/daemon/dist/task-executor.d.ts.map +1 -1
- package/daemon/dist/task-executor.js +100 -4
- package/daemon/dist/task-executor.js.map +1 -1
- package/daemon/dist/tools/system.js +1 -1
- package/daemon/dist/tools/system.js.map +1 -1
- package/daemon/package.json +4 -3
- package/daemon/scripts/bench-message-history.ts +584 -0
- package/daemon/scripts/browser-smoke.mjs +14 -15
- package/dist/bootstrap.js +1 -1
- package/dist/bootstrap.js.map +1 -1
- package/dist/chat.js +1 -1
- package/dist/chat.js.map +1 -1
- package/dist/commands.js +6 -3
- package/dist/commands.js.map +1 -1
- package/dist/computer.js +1 -1
- package/dist/index.js +1 -1
- package/dist/pairing.d.ts +1 -1
- package/dist/pairing.js +2 -2
- package/dist/pairing.js.map +1 -1
- package/dist/repl.js +1 -1
- package/dist/repl.js.map +1 -1
- package/dist/thread.js +10 -7
- package/dist/thread.js.map +1 -1
- package/dist/ui/app.js +3 -3
- package/dist/ui/app.js.map +1 -1
- package/package.json +1 -1
- package/daemon/dist/autonomy/session-autonomy.d.ts.map +0 -1
- package/daemon/dist/autonomy/session-autonomy.js.map +0 -1
- package/daemon/dist/mcp/tools.d.ts +0 -19
- package/daemon/dist/mcp/tools.d.ts.map +0 -1
- package/daemon/dist/mcp/tools.js +0 -132
- package/daemon/dist/mcp/tools.js.map +0 -1
- package/daemon/dist/tools/skill-tools.d.ts +0 -20
- package/daemon/dist/tools/skill-tools.d.ts.map +0 -1
- package/daemon/dist/tools/skill-tools.js +0 -60
- package/daemon/dist/tools/skill-tools.js.map +0 -1
|
@@ -0,0 +1,584 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { performance } from 'node:perf_hooks';
|
|
6
|
+
import { pathToFileURL } from 'node:url';
|
|
7
|
+
|
|
8
|
+
type SampleSummary = {
|
|
9
|
+
runs: number;
|
|
10
|
+
samplesMs: number[];
|
|
11
|
+
minMs: number;
|
|
12
|
+
p50Ms: number;
|
|
13
|
+
p95Ms: number;
|
|
14
|
+
maxMs: number;
|
|
15
|
+
avgMs: number;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
type ThreadHistoryPayload = {
|
|
19
|
+
messages?: unknown[];
|
|
20
|
+
events?: unknown[];
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
type HttpHistorySample = {
|
|
24
|
+
totalMs: number;
|
|
25
|
+
status: number;
|
|
26
|
+
bytes: number;
|
|
27
|
+
jsonParseMs: number;
|
|
28
|
+
replayMs: number;
|
|
29
|
+
messages: number;
|
|
30
|
+
events: number;
|
|
31
|
+
uiMessages: number;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
type SseSample = {
|
|
35
|
+
responseMs: number;
|
|
36
|
+
firstEventMs: number;
|
|
37
|
+
drainMs: number;
|
|
38
|
+
status: number;
|
|
39
|
+
events: number;
|
|
40
|
+
bytes: number;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
type BenchMessageHistoryOptions = {
|
|
44
|
+
baseUrl?: string;
|
|
45
|
+
threadId?: string;
|
|
46
|
+
computerId: string;
|
|
47
|
+
apiKey?: string;
|
|
48
|
+
authorization?: string;
|
|
49
|
+
runs: number;
|
|
50
|
+
limit: number;
|
|
51
|
+
streamSampleMs: number;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
export function summarize(samples: number[]): SampleSummary {
|
|
55
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
56
|
+
const percentile = (value: number) => {
|
|
57
|
+
if (sorted.length === 0) return 0;
|
|
58
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * value) - 1)] ?? 0;
|
|
59
|
+
};
|
|
60
|
+
const avg = samples.reduce((sum, sample) => sum + sample, 0) / Math.max(1, samples.length);
|
|
61
|
+
return {
|
|
62
|
+
runs: samples.length,
|
|
63
|
+
samplesMs: samples.map((sample) => Number(sample.toFixed(3))),
|
|
64
|
+
minMs: Number((sorted[0] ?? 0).toFixed(3)),
|
|
65
|
+
p50Ms: Number(percentile(0.5).toFixed(3)),
|
|
66
|
+
p95Ms: Number(percentile(0.95).toFixed(3)),
|
|
67
|
+
maxMs: Number((sorted.at(-1) ?? 0).toFixed(3)),
|
|
68
|
+
avgMs: Number(avg.toFixed(3)),
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function readPositiveInteger(value: string | undefined, fallback: number): number {
|
|
73
|
+
const parsed = Number.parseInt(value || '', 10);
|
|
74
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function parseBenchMessageHistoryArgs(argv: string[]): BenchMessageHistoryOptions {
|
|
78
|
+
const options: BenchMessageHistoryOptions = {
|
|
79
|
+
baseUrl: process.env.BENCH_MESSAGE_HISTORY_REMOTE_BASE_URL?.trim(),
|
|
80
|
+
threadId: process.env.BENCH_MESSAGE_HISTORY_REMOTE_THREAD_ID?.trim(),
|
|
81
|
+
computerId: process.env.BENCH_MESSAGE_HISTORY_REMOTE_COMPUTER_ID?.trim() || 'local',
|
|
82
|
+
apiKey: process.env.BENCH_MESSAGE_HISTORY_REMOTE_API_KEY?.trim(),
|
|
83
|
+
authorization: (
|
|
84
|
+
process.env.BENCH_MESSAGE_HISTORY_REMOTE_AUTHORIZATION ||
|
|
85
|
+
(process.env.BENCH_MESSAGE_HISTORY_REMOTE_BEARER
|
|
86
|
+
? `Bearer ${process.env.BENCH_MESSAGE_HISTORY_REMOTE_BEARER}`
|
|
87
|
+
: '')
|
|
88
|
+
).trim() || undefined,
|
|
89
|
+
runs: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_RUNS, 12),
|
|
90
|
+
limit: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_LIMIT, 400),
|
|
91
|
+
streamSampleMs: readPositiveInteger(process.env.BENCH_MESSAGE_HISTORY_STREAM_SAMPLE_MS, 5_000),
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
95
|
+
const arg = argv[index];
|
|
96
|
+
const next = argv[index + 1];
|
|
97
|
+
if (arg === '--base-url') {
|
|
98
|
+
options.baseUrl = next?.trim();
|
|
99
|
+
index += 1;
|
|
100
|
+
} else if (arg === '--thread-id') {
|
|
101
|
+
options.threadId = next?.trim();
|
|
102
|
+
index += 1;
|
|
103
|
+
} else if (arg === '--computer-id') {
|
|
104
|
+
options.computerId = next?.trim() || options.computerId;
|
|
105
|
+
index += 1;
|
|
106
|
+
} else if (arg === '--api-key') {
|
|
107
|
+
options.apiKey = next?.trim();
|
|
108
|
+
index += 1;
|
|
109
|
+
} else if (arg === '--authorization') {
|
|
110
|
+
options.authorization = next?.trim();
|
|
111
|
+
index += 1;
|
|
112
|
+
} else if (arg === '--runs') {
|
|
113
|
+
options.runs = readPositiveInteger(next, options.runs);
|
|
114
|
+
index += 1;
|
|
115
|
+
} else if (arg === '--limit') {
|
|
116
|
+
options.limit = readPositiveInteger(next, options.limit);
|
|
117
|
+
index += 1;
|
|
118
|
+
} else if (arg === '--stream-sample-ms') {
|
|
119
|
+
options.streamSampleMs = readPositiveInteger(next, options.streamSampleMs);
|
|
120
|
+
index += 1;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return options;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function buildCodexThreadEndpoint(
|
|
128
|
+
baseUrl: string,
|
|
129
|
+
threadId: string,
|
|
130
|
+
endpoint: 'messages' | 'stream',
|
|
131
|
+
params: Record<string, string> = {},
|
|
132
|
+
computerId = 'local',
|
|
133
|
+
): string {
|
|
134
|
+
const normalizedBaseUrl = baseUrl.replace(/\/+$/, '');
|
|
135
|
+
const url = new URL(
|
|
136
|
+
`${normalizedBaseUrl}/api/computers/${encodeURIComponent(computerId)}/agents/codex/threads/${encodeURIComponent(threadId)}/${endpoint}`,
|
|
137
|
+
);
|
|
138
|
+
for (const [key, value] of Object.entries(params)) {
|
|
139
|
+
url.searchParams.set(key, value);
|
|
140
|
+
}
|
|
141
|
+
return url.toString();
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function summarizeField<T>(samples: T[], selector: (sample: T) => number): SampleSummary {
|
|
145
|
+
return summarize(samples.map(selector));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function denseTimestamp(index: number, suffixMs = 0): string {
|
|
149
|
+
return `2026-06-04T12:${String(Math.floor(index / 60)).padStart(2, '0')}:${String(index % 60).padStart(2, '0')}.${String(suffixMs).padStart(3, '0')}Z`;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function measure<T>(runs: number, fn: () => T): { summary: SampleSummary; last: T } {
|
|
153
|
+
const samples: number[] = [];
|
|
154
|
+
let last: T;
|
|
155
|
+
for (let run = 0; run < runs; run += 1) {
|
|
156
|
+
const startedAt = performance.now();
|
|
157
|
+
last = fn();
|
|
158
|
+
samples.push(performance.now() - startedAt);
|
|
159
|
+
}
|
|
160
|
+
return { summary: summarize(samples), last: last! };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function warmup(runs: number, fn: () => void): void {
|
|
164
|
+
for (let run = 0; run < runs; run += 1) {
|
|
165
|
+
fn();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function measureAsync<T>(runs: number, fn: () => Promise<T>): Promise<{ samples: T[]; total: SampleSummary }> {
|
|
170
|
+
const samples: T[] = [];
|
|
171
|
+
const totals: number[] = [];
|
|
172
|
+
for (let run = 0; run < runs; run += 1) {
|
|
173
|
+
const startedAt = performance.now();
|
|
174
|
+
samples.push(await fn());
|
|
175
|
+
totals.push(performance.now() - startedAt);
|
|
176
|
+
}
|
|
177
|
+
return { samples, total: summarize(totals) };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async function warmupAsync(runs: number, fn: () => Promise<void>): Promise<void> {
|
|
181
|
+
for (let run = 0; run < runs; run += 1) {
|
|
182
|
+
await fn();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function createFixture(agentHome: string, supenHome: string) {
|
|
187
|
+
const threadId = '019e8b00-0000-7000-9000-000000000099';
|
|
188
|
+
const threadsDir = path.join(agentHome, 'threads', '2026', '06', '04');
|
|
189
|
+
fs.mkdirSync(threadsDir, { recursive: true });
|
|
190
|
+
fs.writeFileSync(
|
|
191
|
+
path.join(agentHome, 'thread_index.jsonl'),
|
|
192
|
+
`${JSON.stringify({ id: threadId, thread_name: 'Benchmark thread', updated_at: '2026-06-04T12:00:00.000Z' })}\n`,
|
|
193
|
+
'utf-8',
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
const threadPath = path.join(threadsDir, `rollout-2026-06-04T12-00-00-${threadId}.jsonl`);
|
|
197
|
+
const largeAssistantText = 'assistant-detail '.repeat(1024);
|
|
198
|
+
const lines: string[] = [];
|
|
199
|
+
for (let index = 0; index < 250; index += 1) {
|
|
200
|
+
lines.push(JSON.stringify({
|
|
201
|
+
timestamp: denseTimestamp(index, 0),
|
|
202
|
+
type: 'response_item',
|
|
203
|
+
payload: {
|
|
204
|
+
type: 'message',
|
|
205
|
+
role: 'user',
|
|
206
|
+
content: [{ type: 'input_text', text: `request ${index}` }],
|
|
207
|
+
},
|
|
208
|
+
}));
|
|
209
|
+
lines.push(JSON.stringify({
|
|
210
|
+
timestamp: denseTimestamp(index, 100),
|
|
211
|
+
type: 'response_item',
|
|
212
|
+
payload: {
|
|
213
|
+
type: 'function_call',
|
|
214
|
+
name: 'exec_command',
|
|
215
|
+
call_id: `call-${index}`,
|
|
216
|
+
arguments: JSON.stringify({ cmd: `echo ${index}` }),
|
|
217
|
+
},
|
|
218
|
+
}));
|
|
219
|
+
lines.push(JSON.stringify({
|
|
220
|
+
timestamp: denseTimestamp(index, 200),
|
|
221
|
+
type: 'response_item',
|
|
222
|
+
payload: {
|
|
223
|
+
type: 'message',
|
|
224
|
+
role: 'assistant',
|
|
225
|
+
content: [{ type: 'output_text', text: `answer ${index}\n${largeAssistantText}` }],
|
|
226
|
+
},
|
|
227
|
+
}));
|
|
228
|
+
}
|
|
229
|
+
fs.writeFileSync(threadPath, `${lines.join('\n')}\n`, 'utf-8');
|
|
230
|
+
|
|
231
|
+
const eventLogDir = path.join(supenHome, 'threads', threadId, 'event-log');
|
|
232
|
+
fs.mkdirSync(eventLogDir, { recursive: true });
|
|
233
|
+
const rawEvents: string[] = [];
|
|
234
|
+
for (let index = 0; index < 120; index += 1) {
|
|
235
|
+
rawEvents.push(JSON.stringify({
|
|
236
|
+
event_id: `bench-event-${index}`,
|
|
237
|
+
runtime_event_id: `bench-runtime-event-${index}`,
|
|
238
|
+
thread_id: threadId,
|
|
239
|
+
runtime_thread_id: threadId,
|
|
240
|
+
sequence: index + 1,
|
|
241
|
+
runtime_sequence: index + 1,
|
|
242
|
+
source: 'codex-app-server',
|
|
243
|
+
event_type: 'item/agentMessage/delta',
|
|
244
|
+
raw_payload: {
|
|
245
|
+
type: 'data-codex-event',
|
|
246
|
+
data: {
|
|
247
|
+
eventType: 'item/agentMessage/delta',
|
|
248
|
+
raw: {
|
|
249
|
+
method: 'item/agentMessage/delta',
|
|
250
|
+
params: {
|
|
251
|
+
threadId,
|
|
252
|
+
turnId: 'turn-bench',
|
|
253
|
+
itemId: 'assistant-bench',
|
|
254
|
+
delta: `stream-delta-${index} `,
|
|
255
|
+
},
|
|
256
|
+
},
|
|
257
|
+
},
|
|
258
|
+
},
|
|
259
|
+
received_at: denseTimestamp(index, 300),
|
|
260
|
+
payload_hash: `bench-${index}`,
|
|
261
|
+
}));
|
|
262
|
+
}
|
|
263
|
+
fs.writeFileSync(path.join(eventLogDir, 'raw-events.jsonl'), `${rawEvents.join('\n')}\n`, 'utf-8');
|
|
264
|
+
fs.writeFileSync(
|
|
265
|
+
path.join(eventLogDir, 'head.json'),
|
|
266
|
+
`${JSON.stringify({ last_sequence: rawEvents.length, updated_at: denseTimestamp(rawEvents.length, 0) })}\n`,
|
|
267
|
+
'utf-8',
|
|
268
|
+
);
|
|
269
|
+
|
|
270
|
+
return {
|
|
271
|
+
threadId,
|
|
272
|
+
threadPath,
|
|
273
|
+
jsonlLines: lines.length,
|
|
274
|
+
seededStreamEvents: rawEvents.length,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
async function startLoopbackServer() {
|
|
279
|
+
const { dispatchRequest } = await import('../src/http/router.js');
|
|
280
|
+
const { setCorsHeaders } = await import('../src/http/context.js');
|
|
281
|
+
const server = http.createServer(async (req, res) => {
|
|
282
|
+
if (req.method === 'OPTIONS') {
|
|
283
|
+
setCorsHeaders(req, res);
|
|
284
|
+
res.writeHead(204);
|
|
285
|
+
res.end();
|
|
286
|
+
return;
|
|
287
|
+
}
|
|
288
|
+
setCorsHeaders(req, res);
|
|
289
|
+
await dispatchRequest(req, res, () => {});
|
|
290
|
+
});
|
|
291
|
+
await new Promise<void>((resolve, reject) => {
|
|
292
|
+
server.listen(0, '127.0.0.1', resolve);
|
|
293
|
+
server.on('error', reject);
|
|
294
|
+
});
|
|
295
|
+
const address = server.address();
|
|
296
|
+
if (!address || typeof address === 'string') {
|
|
297
|
+
throw new Error('Expected loopback TCP server address.');
|
|
298
|
+
}
|
|
299
|
+
return {
|
|
300
|
+
server,
|
|
301
|
+
baseUrl: `http://127.0.0.1:${address.port}`,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
async function measureHttpHistory(
|
|
306
|
+
url: string,
|
|
307
|
+
buildThreadUIMessages: (payload: ThreadHistoryPayload, options: { includeSilentEvents: boolean }) => unknown[],
|
|
308
|
+
headers: HeadersInit = {},
|
|
309
|
+
): Promise<HttpHistorySample> {
|
|
310
|
+
const startedAt = performance.now();
|
|
311
|
+
const response = await fetch(url, { headers });
|
|
312
|
+
const text = await response.text();
|
|
313
|
+
const totalMs = performance.now() - startedAt;
|
|
314
|
+
if (!response.ok) {
|
|
315
|
+
throw new Error(`History request failed (${response.status}) for ${url}: ${text.slice(0, 300)}`);
|
|
316
|
+
}
|
|
317
|
+
const parseStartedAt = performance.now();
|
|
318
|
+
const history = JSON.parse(text) as ThreadHistoryPayload;
|
|
319
|
+
const jsonParseMs = performance.now() - parseStartedAt;
|
|
320
|
+
const replayStartedAt = performance.now();
|
|
321
|
+
const uiMessages = buildThreadUIMessages(history, { includeSilentEvents: false });
|
|
322
|
+
const replayMs = performance.now() - replayStartedAt;
|
|
323
|
+
return {
|
|
324
|
+
totalMs,
|
|
325
|
+
status: response.status,
|
|
326
|
+
bytes: Buffer.byteLength(text),
|
|
327
|
+
jsonParseMs,
|
|
328
|
+
replayMs,
|
|
329
|
+
messages: history.messages?.length ?? 0,
|
|
330
|
+
events: history.events?.length ?? 0,
|
|
331
|
+
uiMessages: uiMessages.length,
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
async function measureSse(
|
|
336
|
+
url: string,
|
|
337
|
+
headers: HeadersInit = {},
|
|
338
|
+
options: { targetEvents?: number; sampleMs?: number } = {},
|
|
339
|
+
): Promise<SseSample> {
|
|
340
|
+
const startedAt = performance.now();
|
|
341
|
+
const targetEvents = Math.max(1, options.targetEvents ?? 120);
|
|
342
|
+
const sampleMs = Math.max(1, options.sampleMs ?? 5_000);
|
|
343
|
+
const controller = new AbortController();
|
|
344
|
+
const timeout = setTimeout(() => controller.abort(), sampleMs);
|
|
345
|
+
let response: Response;
|
|
346
|
+
try {
|
|
347
|
+
response = await fetch(url, { headers, signal: controller.signal });
|
|
348
|
+
} catch (error) {
|
|
349
|
+
clearTimeout(timeout);
|
|
350
|
+
if ((error as Error).name === 'AbortError') {
|
|
351
|
+
const elapsedMs = performance.now() - startedAt;
|
|
352
|
+
return {
|
|
353
|
+
responseMs: elapsedMs,
|
|
354
|
+
firstEventMs: 0,
|
|
355
|
+
drainMs: elapsedMs,
|
|
356
|
+
status: 0,
|
|
357
|
+
events: 0,
|
|
358
|
+
bytes: 0,
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
throw error;
|
|
362
|
+
}
|
|
363
|
+
const responseMs = performance.now() - startedAt;
|
|
364
|
+
if (!response.ok) {
|
|
365
|
+
const body = await response.text().catch(() => '');
|
|
366
|
+
clearTimeout(timeout);
|
|
367
|
+
throw new Error(`SSE request failed (${response.status}) for ${url}: ${body.slice(0, 300)}`);
|
|
368
|
+
}
|
|
369
|
+
if (!response.body) {
|
|
370
|
+
clearTimeout(timeout);
|
|
371
|
+
throw new Error(`SSE response body missing (${response.status})`);
|
|
372
|
+
}
|
|
373
|
+
const reader = response.body.getReader();
|
|
374
|
+
const decoder = new TextDecoder();
|
|
375
|
+
let buffer = '';
|
|
376
|
+
let bytes = 0;
|
|
377
|
+
let events = 0;
|
|
378
|
+
let firstEventMs = 0;
|
|
379
|
+
const processLine = (line: string) => {
|
|
380
|
+
if (!line.startsWith('data:')) return;
|
|
381
|
+
const data = line.slice(5).trim();
|
|
382
|
+
if (!data) return;
|
|
383
|
+
events += 1;
|
|
384
|
+
if (firstEventMs === 0) firstEventMs = performance.now() - startedAt;
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
try {
|
|
388
|
+
while (events < targetEvents) {
|
|
389
|
+
const { done, value } = await reader.read();
|
|
390
|
+
if (done) break;
|
|
391
|
+
bytes += value.byteLength;
|
|
392
|
+
buffer += decoder.decode(value, { stream: true });
|
|
393
|
+
const lines = buffer.split(/\r?\n/);
|
|
394
|
+
buffer = lines.pop() ?? '';
|
|
395
|
+
for (const line of lines) processLine(line);
|
|
396
|
+
}
|
|
397
|
+
} catch (error) {
|
|
398
|
+
if ((error as Error).name !== 'AbortError') throw error;
|
|
399
|
+
} finally {
|
|
400
|
+
clearTimeout(timeout);
|
|
401
|
+
}
|
|
402
|
+
await reader.cancel().catch(() => {});
|
|
403
|
+
return {
|
|
404
|
+
responseMs,
|
|
405
|
+
firstEventMs,
|
|
406
|
+
drainMs: performance.now() - startedAt,
|
|
407
|
+
status: response.status,
|
|
408
|
+
events,
|
|
409
|
+
bytes,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function sampleShape<T extends Record<string, unknown>>(sample: T | undefined): T | undefined {
|
|
414
|
+
if (!sample) return undefined;
|
|
415
|
+
return Object.fromEntries(
|
|
416
|
+
Object.entries(sample).map(([key, value]) => [
|
|
417
|
+
key,
|
|
418
|
+
typeof value === 'number' ? Number(value.toFixed(3)) : value,
|
|
419
|
+
]),
|
|
420
|
+
) as T;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async function main() {
|
|
424
|
+
const options = parseBenchMessageHistoryArgs(process.argv.slice(2));
|
|
425
|
+
const agentHome = fs.mkdtempSync(path.join(os.tmpdir(), 'supen-message-history-bench-'));
|
|
426
|
+
const supenHome = path.join(agentHome, '.supen');
|
|
427
|
+
process.env.CODEX_HOME = agentHome;
|
|
428
|
+
process.env.HOME = agentHome;
|
|
429
|
+
process.env.SUPEN_HOME = supenHome;
|
|
430
|
+
process.env.HTTP_API_KEY = '';
|
|
431
|
+
process.env.NODE_ENV = 'test';
|
|
432
|
+
|
|
433
|
+
let server: http.Server | null = null;
|
|
434
|
+
try {
|
|
435
|
+
const fixture = createFixture(agentHome, supenHome);
|
|
436
|
+
const { readCodexThreadHistory } = await import('../src/http/routes/system.js');
|
|
437
|
+
const { directHttpAccessToken } = await import('../src/core/direct-http-access.js');
|
|
438
|
+
const { buildThreadUIMessages } = await import('../../app/app/lib/chat-ui-message-replay.ts');
|
|
439
|
+
|
|
440
|
+
const requestedHistoryLimit = options.limit;
|
|
441
|
+
warmup(2, () => {
|
|
442
|
+
readCodexThreadHistory(fixture.threadId, requestedHistoryLimit);
|
|
443
|
+
});
|
|
444
|
+
const historyBench = measure(options.runs, () => readCodexThreadHistory(fixture.threadId, requestedHistoryLimit));
|
|
445
|
+
const history = historyBench.last ?? { messages: [], events: [] };
|
|
446
|
+
warmup(5, () => {
|
|
447
|
+
buildThreadUIMessages(history, { includeSilentEvents: false });
|
|
448
|
+
});
|
|
449
|
+
const replayBench = measure(80, () =>
|
|
450
|
+
buildThreadUIMessages(history, { includeSilentEvents: false }),
|
|
451
|
+
);
|
|
452
|
+
|
|
453
|
+
const loopback = await startLoopbackServer();
|
|
454
|
+
server = loopback.server;
|
|
455
|
+
const directHeaders = {
|
|
456
|
+
Origin: 'https://hub.supen.ai',
|
|
457
|
+
'X-API-Key': directHttpAccessToken(),
|
|
458
|
+
};
|
|
459
|
+
const historyUrl = buildCodexThreadEndpoint(loopback.baseUrl, fixture.threadId, 'messages', {
|
|
460
|
+
limit: String(requestedHistoryLimit),
|
|
461
|
+
eventLimit: '200',
|
|
462
|
+
});
|
|
463
|
+
const streamUrl = buildCodexThreadEndpoint(loopback.baseUrl, fixture.threadId, 'stream', {
|
|
464
|
+
after: '0',
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
await warmupAsync(2, async () => {
|
|
468
|
+
await measureHttpHistory(historyUrl, buildThreadUIMessages, directHeaders);
|
|
469
|
+
});
|
|
470
|
+
const httpHistoryBench = await measureAsync(options.runs, () =>
|
|
471
|
+
measureHttpHistory(historyUrl, buildThreadUIMessages, directHeaders),
|
|
472
|
+
);
|
|
473
|
+
const sseBench = await measureAsync(Math.max(1, Math.min(options.runs, 8)), () =>
|
|
474
|
+
measureSse(streamUrl, directHeaders, {
|
|
475
|
+
targetEvents: fixture.seededStreamEvents,
|
|
476
|
+
sampleMs: options.streamSampleMs,
|
|
477
|
+
}),
|
|
478
|
+
);
|
|
479
|
+
const remoteBaseUrl = (options.baseUrl || '').trim().replace(/\/+$/, '');
|
|
480
|
+
const remoteThreadId = (options.threadId || '').trim();
|
|
481
|
+
const remoteComputerId = options.computerId.trim() || 'local';
|
|
482
|
+
const remoteHeaders: Record<string, string> = {};
|
|
483
|
+
if (options.apiKey) remoteHeaders['X-API-Key'] = options.apiKey;
|
|
484
|
+
if (options.authorization) remoteHeaders.Authorization = options.authorization;
|
|
485
|
+
const shouldRunRemote = Boolean(remoteBaseUrl && remoteThreadId);
|
|
486
|
+
const remoteHistoryUrl = shouldRunRemote
|
|
487
|
+
? buildCodexThreadEndpoint(
|
|
488
|
+
remoteBaseUrl,
|
|
489
|
+
remoteThreadId,
|
|
490
|
+
'messages',
|
|
491
|
+
{ limit: String(requestedHistoryLimit), eventLimit: '200' },
|
|
492
|
+
remoteComputerId,
|
|
493
|
+
)
|
|
494
|
+
: '';
|
|
495
|
+
const remoteStreamUrl = shouldRunRemote
|
|
496
|
+
? buildCodexThreadEndpoint(remoteBaseUrl, remoteThreadId, 'stream', { after: '0' }, remoteComputerId)
|
|
497
|
+
: '';
|
|
498
|
+
const remoteRuns = Math.max(1, Math.min(options.runs, 6));
|
|
499
|
+
const remoteHistoryBench = shouldRunRemote
|
|
500
|
+
? await measureAsync(remoteRuns, () => measureHttpHistory(remoteHistoryUrl, buildThreadUIMessages, remoteHeaders))
|
|
501
|
+
: null;
|
|
502
|
+
const shouldRunRemoteStream = shouldRunRemote && process.env.BENCH_MESSAGE_HISTORY_REMOTE_STREAM !== '0';
|
|
503
|
+
const remoteSseBench = shouldRunRemoteStream
|
|
504
|
+
? await measureAsync(Math.max(1, Math.min(options.runs, 3)), () =>
|
|
505
|
+
measureSse(remoteStreamUrl, remoteHeaders, {
|
|
506
|
+
targetEvents: Number.parseInt(process.env.BENCH_MESSAGE_HISTORY_REMOTE_STREAM_EVENTS || '1', 10) || 1,
|
|
507
|
+
sampleMs: options.streamSampleMs,
|
|
508
|
+
}),
|
|
509
|
+
)
|
|
510
|
+
: null;
|
|
511
|
+
|
|
512
|
+
console.log(JSON.stringify({
|
|
513
|
+
benchmark: {
|
|
514
|
+
generatedAt: new Date().toISOString(),
|
|
515
|
+
options,
|
|
516
|
+
},
|
|
517
|
+
fixture: {
|
|
518
|
+
threadJsonlBytes: fs.statSync(fixture.threadPath).size,
|
|
519
|
+
jsonlLines: fixture.jsonlLines,
|
|
520
|
+
seededStreamEvents: fixture.seededStreamEvents,
|
|
521
|
+
requestedHistoryLimit,
|
|
522
|
+
loadedMessages: history.messages.length,
|
|
523
|
+
loadedEvents: history.events.length,
|
|
524
|
+
replayMessages: replayBench.last.length,
|
|
525
|
+
},
|
|
526
|
+
inProcess: {
|
|
527
|
+
daemonHistory: historyBench.summary,
|
|
528
|
+
frontendReplay: replayBench.summary,
|
|
529
|
+
},
|
|
530
|
+
loopbackHttp: {
|
|
531
|
+
historyTotal: summarizeField(httpHistoryBench.samples, (sample) => sample.totalMs),
|
|
532
|
+
historyJsonParse: summarizeField(httpHistoryBench.samples, (sample) => sample.jsonParseMs),
|
|
533
|
+
historyReplay: summarizeField(httpHistoryBench.samples, (sample) => sample.replayMs),
|
|
534
|
+
last: sampleShape(httpHistoryBench.samples.at(-1)),
|
|
535
|
+
},
|
|
536
|
+
loopbackSse: {
|
|
537
|
+
response: summarizeField(sseBench.samples, (sample) => sample.responseMs),
|
|
538
|
+
firstEvent: summarizeField(sseBench.samples, (sample) => sample.firstEventMs),
|
|
539
|
+
drain: summarizeField(sseBench.samples, (sample) => sample.drainMs),
|
|
540
|
+
last: sampleShape(sseBench.samples.at(-1)),
|
|
541
|
+
},
|
|
542
|
+
compareRemote: {
|
|
543
|
+
...(remoteHistoryBench
|
|
544
|
+
? {
|
|
545
|
+
baseUrl: remoteBaseUrl,
|
|
546
|
+
computerId: remoteComputerId,
|
|
547
|
+
threadId: remoteThreadId,
|
|
548
|
+
historyTotal: summarizeField(remoteHistoryBench.samples, (sample) => sample.totalMs),
|
|
549
|
+
historyJsonParse: summarizeField(remoteHistoryBench.samples, (sample) => sample.jsonParseMs),
|
|
550
|
+
historyReplay: summarizeField(remoteHistoryBench.samples, (sample) => sample.replayMs),
|
|
551
|
+
lastHistory: sampleShape(remoteHistoryBench.samples.at(-1)),
|
|
552
|
+
...(remoteSseBench
|
|
553
|
+
? {
|
|
554
|
+
sseResponse: summarizeField(remoteSseBench.samples, (sample) => sample.responseMs),
|
|
555
|
+
sseFirstEvent: summarizeField(remoteSseBench.samples, (sample) => sample.firstEventMs),
|
|
556
|
+
sseDrain: summarizeField(remoteSseBench.samples, (sample) => sample.drainMs),
|
|
557
|
+
lastSse: sampleShape(remoteSseBench.samples.at(-1)),
|
|
558
|
+
}
|
|
559
|
+
: {
|
|
560
|
+
sseSkipped:
|
|
561
|
+
'Remote SSE timing is enabled by default when a remote base URL and thread ID are provided. BENCH_MESSAGE_HISTORY_REMOTE_STREAM=0 skips it.',
|
|
562
|
+
}),
|
|
563
|
+
}
|
|
564
|
+
: {
|
|
565
|
+
skipped: true,
|
|
566
|
+
note:
|
|
567
|
+
'Set --base-url plus --thread-id or BENCH_MESSAGE_HISTORY_REMOTE_BASE_URL plus BENCH_MESSAGE_HISTORY_REMOTE_THREAD_ID. Optional: --computer-id, --api-key, --authorization, BENCH_MESSAGE_HISTORY_REMOTE_COMPUTER_ID/API_KEY/AUTHORIZATION/BEARER.',
|
|
568
|
+
}),
|
|
569
|
+
},
|
|
570
|
+
}, null, 2));
|
|
571
|
+
} finally {
|
|
572
|
+
if (server) {
|
|
573
|
+
await new Promise<void>((resolve) => server!.close(() => resolve()));
|
|
574
|
+
}
|
|
575
|
+
fs.rmSync(agentHome, { recursive: true, force: true });
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
580
|
+
main().catch((error) => {
|
|
581
|
+
console.error(error);
|
|
582
|
+
process.exitCode = 1;
|
|
583
|
+
});
|
|
584
|
+
}
|
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
* Headless smoke: loads Supen app, checks console, visits key routes.
|
|
3
3
|
* Run from apps/daemon: node scripts/browser-smoke.mjs
|
|
4
4
|
*
|
|
5
|
-
* Uses Chromium for list routes
|
|
6
|
-
* because Chromium’s renderer sometimes crashes on this app (WebGL/Rive/etc.).
|
|
5
|
+
* Uses Chromium for list routes and the task chat route.
|
|
7
6
|
*/
|
|
8
|
-
import { chromium
|
|
7
|
+
import { chromium } from "playwright";
|
|
9
8
|
|
|
10
9
|
const BASE = process.env.SUPEN_URL || "http://127.0.0.1:2655";
|
|
11
10
|
|
|
12
|
-
const routes = ["/", "/
|
|
11
|
+
const routes = ["/", "/tasks", "/automations", "/plugins", "/computers"];
|
|
13
12
|
|
|
14
13
|
async function main() {
|
|
15
14
|
const browser = await chromium.launch({
|
|
@@ -53,8 +52,8 @@ async function main() {
|
|
|
53
52
|
|
|
54
53
|
await browser.close();
|
|
55
54
|
|
|
56
|
-
// Chat route in a fresh browser
|
|
57
|
-
process.stdout.write(`\n→ Chat flow (first
|
|
55
|
+
// Chat route in a fresh browser.
|
|
56
|
+
process.stdout.write(`\n→ Chat flow (first task, Chromium)\n`);
|
|
58
57
|
const wk = await chromium.launch({ headless: true, executablePath: '/usr/bin/google-chrome' });
|
|
59
58
|
const p2 = await wk.newPage();
|
|
60
59
|
p2.on("pageerror", (e) => errors.push(`pageerror: ${e.message}`));
|
|
@@ -62,18 +61,18 @@ async function main() {
|
|
|
62
61
|
if (msg.type() === "error") errors.push(`console: ${msg.text()}`);
|
|
63
62
|
});
|
|
64
63
|
|
|
65
|
-
const
|
|
66
|
-
await p2.goto(
|
|
64
|
+
const tasksList = `${BASE.replace(/\/$/, "")}/tasks`;
|
|
65
|
+
await p2.goto(tasksList, {
|
|
67
66
|
waitUntil: "domcontentloaded",
|
|
68
67
|
timeout: 45_000,
|
|
69
68
|
});
|
|
70
69
|
await p2.waitForTimeout(1500);
|
|
71
|
-
const
|
|
72
|
-
'main a[href^="/
|
|
70
|
+
const taskLink = p2.locator(
|
|
71
|
+
'main a[href^="/tasks/"]:not([href$="/new"])',
|
|
73
72
|
);
|
|
74
|
-
const n = await
|
|
73
|
+
const n = await taskLink.count();
|
|
75
74
|
if (n > 0) {
|
|
76
|
-
const href = await
|
|
75
|
+
const href = await taskLink.first().getAttribute("href");
|
|
77
76
|
const chatUrl = `${BASE.replace(/\/$/, "")}${href}`;
|
|
78
77
|
process.stdout.write(` navigating: ${chatUrl}\n`);
|
|
79
78
|
await p2.goto(chatUrl, {
|
|
@@ -87,7 +86,7 @@ async function main() {
|
|
|
87
86
|
.first()
|
|
88
87
|
.innerText()
|
|
89
88
|
.catch(() => "");
|
|
90
|
-
const hasLoading = /Loading
|
|
89
|
+
const hasLoading = /Loading task/i.test(bodySample);
|
|
91
90
|
const hasComposer =
|
|
92
91
|
(await p2.locator('textarea, [contenteditable="true"]').count()) > 0;
|
|
93
92
|
process.stdout.write(
|
|
@@ -97,11 +96,11 @@ async function main() {
|
|
|
97
96
|
process.stdout.write(` main (snippet): ${snippet}…\n`);
|
|
98
97
|
if (hasLoading && !hasComposer) {
|
|
99
98
|
errors.push(
|
|
100
|
-
"chat-flow:
|
|
99
|
+
"chat-flow: task view stuck on Loading task or missing composer",
|
|
101
100
|
);
|
|
102
101
|
}
|
|
103
102
|
} else {
|
|
104
|
-
process.stdout.write(" (no
|
|
103
|
+
process.stdout.write(" (no tasks to open — skip)\n");
|
|
105
104
|
}
|
|
106
105
|
|
|
107
106
|
await wk.close();
|
package/dist/bootstrap.js
CHANGED
|
@@ -120,7 +120,7 @@ async function bootstrapGateway(primaryCli) {
|
|
|
120
120
|
const result = await loginToGateway(gatewayUrl, email, password);
|
|
121
121
|
console.log(`\n Logged in as ${result.user?.email || email}`);
|
|
122
122
|
console.log('\n Next steps:');
|
|
123
|
-
console.log(' • Run `supen whoami` to verify your
|
|
123
|
+
console.log(' • Run `supen whoami` to verify your sign-in status');
|
|
124
124
|
console.log(' • Run `supen chat "hi"` to talk through the gateway\n');
|
|
125
125
|
}
|
|
126
126
|
async function bootstrapLocal(primaryCli) {
|