codex-copilot-dx 0.6.21 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -323
- package/bin/cli.mjs +26 -130
- package/package.json +2 -8
- package/scripts/benchmark.mjs +0 -114
- package/src/adapter.mjs +22 -107
- package/src/auth-profile.mjs +18 -130
- package/src/cli-animation.mjs +108 -0
- package/src/cli-auth.mjs +44 -398
- package/src/cli-models.mjs +21 -49
- package/src/cli-options.mjs +53 -75
- package/src/cli-status.mjs +6 -23
- package/src/config.mjs +2 -8
- package/src/copilot.mjs +2 -2
- package/src/doctor.mjs +44 -291
- package/src/model-cache.mjs +4 -11
- package/src/models.mjs +0 -184
- package/src/observability.mjs +1 -38
- package/src/profile-model-runtime.mjs +61 -189
- package/src/profile-routing.mjs +2 -102
- package/src/profile-runtime.mjs +15 -68
- package/src/responses-handler.mjs +1 -1
- package/src/responses-proxy.mjs +1 -1
- package/src/running-adapter.mjs +2 -4
- package/src/stream-errors.mjs +3 -6
- package/src/stream-performance.mjs +1 -1
- package/src/terminal-activity.mjs +40 -34
- package/src/terminal-animation.mjs +249 -0
- package/src/usage.mjs +0 -31
- package/src/user-settings.mjs +50 -0
- package/src/anthropic-handler.mjs +0 -195
- package/src/anthropic.mjs +0 -387
- package/src/claude-config.mjs +0 -57
- package/src/claude-desktop-config.mjs +0 -317
- package/src/cli-error.mjs +0 -50
- package/src/pm-studio-asar.mjs +0 -1244
- package/src/pm-studio-bundle-content.mjs +0 -165
- package/src/pm-studio-relay.mjs +0 -563
- package/src/pm-studio-setup-lock.mjs +0 -336
- package/src/pm-studio-setup.mjs +0 -1581
- package/src/pm-studio-status.mjs +0 -369
- package/src/token-counter-service.mjs +0 -211
- package/src/token-counter-worker.mjs +0 -16
package/bin/cli.mjs
CHANGED
|
@@ -2,11 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import { ensureAuth, openCodex } from "../src/launcher.mjs";
|
|
4
4
|
import { ensureCodexConfig } from "../src/config.mjs";
|
|
5
|
-
import { ensureClaudeConfig } from "../src/claude-config.mjs";
|
|
6
|
-
import { applyClaudeDesktopConfig, formatClaudeDesktopApplyResult, generatedClaudeDesktopApiKey, loadManagedClaudeDesktopApiKey, syncManagedClaudeDesktopModels } from "../src/claude-desktop-config.mjs";
|
|
7
5
|
import { startAdapter } from "../src/adapter.mjs";
|
|
8
6
|
import { defaultCopilotClient, refreshVSCodeVersion } from "../src/copilot.mjs";
|
|
9
|
-
import { claudeDesktopModelIds } from "../src/models.mjs";
|
|
10
7
|
import { status } from "../src/status.mjs";
|
|
11
8
|
import { configureLogging } from "../src/log.mjs";
|
|
12
9
|
import { flushUsageWrites, printUsageSummary } from "../src/usage.mjs";
|
|
@@ -17,10 +14,9 @@ import { assertSafeAdapterHost, isLoopbackHost } from "../src/security.mjs";
|
|
|
17
14
|
import { runInBackground } from "../src/startup.mjs";
|
|
18
15
|
import { cliCommandName, cliHelp, parseAdapterProbeOptions, parseCliArgs, parseRuntimeOptions } from "../src/cli-options.mjs";
|
|
19
16
|
import { formatAdapterStatus, readAdapterStatus } from "../src/cli-status.mjs";
|
|
20
|
-
import { formatCliError } from "../src/cli-error.mjs";
|
|
21
17
|
import { closeHttpServer } from "../src/shutdown.mjs";
|
|
22
18
|
import { runAutoReviewModelCommand } from "../src/auto-review-model.mjs";
|
|
23
|
-
import { autoReviewModelPreference } from "../src/user-settings.mjs";
|
|
19
|
+
import { autoReviewModelPreference, terminalAnimationPreference } from "../src/user-settings.mjs";
|
|
24
20
|
import { fetchLiveCopilotModels, formatLiveCopilotModels } from "../src/cli-models.mjs";
|
|
25
21
|
import { runAuthCommand } from "../src/cli-auth.mjs";
|
|
26
22
|
import { createProfileRuntime } from "../src/profile-runtime.mjs";
|
|
@@ -47,6 +43,16 @@ if (CLI.command === "version") {
|
|
|
47
43
|
console.log(CLI_BANNER);
|
|
48
44
|
process.exit(0);
|
|
49
45
|
}
|
|
46
|
+
if (CLI.command === "animation") {
|
|
47
|
+
try {
|
|
48
|
+
const { runAnimationCommand } = await import("../src/cli-animation.mjs");
|
|
49
|
+
await runAnimationCommand({ commandName: CLI_NAME });
|
|
50
|
+
process.exit(0);
|
|
51
|
+
} catch (e) {
|
|
52
|
+
console.error(status("err", e.message));
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
50
56
|
if (CLI.command === "usage") {
|
|
51
57
|
await printUsageSummary({
|
|
52
58
|
format: CLI.outputFormat || "auto",
|
|
@@ -54,6 +60,15 @@ if (CLI.command === "usage") {
|
|
|
54
60
|
});
|
|
55
61
|
process.exit(0);
|
|
56
62
|
}
|
|
63
|
+
if (CLI.command === "retired") {
|
|
64
|
+
console.error(status("err", `${CLI.integration} integration was retired in ccdx 0.7.0.`));
|
|
65
|
+
if (CLI.integration === "PM Studio") {
|
|
66
|
+
console.error(status("info", "Update or reinstall PM Studio to remove an older CCDX app patch."));
|
|
67
|
+
} else {
|
|
68
|
+
console.error(status("info", "Remove any old CCDX endpoint from the client before using a different provider."));
|
|
69
|
+
}
|
|
70
|
+
process.exit(1);
|
|
71
|
+
}
|
|
57
72
|
if (CLI.command === "auth") {
|
|
58
73
|
try {
|
|
59
74
|
const result = await runAuthCommand({
|
|
@@ -67,12 +82,6 @@ if (CLI.command === "auth") {
|
|
|
67
82
|
output: process.stdout,
|
|
68
83
|
});
|
|
69
84
|
if (result.output) console.log(result.output);
|
|
70
|
-
if (result.action === "login") {
|
|
71
|
-
if (!result.changed) {
|
|
72
|
-
console.log(status("ok", `Claude profile is already configured for ${result.identity?.login || result.identity?.id || "the saved account"}`));
|
|
73
|
-
}
|
|
74
|
-
console.log(status("warn", `Restart the running ${CLI_NAME} adapter to activate the Claude profile`));
|
|
75
|
-
}
|
|
76
85
|
process.exit(0);
|
|
77
86
|
} catch (e) {
|
|
78
87
|
console.error(status("err", e.message));
|
|
@@ -133,31 +142,6 @@ if (CLI.command === "update") {
|
|
|
133
142
|
process.exit(1);
|
|
134
143
|
}
|
|
135
144
|
}
|
|
136
|
-
if (CLI.command === "pms") {
|
|
137
|
-
try {
|
|
138
|
-
if (CLI.action === "status") {
|
|
139
|
-
const { runPmStudioStatus } = await import("../src/pm-studio-status.mjs");
|
|
140
|
-
const result = await runPmStudioStatus({
|
|
141
|
-
commandName: CLI_NAME,
|
|
142
|
-
format: CLI.outputFormat || "auto",
|
|
143
|
-
output: process.stdout,
|
|
144
|
-
});
|
|
145
|
-
process.exit(result.ok ? 0 : 1);
|
|
146
|
-
}
|
|
147
|
-
if (CLI.action === "restore") {
|
|
148
|
-
const { runPmStudioRestore } = await import("../src/pm-studio-setup.mjs");
|
|
149
|
-
await runPmStudioRestore({ commandName: CLI_NAME });
|
|
150
|
-
process.exit(0);
|
|
151
|
-
}
|
|
152
|
-
const { runPmStudioSetup } = await import("../src/pm-studio-setup.mjs");
|
|
153
|
-
await runPmStudioSetup({ commandName: CLI_NAME });
|
|
154
|
-
process.exit(0);
|
|
155
|
-
} catch (e) {
|
|
156
|
-
console.error(formatCliError(e));
|
|
157
|
-
process.exit(1);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
145
|
let RUNTIME;
|
|
162
146
|
try {
|
|
163
147
|
RUNTIME = parseRuntimeOptions(process.env);
|
|
@@ -170,7 +154,6 @@ const ADAPTER_PORT = RUNTIME.adapterPort;
|
|
|
170
154
|
const ADAPTER_HOST = RUNTIME.adapterHost;
|
|
171
155
|
const MODEL_REFRESH_TIMEOUT_MS = RUNTIME.modelRefreshTimeoutMs;
|
|
172
156
|
const EXISTING_ADAPTER_TIMEOUT_MS = RUNTIME.existingAdapterTimeoutMs;
|
|
173
|
-
const CONFIGURE_CLAUDE_DESKTOP = CLI.configureClaudeDesktop || process.env.CCDX_CONFIGURE_CLAUDE_DESKTOP === "1";
|
|
174
157
|
const LOGGING = configureLogging();
|
|
175
158
|
let activeServer = null;
|
|
176
159
|
let modelRefreshTimer = null;
|
|
@@ -181,24 +164,6 @@ if (LOGGING.filePath) {
|
|
|
181
164
|
if (LOGGING.level === "debug") console.log(status("debug", "Debug logging enabled"));
|
|
182
165
|
}
|
|
183
166
|
|
|
184
|
-
function currentClaudeDesktopApiKey() {
|
|
185
|
-
return String(process.env.CCDX_CLAUDE_DESKTOP_API_KEY || process.env.CCDX_PROXY_API_KEY || "").trim();
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
function syncClaudeDesktopProfileModels(modelDefs) {
|
|
189
|
-
const result = syncManagedClaudeDesktopModels({
|
|
190
|
-
port: ADAPTER_PORT,
|
|
191
|
-
host: ADAPTER_HOST,
|
|
192
|
-
modelIds: claudeDesktopModelIds(process.env, { modelDefs }),
|
|
193
|
-
});
|
|
194
|
-
if (result.updated) {
|
|
195
|
-
console.log(status("ok", `Updated Claude App profile models: ${result.modelIds.join(", ")}`));
|
|
196
|
-
} else if (result.error) {
|
|
197
|
-
console.log(status("warn", `Could not update Claude App profile models (${result.error.message})`));
|
|
198
|
-
}
|
|
199
|
-
return result;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
167
|
async function reuseRunningAdapterIfAvailable() {
|
|
203
168
|
const running = await checkRunningAdapter({
|
|
204
169
|
port: ADAPTER_PORT,
|
|
@@ -213,25 +178,6 @@ async function reuseRunningAdapterIfAvailable() {
|
|
|
213
178
|
|
|
214
179
|
console.log(status("ok", `Using existing adapter at ${running.baseUrl}`));
|
|
215
180
|
ensureCodexConfig(ADAPTER_PORT, { host: ADAPTER_HOST });
|
|
216
|
-
ensureClaudeConfig(ADAPTER_PORT, { host: ADAPTER_HOST });
|
|
217
|
-
|
|
218
|
-
if (CONFIGURE_CLAUDE_DESKTOP) {
|
|
219
|
-
const claudeDesktopApiKey = currentClaudeDesktopApiKey();
|
|
220
|
-
if (claudeDesktopApiKey) {
|
|
221
|
-
const result = applyClaudeDesktopConfig({
|
|
222
|
-
port: ADAPTER_PORT,
|
|
223
|
-
host: ADAPTER_HOST,
|
|
224
|
-
gatewayApiKey: claudeDesktopApiKey,
|
|
225
|
-
modelIds: claudeDesktopModelIds(process.env),
|
|
226
|
-
});
|
|
227
|
-
console.log(status("ok", `Configured Claude App gateway profile at ${result.baseUrl}`));
|
|
228
|
-
console.log(formatClaudeDesktopApplyResult(result));
|
|
229
|
-
} else {
|
|
230
|
-
console.log(status("warn", "Existing adapter is running; skip Claude App profile update unless CCDX_CLAUDE_DESKTOP_API_KEY or CCDX_PROXY_API_KEY is set"));
|
|
231
|
-
}
|
|
232
|
-
} else {
|
|
233
|
-
console.log(status("ok", "Claude App support available with --configure-claude-app"));
|
|
234
|
-
}
|
|
235
181
|
|
|
236
182
|
await openCodex();
|
|
237
183
|
console.log(`
|
|
@@ -249,14 +195,13 @@ if (CLI.command === "doctor") {
|
|
|
249
195
|
host: ADAPTER_HOST,
|
|
250
196
|
online: CLI.online,
|
|
251
197
|
compat: CLI.compat,
|
|
252
|
-
profile: CLI.profile,
|
|
253
198
|
});
|
|
254
199
|
process.exit(checks.some((check) => check.kind === "err") ? 1 : 0);
|
|
255
200
|
}
|
|
256
201
|
|
|
257
202
|
console.log(`
|
|
258
203
|
${CLI_BANNER}
|
|
259
|
-
Use Codex Desktop
|
|
204
|
+
Use Codex Desktop and ChatGPT App with GitHub Copilot
|
|
260
205
|
`);
|
|
261
206
|
|
|
262
207
|
async function printUpdateNotice() {
|
|
@@ -279,84 +224,37 @@ try {
|
|
|
279
224
|
await ensureAuth();
|
|
280
225
|
|
|
281
226
|
const profileRuntime = createProfileRuntime({ codexClient: defaultCopilotClient });
|
|
282
|
-
if (profileRuntime.claudeMode === "isolated") {
|
|
283
|
-
if (profileRuntime.claudeProfile.valid) {
|
|
284
|
-
const account = profileRuntime.claudeProfile.identity?.login
|
|
285
|
-
|| profileRuntime.claudeProfile.identity?.id
|
|
286
|
-
|| "the saved account";
|
|
287
|
-
console.log(status("ok", `Claude requests will use the isolated GitHub account ${account}`));
|
|
288
|
-
} else {
|
|
289
|
-
console.log(status("warn", `Claude profile is configured but invalid (${profileRuntime.claudeProfile.reason}); Codex remains available, but Claude requests will fail until you run ${CLI_NAME} auth login claude --reauth and restart`));
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
|
|
293
227
|
const profileModels = createProfileModelRuntime({
|
|
294
228
|
codexClient: profileRuntime.codexClient,
|
|
295
|
-
claudeClient: profileRuntime.claudeClient,
|
|
296
|
-
claudeMode: profileRuntime.claudeMode,
|
|
297
229
|
codexCredentialFingerprint: profileRuntime.codexCredentialFingerprint,
|
|
298
|
-
claudeCredentialFingerprint: profileRuntime.claudeCredentialFingerprint,
|
|
299
230
|
timeoutMs: MODEL_REFRESH_TIMEOUT_MS,
|
|
300
231
|
commandName: CLI_NAME,
|
|
301
232
|
autoReviewModelResolver: () => autoReviewModelPreference().model,
|
|
302
|
-
onClaudeModelsChanged: syncClaudeDesktopProfileModels,
|
|
303
233
|
});
|
|
304
234
|
|
|
305
235
|
// Refresh the VS Code version in the background; fallback is non-blocking.
|
|
306
236
|
void refreshVSCodeVersion();
|
|
307
|
-
|
|
308
|
-
const claudeDesktopModelDefs = modelInitialization.claude.modelDefs;
|
|
309
|
-
syncClaudeDesktopProfileModels(claudeDesktopModelDefs);
|
|
237
|
+
await profileModels.initialize();
|
|
310
238
|
|
|
311
239
|
if (!isLoopbackHost(ADAPTER_HOST)) {
|
|
312
240
|
console.log(status("warn", `ADAPTER_HOST=${ADAPTER_HOST} exposes the adapter beyond loopback because CCDX_ALLOW_LAN=1 is set. Use only on trusted networks.`));
|
|
313
241
|
}
|
|
314
242
|
|
|
315
|
-
const configuredClaudeDesktopApiKey = currentClaudeDesktopApiKey();
|
|
316
|
-
const restoredClaudeDesktopApiKey = configuredClaudeDesktopApiKey ? "" : loadManagedClaudeDesktopApiKey({
|
|
317
|
-
port: ADAPTER_PORT,
|
|
318
|
-
host: ADAPTER_HOST,
|
|
319
|
-
});
|
|
320
|
-
if (restoredClaudeDesktopApiKey) {
|
|
321
|
-
console.log(status("ok", "Restored Claude App gateway key from the managed profile"));
|
|
322
|
-
}
|
|
323
|
-
const claudeDesktopApiKey = configuredClaudeDesktopApiKey
|
|
324
|
-
|| restoredClaudeDesktopApiKey
|
|
325
|
-
|| (CONFIGURE_CLAUDE_DESKTOP ? generatedClaudeDesktopApiKey() : "");
|
|
326
|
-
|
|
327
243
|
// Start the in-process adapter.
|
|
244
|
+
const terminalAnimationTheme = terminalAnimationPreference().theme;
|
|
328
245
|
activeServer = await startAdapter(ADAPTER_PORT, ADAPTER_HOST, {
|
|
329
|
-
claudeDesktopApiKey,
|
|
330
|
-
claudeDesktopModelDefs,
|
|
331
246
|
autoReviewModelResolver: () => autoReviewModelPreference().model,
|
|
332
247
|
codexClient: profileRuntime.codexClient,
|
|
333
|
-
claudeClient: profileRuntime.claudeClient,
|
|
334
|
-
claudeProfile: profileRuntime.claudeProfile,
|
|
335
248
|
codexModelRegistry: profileModels.codexRegistry,
|
|
336
|
-
claudeModelRegistry: profileModels.claudeRegistry,
|
|
337
|
-
claudeMode: profileRuntime.claudeMode,
|
|
338
|
-
isClaudeProfileCurrent: profileRuntime.isClaudeProfileCurrent,
|
|
339
249
|
showRequestId: CLI.showRequestId,
|
|
340
250
|
upstreamTimeoutMs: RUNTIME.upstreamTimeoutMs,
|
|
341
251
|
streamHandshakeTimeoutMs: RUNTIME.streamHandshakeTimeoutMs,
|
|
342
252
|
streamIdleTimeoutMs: RUNTIME.streamIdleTimeoutMs,
|
|
253
|
+
terminalAnimationTheme,
|
|
343
254
|
});
|
|
344
255
|
|
|
345
|
-
// Point Codex and
|
|
256
|
+
// Point Codex and compatible ChatGPT App builds at the adapter.
|
|
346
257
|
ensureCodexConfig(ADAPTER_PORT, { host: ADAPTER_HOST });
|
|
347
|
-
ensureClaudeConfig(ADAPTER_PORT, { host: ADAPTER_HOST });
|
|
348
|
-
if (CONFIGURE_CLAUDE_DESKTOP) {
|
|
349
|
-
const result = applyClaudeDesktopConfig({
|
|
350
|
-
port: ADAPTER_PORT,
|
|
351
|
-
host: ADAPTER_HOST,
|
|
352
|
-
gatewayApiKey: claudeDesktopApiKey,
|
|
353
|
-
modelIds: claudeDesktopModelIds(process.env, { modelDefs: claudeDesktopModelDefs }),
|
|
354
|
-
});
|
|
355
|
-
console.log(status("ok", `Configured Claude App gateway profile at ${result.baseUrl}`));
|
|
356
|
-
console.log(formatClaudeDesktopApplyResult(result));
|
|
357
|
-
} else {
|
|
358
|
-
console.log(status("ok", "Claude App support available with --configure-claude-app"));
|
|
359
|
-
}
|
|
360
258
|
|
|
361
259
|
// Launch Codex when available.
|
|
362
260
|
await openCodex();
|
|
@@ -371,10 +269,8 @@ try {
|
|
|
371
269
|
modelRefreshTimer.unref?.();
|
|
372
270
|
}
|
|
373
271
|
|
|
374
|
-
const readyClients = ["Codex App", "Claude Code"];
|
|
375
|
-
if (claudeDesktopApiKey) readyClients.splice(1, 0, "Claude App");
|
|
376
272
|
console.log(`
|
|
377
|
-
${status("ok",
|
|
273
|
+
${status("ok", "Ready, Codex App and ChatGPT App are ready to use")}
|
|
378
274
|
|
|
379
275
|
Adapter: ${adapterBaseUrl(ADAPTER_HOST, ADAPTER_PORT)}
|
|
380
276
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codex-copilot-dx",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Use Codex Desktop
|
|
3
|
+
"version": "0.7.1",
|
|
4
|
+
"description": "Use Codex Desktop and ChatGPT App with GitHub Copilot",
|
|
5
5
|
"author": "Dale Xiao",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -18,11 +18,6 @@
|
|
|
18
18
|
"codex",
|
|
19
19
|
"copilot",
|
|
20
20
|
"github",
|
|
21
|
-
"claude",
|
|
22
|
-
"claude-app",
|
|
23
|
-
"claude-code",
|
|
24
|
-
"claude-desktop",
|
|
25
|
-
"anthropic",
|
|
26
21
|
"openai",
|
|
27
22
|
"proxy",
|
|
28
23
|
"responses-api"
|
|
@@ -51,7 +46,6 @@
|
|
|
51
46
|
"prepublishOnly": "npm run verify"
|
|
52
47
|
},
|
|
53
48
|
"dependencies": {
|
|
54
|
-
"gpt-tokenizer": "^3.4.0",
|
|
55
49
|
"sharp": "^0.35.3"
|
|
56
50
|
}
|
|
57
51
|
}
|
package/scripts/benchmark.mjs
CHANGED
|
@@ -8,7 +8,6 @@ import { setTimeout as delay } from "node:timers/promises";
|
|
|
8
8
|
import { fileURLToPath } from "node:url";
|
|
9
9
|
import sharp from "sharp";
|
|
10
10
|
import { optimizeImagesInBody, prepareResponsesPayload } from "../src/image-optimization.mjs";
|
|
11
|
-
import { createPmStudioModelRouter } from "../src/profile-routing.mjs";
|
|
12
11
|
|
|
13
12
|
const packageRoot = fileURLToPath(new URL("../", import.meta.url));
|
|
14
13
|
|
|
@@ -23,14 +22,6 @@ function runProbe(source) {
|
|
|
23
22
|
return JSON.parse(result.stdout.trim());
|
|
24
23
|
}
|
|
25
24
|
|
|
26
|
-
function median(values) {
|
|
27
|
-
const ordered = [...values].sort((left, right) => left - right);
|
|
28
|
-
const middle = Math.floor(ordered.length / 2);
|
|
29
|
-
return ordered.length % 2 === 0
|
|
30
|
-
? (ordered[middle - 1] + ordered[middle]) / 2
|
|
31
|
-
: ordered[middle];
|
|
32
|
-
}
|
|
33
|
-
|
|
34
25
|
function sseLinearityProbe() {
|
|
35
26
|
return runProbe(`
|
|
36
27
|
import { webStreamLines } from "./src/stream.mjs";
|
|
@@ -220,38 +211,6 @@ function largePayloadProbe(targetMiB, concurrency) {
|
|
|
220
211
|
}
|
|
221
212
|
}
|
|
222
213
|
|
|
223
|
-
function largeTokenCountProbe(mode) {
|
|
224
|
-
const samples = Array.from({ length: 3 }, () => runProbe(`
|
|
225
|
-
const text = "a ".repeat(2 * 1024 * 1024);
|
|
226
|
-
const started = performance.now();
|
|
227
|
-
let tokens;
|
|
228
|
-
if (${JSON.stringify(mode)} === "proxy") {
|
|
229
|
-
const { countTokens } = await import("./src/anthropic.mjs");
|
|
230
|
-
tokens = (await countTokens({ messages: [{ role: "user", content: text }] })).input_tokens;
|
|
231
|
-
} else {
|
|
232
|
-
const { encode } = await import("gpt-tokenizer");
|
|
233
|
-
const encoded = encode(text);
|
|
234
|
-
tokens = encoded.length;
|
|
235
|
-
}
|
|
236
|
-
const memory = process.memoryUsage();
|
|
237
|
-
process.stdout.write(JSON.stringify({
|
|
238
|
-
input_mib: +(Buffer.byteLength(text) / 1048576).toFixed(1),
|
|
239
|
-
tokens,
|
|
240
|
-
elapsed_ms: +(performance.now() - started).toFixed(1),
|
|
241
|
-
rss_mib: +(memory.rss / 1048576).toFixed(1),
|
|
242
|
-
heap_used_mib: +(memory.heapUsed / 1048576).toFixed(1),
|
|
243
|
-
}));
|
|
244
|
-
`));
|
|
245
|
-
return {
|
|
246
|
-
input_mib: samples[0].input_mib,
|
|
247
|
-
tokens: samples[0].tokens,
|
|
248
|
-
elapsed_ms: +median(samples.map((sample) => sample.elapsed_ms)).toFixed(1),
|
|
249
|
-
rss_mib: +median(samples.map((sample) => sample.rss_mib)).toFixed(1),
|
|
250
|
-
heap_used_mib: +median(samples.map((sample) => sample.heap_used_mib)).toFixed(1),
|
|
251
|
-
samples: samples.length,
|
|
252
|
-
};
|
|
253
|
-
}
|
|
254
|
-
|
|
255
214
|
function toolOutputParseCacheProbe() {
|
|
256
215
|
return runProbe(`
|
|
257
216
|
import { enforceResponsesImageLimit } from "./src/responses-image-limit.mjs";
|
|
@@ -355,29 +314,6 @@ const adapterImport = runProbe(`
|
|
|
355
314
|
}));
|
|
356
315
|
`);
|
|
357
316
|
|
|
358
|
-
const tokenCount = runProbe(`
|
|
359
|
-
const importStarted = performance.now();
|
|
360
|
-
const { countTokens } = await import("./src/anthropic.mjs");
|
|
361
|
-
const importMs = performance.now() - importStarted;
|
|
362
|
-
const body = { messages: [{ role: "user", content: "a ".repeat(50000) }] };
|
|
363
|
-
const firstStarted = performance.now();
|
|
364
|
-
const first = await countTokens(body);
|
|
365
|
-
const firstMs = performance.now() - firstStarted;
|
|
366
|
-
const warmStarted = performance.now();
|
|
367
|
-
const warm = await countTokens(body);
|
|
368
|
-
const warmMs = performance.now() - warmStarted;
|
|
369
|
-
globalThis.gc?.();
|
|
370
|
-
const memory = process.memoryUsage();
|
|
371
|
-
console.log(JSON.stringify({
|
|
372
|
-
module_import_ms: +importMs.toFixed(1),
|
|
373
|
-
first_call_ms: +firstMs.toFixed(1),
|
|
374
|
-
warm_call_ms: +warmMs.toFixed(1),
|
|
375
|
-
input_tokens: first.input_tokens,
|
|
376
|
-
deterministic: first.input_tokens === warm.input_tokens,
|
|
377
|
-
rss_mib: +(memory.rss / 1048576).toFixed(1),
|
|
378
|
-
}));
|
|
379
|
-
`);
|
|
380
|
-
|
|
381
317
|
const repeatedImage = "data:image/png;base64,QUJDRA==";
|
|
382
318
|
const duplicateBody = {
|
|
383
319
|
input: [{
|
|
@@ -402,34 +338,6 @@ const duplicateImages = {
|
|
|
402
338
|
optimizer_calls: duplicateCalls,
|
|
403
339
|
};
|
|
404
340
|
|
|
405
|
-
const routingCatalog = {
|
|
406
|
-
data: Array.from({ length: 1000 }, (_, index) => index % 5 === 0
|
|
407
|
-
? {
|
|
408
|
-
id: `claude-benchmark-${index}`,
|
|
409
|
-
vendor: "Anthropic",
|
|
410
|
-
model_picker_enabled: true,
|
|
411
|
-
supported_endpoints: ["/chat/completions"],
|
|
412
|
-
}
|
|
413
|
-
: { id: `gpt-benchmark-${index}`, vendor: "OpenAI", supported_endpoints: ["/responses"] }),
|
|
414
|
-
};
|
|
415
|
-
const modelRouter = createPmStudioModelRouter({
|
|
416
|
-
getCatalog: () => routingCatalog,
|
|
417
|
-
isClaudeEnabled: () => true,
|
|
418
|
-
});
|
|
419
|
-
const routingIterations = 100000;
|
|
420
|
-
const routingStarted = performance.now();
|
|
421
|
-
let claudeClassifications = 0;
|
|
422
|
-
for (let index = 0; index < routingIterations; index += 1) {
|
|
423
|
-
if (modelRouter.classify("claude-benchmark-0") === "claude") claudeClassifications += 1;
|
|
424
|
-
}
|
|
425
|
-
const pmModelRouting = {
|
|
426
|
-
catalog_models: routingCatalog.data.length,
|
|
427
|
-
iterations: routingIterations,
|
|
428
|
-
elapsed_ms: +(performance.now() - routingStarted).toFixed(1),
|
|
429
|
-
claude_classifications: claudeClassifications,
|
|
430
|
-
availability_rebuilds: modelRouter.diagnostics().rebuilds,
|
|
431
|
-
};
|
|
432
|
-
|
|
433
341
|
const pixels = deterministicPixels(1024 * 1024 * 3, 0x87654321);
|
|
434
342
|
const webp = await sharp(pixels, { raw: { width: 1024, height: 1024, channels: 3 } })
|
|
435
343
|
.webp({ quality: 100 })
|
|
@@ -474,18 +382,12 @@ const report = {
|
|
|
474
382
|
? "Relative performance and resource checks; absolute timings vary by machine."
|
|
475
383
|
: "Report-only benchmark; timings vary by machine and are not pass/fail thresholds.",
|
|
476
384
|
adapter_import: adapterImport,
|
|
477
|
-
token_count_100kb: tokenCount,
|
|
478
385
|
duplicate_images: duplicateImages,
|
|
479
|
-
pm_model_routing: pmModelRouting,
|
|
480
386
|
oversized_payload: oversizedPayload,
|
|
481
387
|
};
|
|
482
388
|
|
|
483
389
|
if (checkMode) {
|
|
484
390
|
report.sse_linearity = sseLinearityProbe();
|
|
485
|
-
report.large_token_count = {
|
|
486
|
-
proxy: largeTokenCountProbe("proxy"),
|
|
487
|
-
materialized_array: largeTokenCountProbe("array"),
|
|
488
|
-
};
|
|
489
391
|
report.tool_output_parse_cache = toolOutputParseCacheProbe();
|
|
490
392
|
report.large_payload_peak = [
|
|
491
393
|
largePayloadProbe(5, 4),
|
|
@@ -507,22 +409,6 @@ console.log(JSON.stringify(report, null, 2));
|
|
|
507
409
|
|
|
508
410
|
if (checkMode) {
|
|
509
411
|
const failures = [];
|
|
510
|
-
if (report.pm_model_routing.availability_rebuilds !== 1
|
|
511
|
-
|| report.pm_model_routing.claude_classifications !== report.pm_model_routing.iterations) {
|
|
512
|
-
failures.push("PM Studio model routing did not reuse the memoized catalog classification");
|
|
513
|
-
}
|
|
514
|
-
const proxy = report.large_token_count.proxy;
|
|
515
|
-
const array = report.large_token_count.materialized_array;
|
|
516
|
-
if (proxy.tokens !== array.tokens) failures.push(`token counts differ: proxy=${proxy.tokens} array=${array.tokens}`);
|
|
517
|
-
if (proxy.elapsed_ms > array.elapsed_ms * 1.25) {
|
|
518
|
-
failures.push(`token counter is materially slower: proxy=${proxy.elapsed_ms}ms array=${array.elapsed_ms}ms`);
|
|
519
|
-
}
|
|
520
|
-
if (proxy.rss_mib > array.rss_mib * 0.9) {
|
|
521
|
-
failures.push(`token counter RSS regression: proxy=${proxy.rss_mib}MiB array=${array.rss_mib}MiB`);
|
|
522
|
-
}
|
|
523
|
-
if (proxy.heap_used_mib > array.heap_used_mib * 0.75) {
|
|
524
|
-
failures.push(`token counter heap regression: proxy=${proxy.heap_used_mib}MiB array=${array.heap_used_mib}MiB`);
|
|
525
|
-
}
|
|
526
412
|
if (report.tool_output_parse_cache.parse_calls !== 1) {
|
|
527
413
|
failures.push(`stringified tool output parsed ${report.tool_output_parse_cache.parse_calls} times instead of once`);
|
|
528
414
|
}
|