@tiens.nguyen/gonext-cli 1.0.350
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -0
- package/coding-key-prompt.mjs +47 -0
- package/daemon-control.mjs +232 -0
- package/device-login.mjs +141 -0
- package/gonext-cli.mjs +3216 -0
- package/gonext-repl.mjs +2648 -0
- package/gonext_agent_chat.py +7576 -0
- package/gonext_mlx_embed.py +155 -0
- package/gonext_probe_agent.py +93 -0
- package/gonext_transcribe.py +130 -0
- package/model-doctor.mjs +571 -0
- package/package.json +53 -0
- package/thinking_words.txt +1003 -0
package/gonext-cli.mjs
ADDED
|
@@ -0,0 +1,3216 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* GoNext local worker:
|
|
4
|
+
* - `gonext-cli set <workerKey> [--api-base URL]`
|
|
5
|
+
* writes ~/.gonext/worker.env
|
|
6
|
+
* - `gonext-cli ws-ping-test` — POST worker ws-ping (needs GONEXT_* env)
|
|
7
|
+
* - `gonext-cli simulate-chat [text]` — claim next chat job, push fake reply like the real worker (needs GONEXT_* env)
|
|
8
|
+
* - `gonext-cli embed --model <mlx-embed-model> [--port 8085]` — run the MLX embeddings server (RAG); use a separate terminal
|
|
9
|
+
* - `gonext-cli workspace add|remove|list …` — manage local code folders the agent may read/edit/test in
|
|
10
|
+
* - `gonext-cli` — starts polling loop (claims jobs and runs models)
|
|
11
|
+
*/
|
|
12
|
+
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
|
13
|
+
import { execFile as execFileCallback, spawn } from "node:child_process";
|
|
14
|
+
import { createHash } from "node:crypto";
|
|
15
|
+
import { homedir, platform, tmpdir } from "node:os";
|
|
16
|
+
import { dirname, extname, join } from "node:path";
|
|
17
|
+
import { fileURLToPath } from "node:url";
|
|
18
|
+
import { promisify } from "node:util";
|
|
19
|
+
import { createRequire } from "node:module";
|
|
20
|
+
import dotenv from "dotenv";
|
|
21
|
+
import OpenAI from "openai";
|
|
22
|
+
|
|
23
|
+
const WORKER_DIR = dirname(fileURLToPath(import.meta.url));
|
|
24
|
+
|
|
25
|
+
// Running worker version, read from the package's own package.json (single source of
|
|
26
|
+
// truth) so the startup banner shows exactly which build a box is running (task #105).
|
|
27
|
+
const WORKER_VERSION = (() => {
|
|
28
|
+
try {
|
|
29
|
+
return createRequire(import.meta.url)("./package.json").version ?? "unknown";
|
|
30
|
+
} catch {
|
|
31
|
+
return "unknown";
|
|
32
|
+
}
|
|
33
|
+
})();
|
|
34
|
+
|
|
35
|
+
/** Avoid `node:child_process/promises` — not available on some Node builds / older runtimes. */
|
|
36
|
+
const execFile = promisify(execFileCallback);
|
|
37
|
+
|
|
38
|
+
const ENV_FILE = join(homedir(), ".gonext", "worker.env");
|
|
39
|
+
const preloadedWorkerKey = String(process.env.GONEXT_WORKER_KEY ?? "").trim();
|
|
40
|
+
const envFileLoad = dotenv.config({ path: ENV_FILE });
|
|
41
|
+
dotenv.config();
|
|
42
|
+
|
|
43
|
+
const args = process.argv.slice(2);
|
|
44
|
+
|
|
45
|
+
function workerKeyFingerprint(secret) {
|
|
46
|
+
const v = String(secret ?? "").trim();
|
|
47
|
+
if (!v) return "";
|
|
48
|
+
return createHash("sha256").update(v, "utf8").digest("hex").slice(0, 12);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function maskWorkerKey(secret) {
|
|
52
|
+
const v = String(secret ?? "").trim();
|
|
53
|
+
if (!v) return "(empty)";
|
|
54
|
+
if (v.length <= 12) return `${v.slice(0, 3)}***`;
|
|
55
|
+
return `${v.slice(0, 8)}...${v.slice(-4)}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function printHelp() {
|
|
59
|
+
console.log(`
|
|
60
|
+
gonext-cli
|
|
61
|
+
|
|
62
|
+
Usage:
|
|
63
|
+
gonext-cli
|
|
64
|
+
gonext-cli login [--api-base <url>] # browser sign-in → writes ~/.gonext/worker.env
|
|
65
|
+
gonext-cli set <workerKey> [--api-base <url>]
|
|
66
|
+
gonext-cli start | stop | status | restart # run the polling daemon in the background
|
|
67
|
+
gonext-cli logs [--lines N] # tail the background daemon log
|
|
68
|
+
gonext-cli doctor # download + start the base chat model if needed
|
|
69
|
+
gonext-cli ws-ping-test # POST /api/worker/ws-ping-test (needs GONEXT_* env)
|
|
70
|
+
gonext-cli simulate-chat [text...] # claim next chat job, fake reply (stop normal worker first)
|
|
71
|
+
|
|
72
|
+
Examples:
|
|
73
|
+
gonext-cli set abc123 --api-base https://hwohu56e8d.execute-api.ap-southeast-1.amazonaws.com
|
|
74
|
+
gonext-cli
|
|
75
|
+
gonext-cli ws-ping-test
|
|
76
|
+
gonext-cli simulate-chat "Hello from a fake model"
|
|
77
|
+
|
|
78
|
+
Env (optional):
|
|
79
|
+
GONEXT_SIMULATE_TEXT default body for simulate-chat when no args
|
|
80
|
+
GONEXT_MLX_LM_PYTHON Python executable for MLX LM native probe (default: python3)
|
|
81
|
+
GONEXT_OCR_MODEL_PATH Local GLM-OCR-bf16 directory (default: ~/mlx-models/GLM-OCR-bf16)
|
|
82
|
+
GONEXT_OCR_DEBUG=1 Print raw OCR stdout/stderr snippets for troubleshooting
|
|
83
|
+
`);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function parseFlag(name) {
|
|
87
|
+
const idx = args.indexOf(name);
|
|
88
|
+
if (idx >= 0 && args[idx + 1]) {
|
|
89
|
+
return args[idx + 1];
|
|
90
|
+
}
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function setConfig() {
|
|
95
|
+
const workerKey = args[1]?.trim();
|
|
96
|
+
if (!workerKey) {
|
|
97
|
+
console.error("Missing worker key. Usage: gonext-cli set <workerKey>");
|
|
98
|
+
process.exit(1);
|
|
99
|
+
}
|
|
100
|
+
const currentRaw = await readFile(ENV_FILE, "utf8").catch(() => "");
|
|
101
|
+
const current = dotenv.parse(currentRaw);
|
|
102
|
+
const apiBaseFromFlag = parseFlag("--api-base");
|
|
103
|
+
const next = {
|
|
104
|
+
GONEXT_API_BASE: (
|
|
105
|
+
apiBaseFromFlag ??
|
|
106
|
+
current.GONEXT_API_BASE ??
|
|
107
|
+
process.env.GONEXT_API_BASE ??
|
|
108
|
+
""
|
|
109
|
+
).replace(/\/+$/, ""),
|
|
110
|
+
GONEXT_WORKER_KEY: workerKey,
|
|
111
|
+
GONEXT_POLL_MS: "500",
|
|
112
|
+
};
|
|
113
|
+
await mkdir(join(homedir(), ".gonext"), { recursive: true });
|
|
114
|
+
await writeFile(
|
|
115
|
+
ENV_FILE,
|
|
116
|
+
`GONEXT_API_BASE=${next.GONEXT_API_BASE}\nGONEXT_WORKER_KEY=${next.GONEXT_WORKER_KEY}\nGONEXT_POLL_MS=${next.GONEXT_POLL_MS}\n`,
|
|
117
|
+
"utf8"
|
|
118
|
+
);
|
|
119
|
+
console.log(`Saved ${ENV_FILE}`);
|
|
120
|
+
if (!next.GONEXT_API_BASE) {
|
|
121
|
+
console.log("Tip: set API base too: gonext-cli set <workerKey> --api-base <https-url>");
|
|
122
|
+
}
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
127
|
+
printHelp();
|
|
128
|
+
process.exit(0);
|
|
129
|
+
}
|
|
130
|
+
if (args[0] === "set") {
|
|
131
|
+
await setConfig();
|
|
132
|
+
process.exit(0);
|
|
133
|
+
}
|
|
134
|
+
if (args[0] === "login") {
|
|
135
|
+
// Task #127, Phase 1: browser device-pairing → writes worker.env. `--api-base` overrides
|
|
136
|
+
// which API to pair against (defaults to the production API).
|
|
137
|
+
const { deviceLogin, DEFAULT_API_BASE } = await import("./device-login.mjs");
|
|
138
|
+
const apiBaseFlag = parseFlag("--api-base");
|
|
139
|
+
try {
|
|
140
|
+
await deviceLogin({
|
|
141
|
+
apiBase:
|
|
142
|
+
apiBaseFlag ||
|
|
143
|
+
(process.env.GONEXT_API_BASE ?? "").trim() ||
|
|
144
|
+
DEFAULT_API_BASE,
|
|
145
|
+
});
|
|
146
|
+
process.exit(0);
|
|
147
|
+
} catch (e) {
|
|
148
|
+
console.error(`[gonext-worker] login failed: ${e instanceof Error ? e.message : e}`);
|
|
149
|
+
if (e && e.code === "PAIRING_UNSUPPORTED") {
|
|
150
|
+
console.error(
|
|
151
|
+
"Set it manually instead: gonext-cli set <workerKey> --api-base <url>"
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
process.exit(1);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// Task #127, Phase 2: daemon lifecycle from a plain terminal (no GoTerminal). start runs
|
|
158
|
+
// the polling worker DETACHED; stop/status/logs manage it. All no-op-safe (BC4: never
|
|
159
|
+
// double-spawn a second poller).
|
|
160
|
+
if (["start", "stop", "status", "logs", "restart"].includes(args[0])) {
|
|
161
|
+
const ctl = await import("./daemon-control.mjs");
|
|
162
|
+
if (args[0] === "status") {
|
|
163
|
+
const s = await ctl.daemonStatus();
|
|
164
|
+
if (s.running) {
|
|
165
|
+
const who = s.pidFilePid ? `pid ${s.pidFilePid}` : `pid ${s.pids.join(", ")}`;
|
|
166
|
+
console.log(`[gonext-worker] daemon RUNNING (${who}). Logs: ${ctl.LOG_FILE}`);
|
|
167
|
+
} else {
|
|
168
|
+
console.log("[gonext-worker] daemon is NOT running. Start it with: gonext-cli start");
|
|
169
|
+
}
|
|
170
|
+
process.exit(0);
|
|
171
|
+
}
|
|
172
|
+
if (args[0] === "logs") {
|
|
173
|
+
const n = Number(parseFlag("--lines")) || 60;
|
|
174
|
+
const txt = await ctl.tailLog(n);
|
|
175
|
+
console.log(txt || `(no daemon log yet at ${ctl.LOG_FILE})`);
|
|
176
|
+
process.exit(0);
|
|
177
|
+
}
|
|
178
|
+
if (args[0] === "stop") {
|
|
179
|
+
const { killed } = await ctl.stopDaemon();
|
|
180
|
+
console.log(killed ? `[gonext-worker] stopped ${killed} daemon process(es).` : "[gonext-worker] no daemon was running.");
|
|
181
|
+
process.exit(0);
|
|
182
|
+
}
|
|
183
|
+
// start / restart both ensure exactly one daemon is running, config permitting.
|
|
184
|
+
if (args[0] === "restart") await ctl.stopDaemon();
|
|
185
|
+
// Don't spawn a daemon that would just exit(1): require a complete worker.env first.
|
|
186
|
+
const { hasWorkerConfig } = await import("./device-login.mjs");
|
|
187
|
+
if (!(await hasWorkerConfig())) {
|
|
188
|
+
console.error(
|
|
189
|
+
"[gonext-worker] not configured — run `gonext-cli login` (or `set`) first."
|
|
190
|
+
);
|
|
191
|
+
process.exit(1);
|
|
192
|
+
}
|
|
193
|
+
const r = await ctl.startDaemon();
|
|
194
|
+
if (r.started) {
|
|
195
|
+
console.log(`[gonext-worker] daemon started (pid ${r.pid}). Logs: ${ctl.LOG_FILE}`);
|
|
196
|
+
} else {
|
|
197
|
+
console.log("[gonext-worker] daemon already running — nothing to do.");
|
|
198
|
+
}
|
|
199
|
+
process.exit(0);
|
|
200
|
+
}
|
|
201
|
+
// Task #127, Phase 3: ensure the base chat model is downloaded + running (no GoTerminal).
|
|
202
|
+
if (args[0] === "doctor") {
|
|
203
|
+
const { hasWorkerConfig } = await import("./device-login.mjs");
|
|
204
|
+
if (!(await hasWorkerConfig())) {
|
|
205
|
+
console.error("[gonext-worker] not configured — run `gonext-cli login` first.");
|
|
206
|
+
process.exit(1);
|
|
207
|
+
}
|
|
208
|
+
const { runDoctor } = await import("./model-doctor.mjs");
|
|
209
|
+
const rl0 = (await import("node:readline")).createInterface({
|
|
210
|
+
input: process.stdin,
|
|
211
|
+
output: process.stdout,
|
|
212
|
+
});
|
|
213
|
+
const interactive = process.stdin.isTTY && process.stdout.isTTY;
|
|
214
|
+
const confirm = (q, defYes) =>
|
|
215
|
+
// Non-interactive: never download GBs / start a server unattended — probe + advise only.
|
|
216
|
+
interactive
|
|
217
|
+
? new Promise((res) =>
|
|
218
|
+
rl0.question(` ${q} [${defYes ? "Y/n" : "y/N"}] `, (a) => {
|
|
219
|
+
const s = a.trim().toLowerCase();
|
|
220
|
+
res(s ? s === "y" || s === "yes" : Boolean(defYes));
|
|
221
|
+
})
|
|
222
|
+
)
|
|
223
|
+
: Promise.resolve(false);
|
|
224
|
+
try {
|
|
225
|
+
await runDoctor({
|
|
226
|
+
apiBase: String(process.env.GONEXT_API_BASE ?? "").trim(),
|
|
227
|
+
workerKey: String(process.env.GONEXT_WORKER_KEY ?? "").trim(),
|
|
228
|
+
log: (s) => console.log(s),
|
|
229
|
+
confirm,
|
|
230
|
+
});
|
|
231
|
+
} finally {
|
|
232
|
+
rl0.close();
|
|
233
|
+
}
|
|
234
|
+
process.exit(0);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const apiBase = String(process.env.GONEXT_API_BASE ?? "")
|
|
238
|
+
.trim()
|
|
239
|
+
.replace(/\/+$/, "");
|
|
240
|
+
const workerKey = String(process.env.GONEXT_WORKER_KEY ?? "").trim();
|
|
241
|
+
const envFileWorkerKey = String(envFileLoad.parsed?.GONEXT_WORKER_KEY ?? "").trim();
|
|
242
|
+
const workerKeySource = (() => {
|
|
243
|
+
if (preloadedWorkerKey && workerKey === preloadedWorkerKey) {
|
|
244
|
+
return "process.env";
|
|
245
|
+
}
|
|
246
|
+
if (envFileWorkerKey && workerKey === envFileWorkerKey) {
|
|
247
|
+
return ENV_FILE;
|
|
248
|
+
}
|
|
249
|
+
return "unknown";
|
|
250
|
+
})();
|
|
251
|
+
const pollMs = 500;
|
|
252
|
+
const localHealthConcurrency = 4;
|
|
253
|
+
|
|
254
|
+
const CHUNK_PATH = "/api/worker/job-chunk";
|
|
255
|
+
const OCR_PROMPT = "Text Recognition:";
|
|
256
|
+
const OCR_MAX_TOKENS = 2048;
|
|
257
|
+
const OCR_TIMEOUT_MS = 180_000;
|
|
258
|
+
const OCR_DEBUG = String(process.env.GONEXT_OCR_DEBUG ?? "").trim() === "1";
|
|
259
|
+
const OCR_SKIP_CORRECTION = String(process.env.GONEXT_OCR_SKIP_CORRECTION ?? "").trim() === "1";
|
|
260
|
+
const OCR_CORRECT_TIMEOUT_MS = 120_000;
|
|
261
|
+
// Fire-and-forget warm-up of the agent's Ollama coding model. Generous because a
|
|
262
|
+
// cold load of a big/multimodal model into VRAM can take ~20s+; the request isn't
|
|
263
|
+
// awaited, so a long ceiling just avoids aborting an in-progress load.
|
|
264
|
+
const AGENT_WARMUP_TIMEOUT_MS = 180_000;
|
|
265
|
+
// How often the background keep-warm re-asserts the coding model's keep_alive:-1.
|
|
266
|
+
const AGENT_WARMUP_INTERVAL_MS = 240_000; // 4 min — under Ollama's 5-min default unload
|
|
267
|
+
// OCR translation/correction is hardcoded to a managed Ollama service + model.
|
|
268
|
+
// (The user-configurable "OCR translation model" chooser was removed.)
|
|
269
|
+
const OCR_TRANSLATE_OLLAMA_URL = "https://ollama1.gomarsic.cc";
|
|
270
|
+
const OCR_TRANSLATE_OLLAMA_MODEL = "qwen3:14b";
|
|
271
|
+
// Ollama defaults num_ctx to 2048 and silently truncates longer inputs (keeping
|
|
272
|
+
// only the tail), which would drop the start of a long OCR page. Pin the context
|
|
273
|
+
// window to 32k so full-page/multi-paragraph OCR is corrected in one pass.
|
|
274
|
+
const OCR_TRANSLATE_NUM_CTX = 32768;
|
|
275
|
+
|
|
276
|
+
async function workerFetch(path, init = {}) {
|
|
277
|
+
const url = `${apiBase}${path.startsWith("/") ? path : `/${path}`}`;
|
|
278
|
+
const headers = {
|
|
279
|
+
"Content-Type": "application/json",
|
|
280
|
+
"X-Worker-Key": workerKey,
|
|
281
|
+
...(init.headers ?? {}),
|
|
282
|
+
};
|
|
283
|
+
return fetch(url, { ...init, headers });
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
async function ensureWorkerOk(res, context) {
|
|
287
|
+
if (res.ok) return;
|
|
288
|
+
const snippet = (await res.text().catch(() => "")).trim().slice(0, 500);
|
|
289
|
+
throw new Error(
|
|
290
|
+
`${context} failed ${res.status}${snippet ? `: ${snippet}` : ""}`
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
function assertWorkerRuntimeConfig(mode) {
|
|
295
|
+
if (!workerKey) {
|
|
296
|
+
console.error(
|
|
297
|
+
`[gonext-worker] ${mode}: missing GONEXT_WORKER_KEY. Run: gonext-cli set <workerKey> --api-base <url>`
|
|
298
|
+
);
|
|
299
|
+
process.exit(1);
|
|
300
|
+
}
|
|
301
|
+
if (!apiBase) {
|
|
302
|
+
console.error(
|
|
303
|
+
`[gonext-worker] ${mode}: missing GONEXT_API_BASE. Run: gonext-cli set <workerKey> --api-base <url>`
|
|
304
|
+
);
|
|
305
|
+
process.exit(1);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
if (args[0] === "ws-ping-test") {
|
|
310
|
+
assertWorkerRuntimeConfig("ws-ping-test");
|
|
311
|
+
const url = `${apiBase}/api/worker/ws-ping-test`;
|
|
312
|
+
const res = await fetch(url, {
|
|
313
|
+
method: "POST",
|
|
314
|
+
headers: {
|
|
315
|
+
"Content-Type": "application/json",
|
|
316
|
+
"X-Worker-Key": workerKey,
|
|
317
|
+
},
|
|
318
|
+
body: "{}",
|
|
319
|
+
});
|
|
320
|
+
const text = await res.text();
|
|
321
|
+
if (!res.ok) {
|
|
322
|
+
console.error(`[gonext-worker] ws-ping-test failed ${res.status}: ${text}`);
|
|
323
|
+
if (res.status === 404 && text.includes("Cannot POST")) {
|
|
324
|
+
console.error(
|
|
325
|
+
"[gonext-worker] DEPLOY: this API does not have POST /api/worker/ws-ping-test yet. From repo root: sam build && sam deploy (or ship updated server-bundle to your Node host)."
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
process.exit(1);
|
|
329
|
+
}
|
|
330
|
+
try {
|
|
331
|
+
const parsed = JSON.parse(text);
|
|
332
|
+
console.log(
|
|
333
|
+
`[gonext-worker] ws-ping-test OK user=${parsed.userId ?? "unknown"} host=${
|
|
334
|
+
parsed.workerHostId ?? "legacy"
|
|
335
|
+
} keyfp=${parsed.workerKeyFingerprint ?? workerKeyFingerprint(workerKey)}`
|
|
336
|
+
);
|
|
337
|
+
} catch {
|
|
338
|
+
console.log("[gonext-worker] ws-ping-test OK:", text);
|
|
339
|
+
}
|
|
340
|
+
process.exit(0);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (args[0] === "workspace") {
|
|
344
|
+
// Manage the local workspaces the agent may READ and EDIT code in. Registration
|
|
345
|
+
// happens HERE on the Mac (never from the web) — the registry is the security
|
|
346
|
+
// boundary for the agent's coding tools (grep/read/edit/run stay inside these roots).
|
|
347
|
+
// gonext-cli workspace add <path> [--name <name>] [--allow-run]
|
|
348
|
+
// gonext-cli workspace remove <path-or-name>
|
|
349
|
+
// gonext-cli workspace list
|
|
350
|
+
const wsFile = join(homedir(), ".gonext", "workspaces.json");
|
|
351
|
+
const readWs = async () => {
|
|
352
|
+
try {
|
|
353
|
+
const raw = JSON.parse(await readFile(wsFile, "utf8"));
|
|
354
|
+
return Array.isArray(raw?.workspaces) ? raw.workspaces : [];
|
|
355
|
+
} catch {
|
|
356
|
+
return [];
|
|
357
|
+
}
|
|
358
|
+
};
|
|
359
|
+
const writeWs = async (workspaces) => {
|
|
360
|
+
await mkdir(dirname(wsFile), { recursive: true });
|
|
361
|
+
await writeFile(wsFile, JSON.stringify({ workspaces }, null, 2) + "\n");
|
|
362
|
+
};
|
|
363
|
+
const sub = args[1];
|
|
364
|
+
const { resolve } = await import("node:path");
|
|
365
|
+
const { existsSync, statSync } = await import("node:fs");
|
|
366
|
+
if (sub === "add") {
|
|
367
|
+
let p = args[2] || "";
|
|
368
|
+
if (!p) {
|
|
369
|
+
console.error("Usage: gonext-cli workspace add <path> [--name <name>] [--allow-run]");
|
|
370
|
+
process.exit(1);
|
|
371
|
+
}
|
|
372
|
+
if (p.startsWith("~/") || p === "~") p = join(homedir(), p.slice(p.length > 1 ? 2 : 1));
|
|
373
|
+
p = resolve(p);
|
|
374
|
+
if (!existsSync(p) || !statSync(p).isDirectory()) {
|
|
375
|
+
console.error(`[gonext-worker] workspace add: not a directory: ${p}`);
|
|
376
|
+
process.exit(1);
|
|
377
|
+
}
|
|
378
|
+
const nameIdx = args.indexOf("--name");
|
|
379
|
+
const name = nameIdx >= 0 && args[nameIdx + 1] ? args[nameIdx + 1] : p.split("/").filter(Boolean).pop();
|
|
380
|
+
const allowRun = args.includes("--allow-run");
|
|
381
|
+
const list = (await readWs()).filter((w) => w.path !== p);
|
|
382
|
+
list.push({ name, path: p, allowRun, addedAt: new Date().toISOString() });
|
|
383
|
+
await writeWs(list);
|
|
384
|
+
console.log(
|
|
385
|
+
`[gonext-worker] workspace registered: ${name} → ${p}` +
|
|
386
|
+
(allowRun ? " (run-commands ENABLED)" : " (run-commands disabled; add --allow-run to enable tests)")
|
|
387
|
+
);
|
|
388
|
+
process.exit(0);
|
|
389
|
+
}
|
|
390
|
+
if (sub === "remove") {
|
|
391
|
+
const key = args[2] || "";
|
|
392
|
+
const list = await readWs();
|
|
393
|
+
const next = list.filter((w) => w.path !== key && w.name !== key);
|
|
394
|
+
if (next.length === list.length) {
|
|
395
|
+
console.error(`[gonext-worker] workspace remove: no workspace matches '${key}'`);
|
|
396
|
+
process.exit(1);
|
|
397
|
+
}
|
|
398
|
+
await writeWs(next);
|
|
399
|
+
console.log(`[gonext-worker] workspace removed: ${key}`);
|
|
400
|
+
process.exit(0);
|
|
401
|
+
}
|
|
402
|
+
if (sub === "revert") {
|
|
403
|
+
// Restore the files an agent run changed: copy backups back over edits and delete
|
|
404
|
+
// created files. Uses the manifest the python tools persist per change.
|
|
405
|
+
// gonext-cli workspace revert → latest run
|
|
406
|
+
// gonext-cli workspace revert <runId> → a specific run (see backups dir)
|
|
407
|
+
const { readdirSync, existsSync: ex } = await import("node:fs");
|
|
408
|
+
const { copyFile, unlink, rmdir } = await import("node:fs/promises");
|
|
409
|
+
const backupsRoot = join(homedir(), ".gonext", "workspace-backups");
|
|
410
|
+
let runId = args[2] || "";
|
|
411
|
+
if (!runId) {
|
|
412
|
+
const runs = (() => {
|
|
413
|
+
try { return readdirSync(backupsRoot).filter((d) => !d.startsWith(".")).sort(); }
|
|
414
|
+
catch { return []; }
|
|
415
|
+
})();
|
|
416
|
+
if (runs.length === 0) {
|
|
417
|
+
console.error("[gonext-worker] workspace revert: no backup runs found.");
|
|
418
|
+
process.exit(1);
|
|
419
|
+
}
|
|
420
|
+
runId = runs[runs.length - 1];
|
|
421
|
+
}
|
|
422
|
+
const manifestPath = join(backupsRoot, runId, "manifest.json");
|
|
423
|
+
let manifest;
|
|
424
|
+
try {
|
|
425
|
+
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
|
426
|
+
} catch {
|
|
427
|
+
console.error(`[gonext-worker] workspace revert: no manifest for run ${runId} (${manifestPath})`);
|
|
428
|
+
process.exit(1);
|
|
429
|
+
}
|
|
430
|
+
let restored = 0, deleted = 0, failed = 0;
|
|
431
|
+
for (const c of manifest?.changes ?? []) {
|
|
432
|
+
try {
|
|
433
|
+
if (c.action === "edit" && c.backup && ex(c.backup)) {
|
|
434
|
+
await copyFile(c.backup, c.path);
|
|
435
|
+
restored += 1;
|
|
436
|
+
console.log(` restored ${c.path}`);
|
|
437
|
+
} else if (c.action === "create" && ex(c.path)) {
|
|
438
|
+
await unlink(c.path);
|
|
439
|
+
deleted += 1;
|
|
440
|
+
console.log(` deleted ${c.path}`);
|
|
441
|
+
} else if (c.action === "create_dir" && ex(c.path)) {
|
|
442
|
+
// rmdir only removes an EMPTY dir — a revert must never destroy content
|
|
443
|
+
// the user (or a later run) put inside the created folder afterwards.
|
|
444
|
+
await rmdir(c.path);
|
|
445
|
+
deleted += 1;
|
|
446
|
+
console.log(` removed ${c.path}/`);
|
|
447
|
+
}
|
|
448
|
+
} catch (err) {
|
|
449
|
+
failed += 1;
|
|
450
|
+
console.error(` FAILED ${c.path}: ${err.message}`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
console.log(
|
|
454
|
+
`[gonext-worker] workspace revert ${runId}: ${restored} restored, ${deleted} created-file(s) deleted` +
|
|
455
|
+
(failed ? `, ${failed} FAILED` : "") +
|
|
456
|
+
" (backups kept)"
|
|
457
|
+
);
|
|
458
|
+
process.exit(failed ? 1 : 0);
|
|
459
|
+
}
|
|
460
|
+
// default: list
|
|
461
|
+
const list = await readWs();
|
|
462
|
+
if (list.length === 0) {
|
|
463
|
+
console.log("[gonext-worker] no workspaces registered. Add one:\n gonext-cli workspace add ~/Projects/my-repo --allow-run");
|
|
464
|
+
} else {
|
|
465
|
+
for (const w of list) {
|
|
466
|
+
console.log(` ${w.name} ${w.path} run:${w.allowRun ? "yes" : "no"}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
process.exit(0);
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
if (args[0] === "embed") {
|
|
473
|
+
// Start the MLX embeddings server (OpenAI-compatible /v1/embeddings) for the RAG
|
|
474
|
+
// feature, in the FOREGROUND of this terminal. Run it in a SEPARATE terminal from the
|
|
475
|
+
// worker (the worker keeps polling; this holds one terminal for the embed model).
|
|
476
|
+
// gonext-cli embed --model ~/mlx-models/<embed-model> [--port 8085] [--host 127.0.0.1]
|
|
477
|
+
// Then set web Settings → Agent → RAG → "Embedding server URL" to http://<host>:<port>.
|
|
478
|
+
const flag = (name, fallback) => {
|
|
479
|
+
const i = args.indexOf(name);
|
|
480
|
+
return i >= 0 && typeof args[i + 1] === "string" ? args[i + 1] : fallback;
|
|
481
|
+
};
|
|
482
|
+
let model = flag("--model", process.env.GONEXT_EMBED_MODEL || "");
|
|
483
|
+
const port = flag("--port", process.env.GONEXT_EMBED_PORT || "8085");
|
|
484
|
+
const host = flag("--host", process.env.GONEXT_EMBED_HOST || "127.0.0.1");
|
|
485
|
+
if (!model) {
|
|
486
|
+
console.error(
|
|
487
|
+
"[gonext-worker] embed: --model is required.\n" +
|
|
488
|
+
" Usage: gonext-cli embed --model ~/mlx-models/Qwen3-Embedding-8B-4bit-DWQ [--port 8085] [--host 127.0.0.1]\n" +
|
|
489
|
+
" (or set GONEXT_EMBED_MODEL). Then set the RAG embedding server URL to http://<host>:<port> in web Settings."
|
|
490
|
+
);
|
|
491
|
+
process.exit(1);
|
|
492
|
+
}
|
|
493
|
+
// Expand a leading ~ to the home dir (spawn does not do shell expansion).
|
|
494
|
+
if (model.startsWith("~/") || model === "~") {
|
|
495
|
+
model = join(homedir(), model.slice(model.length > 1 ? 2 : 1));
|
|
496
|
+
}
|
|
497
|
+
const python =
|
|
498
|
+
(process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "").trim() ||
|
|
499
|
+
"python3";
|
|
500
|
+
const script = join(WORKER_DIR, "gonext_mlx_embed.py");
|
|
501
|
+
console.log(
|
|
502
|
+
`[gonext-worker] starting MLX embeddings server → ${python} ${script} --model ${model} --port ${port} --host ${host}`
|
|
503
|
+
);
|
|
504
|
+
const child = spawn(
|
|
505
|
+
python,
|
|
506
|
+
[script, "--model", model, "--port", String(port), "--host", String(host)],
|
|
507
|
+
{ stdio: "inherit" }
|
|
508
|
+
);
|
|
509
|
+
child.on("error", (err) => {
|
|
510
|
+
console.error(`[gonext-worker] embed: failed to start python (${python}): ${err.message}`);
|
|
511
|
+
process.exit(1);
|
|
512
|
+
});
|
|
513
|
+
// Forward Ctrl-C to the child so the model unloads cleanly.
|
|
514
|
+
const stop = () => child.kill("SIGINT");
|
|
515
|
+
process.on("SIGINT", stop);
|
|
516
|
+
process.on("SIGTERM", stop);
|
|
517
|
+
// Block the module from continuing to the polling loop; exit when the child exits.
|
|
518
|
+
await new Promise((resolve) => child.on("exit", (code) => resolve(code ?? 0))).then(
|
|
519
|
+
(code) => process.exit(code)
|
|
520
|
+
);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
if (args[0] === "simulate-chat") {
|
|
524
|
+
assertWorkerRuntimeConfig("simulate-chat");
|
|
525
|
+
const reply =
|
|
526
|
+
args.slice(1).join(" ").trim() ||
|
|
527
|
+
String(process.env.GONEXT_SIMULATE_TEXT ?? "").trim() ||
|
|
528
|
+
"Simulated model reply: hello from gonext-cli simulate-chat.";
|
|
529
|
+
const jobRes = await workerFetch("/api/worker/jobs/next", { method: "POST" });
|
|
530
|
+
if (jobRes.status === 204) {
|
|
531
|
+
console.error(
|
|
532
|
+
"[gonext-worker] simulate-chat: no pending job. In the web app, send a message that returns 202 (async local chat), then run this again. Stop the normal worker so it does not claim the job first."
|
|
533
|
+
);
|
|
534
|
+
process.exit(1);
|
|
535
|
+
}
|
|
536
|
+
if (!jobRes.ok) {
|
|
537
|
+
const t = await jobRes.text().catch(() => "");
|
|
538
|
+
console.error(`[gonext-worker] simulate-chat: jobs/next failed ${jobRes.status}: ${t}`);
|
|
539
|
+
process.exit(1);
|
|
540
|
+
}
|
|
541
|
+
const job = await jobRes.json();
|
|
542
|
+
const jobId = job?.jobId;
|
|
543
|
+
if (!jobId) {
|
|
544
|
+
console.error("[gonext-worker] simulate-chat: invalid jobs/next response (missing jobId).");
|
|
545
|
+
process.exit(1);
|
|
546
|
+
}
|
|
547
|
+
const isLocalHealth =
|
|
548
|
+
job.jobType === "local_health" || job.modelKey === "local_health";
|
|
549
|
+
const isOcrJob =
|
|
550
|
+
job.jobType === "ocr" ||
|
|
551
|
+
(job.payload &&
|
|
552
|
+
typeof job.payload === "object" &&
|
|
553
|
+
typeof job.payload.ocrId === "string");
|
|
554
|
+
if (isLocalHealth || isOcrJob) {
|
|
555
|
+
await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
556
|
+
method: "PATCH",
|
|
557
|
+
body: JSON.stringify({
|
|
558
|
+
jobStatus: "failed",
|
|
559
|
+
errorMessage:
|
|
560
|
+
isLocalHealth
|
|
561
|
+
? "simulate-chat: claimed a local_health job. Mark failed so you can retry. Queue a chat message (not Settings refresh) and stop the normal worker before simulate-chat."
|
|
562
|
+
: "simulate-chat: claimed an OCR job. Mark failed so you can retry. Queue a chat message and stop the normal worker before simulate-chat.",
|
|
563
|
+
totalTimeSeconds: 0,
|
|
564
|
+
}),
|
|
565
|
+
});
|
|
566
|
+
console.error(
|
|
567
|
+
isLocalHealth
|
|
568
|
+
? "[gonext-worker] simulate-chat: claimed local_health instead of chat. Job marked failed. Retry with only a pending chat job."
|
|
569
|
+
: "[gonext-worker] simulate-chat: claimed OCR instead of chat. Job marked failed. Retry with only a pending chat job."
|
|
570
|
+
);
|
|
571
|
+
process.exit(1);
|
|
572
|
+
}
|
|
573
|
+
const start = Date.now();
|
|
574
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
575
|
+
method: "PATCH",
|
|
576
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
577
|
+
});
|
|
578
|
+
if (!runRes.ok) {
|
|
579
|
+
const errBody = await runRes.text().catch(() => "");
|
|
580
|
+
console.error(
|
|
581
|
+
`[gonext-worker] simulate-chat: mark running failed ${runRes.status}${errBody ? `: ${errBody}` : ""}`
|
|
582
|
+
);
|
|
583
|
+
process.exit(1);
|
|
584
|
+
}
|
|
585
|
+
const sliceSize = 160;
|
|
586
|
+
for (let i = 0; i < reply.length; i += sliceSize) {
|
|
587
|
+
const slice = reply.slice(i, i + sliceSize);
|
|
588
|
+
if (!slice) continue;
|
|
589
|
+
const chunkRes = await workerFetch(CHUNK_PATH, {
|
|
590
|
+
method: "POST",
|
|
591
|
+
body: JSON.stringify({ jobId, text: slice }),
|
|
592
|
+
});
|
|
593
|
+
if (!chunkRes.ok && chunkRes.status !== 204) {
|
|
594
|
+
const snippet = (await chunkRes.text().catch(() => "")).trim().slice(0, 400);
|
|
595
|
+
console.error(
|
|
596
|
+
`[gonext-worker] simulate-chat: job-chunk failed ${chunkRes.status} jobId=${jobId}` +
|
|
597
|
+
(snippet ? ` response=${snippet}` : "")
|
|
598
|
+
);
|
|
599
|
+
process.exit(1);
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
603
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
604
|
+
method: "PATCH",
|
|
605
|
+
body: JSON.stringify({
|
|
606
|
+
jobStatus: "completed",
|
|
607
|
+
resultText: reply,
|
|
608
|
+
tokenCount: Math.max(1, Math.ceil(reply.length / 4)),
|
|
609
|
+
totalTimeSeconds,
|
|
610
|
+
}),
|
|
611
|
+
});
|
|
612
|
+
if (!doneRes.ok) {
|
|
613
|
+
const snippet = (await doneRes.text().catch(() => "")).trim().slice(0, 400);
|
|
614
|
+
console.error(
|
|
615
|
+
`[gonext-worker] simulate-chat: complete PATCH failed ${doneRes.status}` +
|
|
616
|
+
(snippet ? `: ${snippet}` : "")
|
|
617
|
+
);
|
|
618
|
+
process.exit(1);
|
|
619
|
+
}
|
|
620
|
+
console.log(
|
|
621
|
+
`[gonext-worker] simulate-chat OK jobId=${jobId} (${totalTimeSeconds.toFixed(2)}s, ${reply.length} chars) — same HTTP path as the real worker (API pushes worker_job_chunk / worker_job to the web app).`
|
|
622
|
+
);
|
|
623
|
+
process.exit(0);
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
assertWorkerRuntimeConfig("worker");
|
|
627
|
+
|
|
628
|
+
console.log(`[gonext-worker] gonext-cli v${WORKER_VERSION}`);
|
|
629
|
+
console.log(
|
|
630
|
+
`[gonext-worker] API base ${apiBase} — streaming chunks POST ${apiBase}${CHUNK_PATH} — keyfp=${workerKeyFingerprint(
|
|
631
|
+
workerKey
|
|
632
|
+
)}`
|
|
633
|
+
);
|
|
634
|
+
console.log(
|
|
635
|
+
`[gonext-worker] worker key ${maskWorkerKey(workerKey)} (keyfp=${workerKeyFingerprint(
|
|
636
|
+
workerKey
|
|
637
|
+
)}, source=${workerKeySource})`
|
|
638
|
+
);
|
|
639
|
+
|
|
640
|
+
function toOpenAIMessages(messages) {
|
|
641
|
+
return (Array.isArray(messages) ? messages : []).map((m) => {
|
|
642
|
+
if (m.role === "user" && m.attachments?.length) {
|
|
643
|
+
return {
|
|
644
|
+
role: m.role,
|
|
645
|
+
content: [
|
|
646
|
+
{ type: "text", text: m.content },
|
|
647
|
+
...m.attachments.map((a) => ({
|
|
648
|
+
type: "image_url",
|
|
649
|
+
image_url: { url: `data:${a.mimeType};base64,${a.data}` },
|
|
650
|
+
})),
|
|
651
|
+
],
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
return { role: m.role, content: m.content };
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
function parseCompletionTokens(usage) {
|
|
659
|
+
if (!usage || typeof usage !== "object") {
|
|
660
|
+
return null;
|
|
661
|
+
}
|
|
662
|
+
if (typeof usage.completion_tokens === "number") {
|
|
663
|
+
return usage.completion_tokens;
|
|
664
|
+
}
|
|
665
|
+
if (typeof usage.output_tokens === "number") {
|
|
666
|
+
return usage.output_tokens;
|
|
667
|
+
}
|
|
668
|
+
return null;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
function shouldRetryWithoutUsage(err) {
|
|
672
|
+
const msg =
|
|
673
|
+
err instanceof Error ? err.message.toLowerCase() : String(err).toLowerCase();
|
|
674
|
+
return msg.includes("stream_options") || msg.includes("include_usage");
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
async function runChatJob(job) {
|
|
678
|
+
const { jobId, payload } = job;
|
|
679
|
+
if (!payload || !Array.isArray(payload.messages)) {
|
|
680
|
+
throw new Error("Invalid chat payload: messages array is missing.");
|
|
681
|
+
}
|
|
682
|
+
const start = Date.now();
|
|
683
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
684
|
+
method: "PATCH",
|
|
685
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
686
|
+
});
|
|
687
|
+
if (!runRes.ok) {
|
|
688
|
+
const errBody = await runRes.text().catch(() => "");
|
|
689
|
+
throw new Error(
|
|
690
|
+
`mark running failed ${runRes.status}${errBody ? `: ${errBody}` : ""}`
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
const client = new OpenAI({
|
|
695
|
+
baseURL: payload.baseURL,
|
|
696
|
+
apiKey: payload.apiKey || "ollama",
|
|
697
|
+
maxRetries: 0,
|
|
698
|
+
timeout: 90_000,
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
let buf = "";
|
|
702
|
+
let flushTimer = null;
|
|
703
|
+
let fullText = "";
|
|
704
|
+
/** Chains debounced chunk POSTs so we never PATCH `completed` while a chunk POST is still in flight. */
|
|
705
|
+
let flushTail = Promise.resolve();
|
|
706
|
+
|
|
707
|
+
/** Batch streamed text: fewer HTTPS round-trips to the API than a 12ms debounce per flush. */
|
|
708
|
+
const CHUNK_DEBOUNCE_MS = 80;
|
|
709
|
+
const CHUNK_MAX_BUF = 6144;
|
|
710
|
+
|
|
711
|
+
const flushChunks = async () => {
|
|
712
|
+
const t = buf;
|
|
713
|
+
buf = "";
|
|
714
|
+
if (!t) return;
|
|
715
|
+
const res = await workerFetch(CHUNK_PATH, {
|
|
716
|
+
method: "POST",
|
|
717
|
+
body: JSON.stringify({ jobId, text: t }),
|
|
718
|
+
});
|
|
719
|
+
if (!res.ok && res.status !== 204) {
|
|
720
|
+
const snippet = (await res.text().catch(() => "")).trim().slice(0, 400);
|
|
721
|
+
const url = `${apiBase}${CHUNK_PATH}`;
|
|
722
|
+
const benign409 =
|
|
723
|
+
res.status === 409 && snippet.includes('"jobStatus":"completed"');
|
|
724
|
+
if (!benign409) {
|
|
725
|
+
console.error(
|
|
726
|
+
`[gonext-worker] job-chunk POST failed status=${res.status} url=${url} jobId=${jobId}` +
|
|
727
|
+
(snippet ? ` response=${snippet}` : "")
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
if (res.status === 404) {
|
|
731
|
+
if (snippet.includes("Cannot POST")) {
|
|
732
|
+
console.error(
|
|
733
|
+
"[gonext-worker] DEPLOY: Lambda/API still runs old Express without POST /api/worker/job-chunk. From repo: cd api && npm ci && sam build && sam deploy — or publish the latest bundled handler so execute-api serves current routes."
|
|
734
|
+
);
|
|
735
|
+
} else {
|
|
736
|
+
console.error(
|
|
737
|
+
'[gonext-worker] hint: JSON {"error":"Job not found"} = Dynamo lookup failed (wrong worker user vs job owner). Otherwise redeploy API.'
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
const enqueueText = (s) => {
|
|
745
|
+
if (!s) return;
|
|
746
|
+
fullText += s;
|
|
747
|
+
buf += s;
|
|
748
|
+
if (buf.length >= CHUNK_MAX_BUF) {
|
|
749
|
+
if (flushTimer) {
|
|
750
|
+
clearTimeout(flushTimer);
|
|
751
|
+
flushTimer = null;
|
|
752
|
+
}
|
|
753
|
+
flushTail = flushTail.then(() => flushChunks()).catch((err) => {
|
|
754
|
+
console.error("[gonext-worker] chunk flush error:", err);
|
|
755
|
+
});
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
if (!flushTimer) {
|
|
759
|
+
flushTimer = setTimeout(() => {
|
|
760
|
+
flushTimer = null;
|
|
761
|
+
flushTail = flushTail.then(() => flushChunks()).catch((err) => {
|
|
762
|
+
console.error("[gonext-worker] chunk flush error:", err);
|
|
763
|
+
});
|
|
764
|
+
}, CHUNK_DEBOUNCE_MS);
|
|
765
|
+
}
|
|
766
|
+
};
|
|
767
|
+
|
|
768
|
+
try {
|
|
769
|
+
const streamRequest = {
|
|
770
|
+
model: payload.modelId,
|
|
771
|
+
messages: toOpenAIMessages(payload.messages),
|
|
772
|
+
stream: true,
|
|
773
|
+
temperature: 0,
|
|
774
|
+
};
|
|
775
|
+
const stream = await client.chat.completions
|
|
776
|
+
.create({
|
|
777
|
+
...streamRequest,
|
|
778
|
+
stream_options: { include_usage: true },
|
|
779
|
+
})
|
|
780
|
+
.catch(async (e) => {
|
|
781
|
+
if (!shouldRetryWithoutUsage(e)) {
|
|
782
|
+
throw e;
|
|
783
|
+
}
|
|
784
|
+
return client.chat.completions.create(streamRequest);
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
let tokenCount = 0;
|
|
788
|
+
let completionTokensFromUsage = null;
|
|
789
|
+
let isStartThinking = false;
|
|
790
|
+
let isEndThinking = false;
|
|
791
|
+
|
|
792
|
+
for await (const chunk of stream) {
|
|
793
|
+
const usageTokens = parseCompletionTokens(chunk.usage);
|
|
794
|
+
if (usageTokens !== null) {
|
|
795
|
+
completionTokensFromUsage = usageTokens;
|
|
796
|
+
}
|
|
797
|
+
const delta = chunk.choices[0]?.delta;
|
|
798
|
+
const content = delta?.content ?? "";
|
|
799
|
+
const reasoningContent = delta?.reasoning_content;
|
|
800
|
+
tokenCount += 1;
|
|
801
|
+
|
|
802
|
+
if (reasoningContent) {
|
|
803
|
+
if (!isStartThinking) {
|
|
804
|
+
isStartThinking = true;
|
|
805
|
+
enqueueText("<think>");
|
|
806
|
+
}
|
|
807
|
+
enqueueText(reasoningContent);
|
|
808
|
+
} else {
|
|
809
|
+
if (isStartThinking && !isEndThinking) {
|
|
810
|
+
isEndThinking = true;
|
|
811
|
+
enqueueText("</think>");
|
|
812
|
+
}
|
|
813
|
+
if (content) {
|
|
814
|
+
enqueueText(content);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
if (flushTimer) {
|
|
820
|
+
clearTimeout(flushTimer);
|
|
821
|
+
flushTimer = null;
|
|
822
|
+
}
|
|
823
|
+
await flushTail;
|
|
824
|
+
await flushChunks();
|
|
825
|
+
|
|
826
|
+
logModelResponseToWorker(jobId, payload.modelId, fullText);
|
|
827
|
+
|
|
828
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
829
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
830
|
+
method: "PATCH",
|
|
831
|
+
body: JSON.stringify({
|
|
832
|
+
jobStatus: "completed",
|
|
833
|
+
resultText: fullText,
|
|
834
|
+
tokenCount:
|
|
835
|
+
completionTokensFromUsage !== null
|
|
836
|
+
? completionTokensFromUsage
|
|
837
|
+
: Math.max(1, tokenCount),
|
|
838
|
+
totalTimeSeconds,
|
|
839
|
+
}),
|
|
840
|
+
});
|
|
841
|
+
await ensureWorkerOk(doneRes, `complete PATCH jobId=${jobId}`);
|
|
842
|
+
console.log(`[gonext-worker] completed ${jobId} (${totalTimeSeconds.toFixed(1)}s)`);
|
|
843
|
+
} catch (e) {
|
|
844
|
+
if (flushTimer) {
|
|
845
|
+
clearTimeout(flushTimer);
|
|
846
|
+
flushTimer = null;
|
|
847
|
+
}
|
|
848
|
+
await flushTail;
|
|
849
|
+
await flushChunks().catch(() => {});
|
|
850
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
851
|
+
const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
852
|
+
method: "PATCH",
|
|
853
|
+
body: JSON.stringify({
|
|
854
|
+
jobStatus: "failed",
|
|
855
|
+
errorMessage: message,
|
|
856
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
857
|
+
}),
|
|
858
|
+
});
|
|
859
|
+
if (!failRes.ok) {
|
|
860
|
+
const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
|
|
861
|
+
console.error(
|
|
862
|
+
`[gonext-worker] failed status PATCH also failed ${failRes.status} jobId=${jobId}` +
|
|
863
|
+
(snippet ? ` response=${snippet}` : "")
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
console.error(`[gonext-worker] failed ${jobId}:`, message);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
function categorizeHttpStatus(status) {
|
|
871
|
+
if (status >= 200 && status < 300) return "2xx";
|
|
872
|
+
if (status >= 300 && status < 400) return "3xx";
|
|
873
|
+
if (status >= 400 && status < 500) return "4xx";
|
|
874
|
+
if (status >= 500 && status < 600) return "5xx";
|
|
875
|
+
return "network_error";
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
function probeVerdict(status, expectedStatus) {
|
|
879
|
+
if (expectedStatus === undefined || expectedStatus === null) {
|
|
880
|
+
return status >= 200 && status < 300 ? "pass" : "fail";
|
|
881
|
+
}
|
|
882
|
+
const list = Array.isArray(expectedStatus) ? expectedStatus : [expectedStatus];
|
|
883
|
+
return list.includes(status) ? "pass" : "fail";
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/** tool_only / agentic summary: ask the selected model to interpret the measured result. */
|
|
887
|
+
async function summarizeProbeWithModel(payload, result) {
|
|
888
|
+
const client = new OpenAI({
|
|
889
|
+
baseURL: payload.agentBaseURL,
|
|
890
|
+
apiKey: payload.agentApiKey || "local",
|
|
891
|
+
maxRetries: 0,
|
|
892
|
+
timeout: 60_000,
|
|
893
|
+
});
|
|
894
|
+
const lines = [
|
|
895
|
+
`HTTP probe result for ${result.request.method} ${result.request.url}`,
|
|
896
|
+
];
|
|
897
|
+
if (result.response) {
|
|
898
|
+
lines.push(
|
|
899
|
+
`Status: ${result.response.status} ${result.response.statusText} (${result.category})`,
|
|
900
|
+
`Latency: ${result.response.latencyMs} ms`,
|
|
901
|
+
`Body (first chars): ${result.response.bodySnippet || "(empty)"}`
|
|
902
|
+
);
|
|
903
|
+
} else {
|
|
904
|
+
lines.push(`No response — ${result.category}: ${result.error ?? "unknown error"}`);
|
|
905
|
+
}
|
|
906
|
+
lines.push(
|
|
907
|
+
`Verdict: ${result.verdict}.`,
|
|
908
|
+
"In 1-2 sentences, say whether the endpoint is healthy and what this status means."
|
|
909
|
+
);
|
|
910
|
+
const completion = await client.chat.completions.create({
|
|
911
|
+
model: payload.agentModelId,
|
|
912
|
+
messages: [
|
|
913
|
+
{
|
|
914
|
+
role: "system",
|
|
915
|
+
content: "You are an API health assistant. Be concise and factual.",
|
|
916
|
+
},
|
|
917
|
+
{ role: "user", content: lines.join("\n") },
|
|
918
|
+
],
|
|
919
|
+
temperature: 0,
|
|
920
|
+
max_tokens: 200,
|
|
921
|
+
});
|
|
922
|
+
return completion.choices?.[0]?.message?.content?.trim() || "";
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
/** Deterministic HTTP measurement: real status/latency/headers/body, no model. */
|
|
926
|
+
async function performHttpMeasurement(params) {
|
|
927
|
+
let response = null;
|
|
928
|
+
let category;
|
|
929
|
+
let errorMessage;
|
|
930
|
+
const reqStart = Date.now();
|
|
931
|
+
const ac = new AbortController();
|
|
932
|
+
const timer = setTimeout(() => ac.abort(), params.timeoutMs);
|
|
933
|
+
try {
|
|
934
|
+
const res = await fetch(params.url, {
|
|
935
|
+
method: params.method,
|
|
936
|
+
headers: params.headers,
|
|
937
|
+
body: params.sendBody ? params.body : undefined,
|
|
938
|
+
signal: ac.signal,
|
|
939
|
+
redirect: "manual",
|
|
940
|
+
});
|
|
941
|
+
const latencyMs = Date.now() - reqStart;
|
|
942
|
+
let text = "";
|
|
943
|
+
try {
|
|
944
|
+
text = await res.text();
|
|
945
|
+
} catch {
|
|
946
|
+
text = "";
|
|
947
|
+
}
|
|
948
|
+
const SNIP = 2048;
|
|
949
|
+
const bodySnippet =
|
|
950
|
+
text.length > SNIP
|
|
951
|
+
? `${text.slice(0, SNIP)} …[truncated ${text.length - SNIP} chars]`
|
|
952
|
+
: text;
|
|
953
|
+
const resHeaders = {};
|
|
954
|
+
res.headers.forEach((v, k) => {
|
|
955
|
+
resHeaders[k] = v;
|
|
956
|
+
});
|
|
957
|
+
category = categorizeHttpStatus(res.status);
|
|
958
|
+
response = {
|
|
959
|
+
status: res.status,
|
|
960
|
+
statusText: res.statusText || "",
|
|
961
|
+
ok: res.status >= 200 && res.status < 300,
|
|
962
|
+
latencyMs,
|
|
963
|
+
headers: resHeaders,
|
|
964
|
+
bodySnippet,
|
|
965
|
+
bodyBytes: Buffer.byteLength(text),
|
|
966
|
+
};
|
|
967
|
+
} catch (err) {
|
|
968
|
+
const aborted =
|
|
969
|
+
ac.signal.aborted ||
|
|
970
|
+
(err && typeof err === "object" && err.name === "AbortError");
|
|
971
|
+
category = aborted ? "timeout" : "network_error";
|
|
972
|
+
errorMessage = aborted
|
|
973
|
+
? `Request timed out after ${params.timeoutMs} ms`
|
|
974
|
+
: err instanceof Error
|
|
975
|
+
? err.message
|
|
976
|
+
: String(err);
|
|
977
|
+
} finally {
|
|
978
|
+
clearTimeout(timer);
|
|
979
|
+
}
|
|
980
|
+
return { response, category, errorMessage };
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
/**
|
|
984
|
+
* Spawn a child process, feed it stdin, and call onLine for each NDJSON stdout
|
|
985
|
+
* line as they arrive. Resolves when the process exits cleanly; rejects on
|
|
986
|
+
* non-zero exit or timeout. stderr is collected and appended to error messages.
|
|
987
|
+
*/
|
|
988
|
+
function runProcessWithStreamingStdout(cmd, cmdArgs, stdinStr, timeoutMs, onLine, extraEnv, abortSignal, onTimeout) {
|
|
989
|
+
return new Promise((resolve, reject) => {
|
|
990
|
+
const spawnOpts = { stdio: ["pipe", "pipe", "pipe"] };
|
|
991
|
+
if (extraEnv && Object.keys(extraEnv).length > 0) {
|
|
992
|
+
spawnOpts.env = { ...process.env, ...extraEnv };
|
|
993
|
+
}
|
|
994
|
+
const child = spawn(cmd, cmdArgs, spawnOpts);
|
|
995
|
+
let stderr = "";
|
|
996
|
+
let lineBuffer = "";
|
|
997
|
+
let timedOut = false;
|
|
998
|
+
let aborted = false;
|
|
999
|
+
let closed = false;
|
|
1000
|
+
let timerLive = true;
|
|
1001
|
+
// On the time budget: with no onTimeout handler (probe/legacy) this is a hard cap —
|
|
1002
|
+
// kill immediately. With a handler (task #81, interactive terminal) ask the user
|
|
1003
|
+
// whether to keep waiting; the child keeps running/streaming during the ~60s prompt.
|
|
1004
|
+
// "Keep waiting" disables the cap for the rest of the run; No / no-answer kills it.
|
|
1005
|
+
const fireTimeout = async () => {
|
|
1006
|
+
if (!timerLive || closed || aborted) return;
|
|
1007
|
+
if (typeof onTimeout !== "function") {
|
|
1008
|
+
timedOut = true;
|
|
1009
|
+
timerLive = false;
|
|
1010
|
+
child.kill("SIGKILL");
|
|
1011
|
+
return;
|
|
1012
|
+
}
|
|
1013
|
+
let extend = false;
|
|
1014
|
+
try {
|
|
1015
|
+
extend = await onTimeout();
|
|
1016
|
+
} catch {
|
|
1017
|
+
extend = false;
|
|
1018
|
+
}
|
|
1019
|
+
if (closed || aborted || !timerLive) return; // finished/cancelled during the prompt
|
|
1020
|
+
if (extend) {
|
|
1021
|
+
timerLive = false; // ignore the time budget for the rest of this run
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
timedOut = true;
|
|
1025
|
+
timerLive = false;
|
|
1026
|
+
child.kill("SIGKILL");
|
|
1027
|
+
};
|
|
1028
|
+
const timer = setTimeout(fireTimeout, timeoutMs);
|
|
1029
|
+
// User-requested cancel (gonext Ctrl+C): kill the Python agent so it stops burning
|
|
1030
|
+
// steps/tokens. On close we resolve cleanly (not reject) — the caller checks the
|
|
1031
|
+
// cancel flag and leaves the job in its already-"cancelled" state.
|
|
1032
|
+
const onAbort = () => {
|
|
1033
|
+
if (aborted) return;
|
|
1034
|
+
aborted = true;
|
|
1035
|
+
child.kill("SIGKILL");
|
|
1036
|
+
};
|
|
1037
|
+
if (abortSignal) {
|
|
1038
|
+
if (abortSignal.aborted) onAbort();
|
|
1039
|
+
else abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
1040
|
+
}
|
|
1041
|
+
child.stdout.on("data", (d) => {
|
|
1042
|
+
lineBuffer += d.toString("utf8");
|
|
1043
|
+
const parts = lineBuffer.split("\n");
|
|
1044
|
+
lineBuffer = parts.pop() ?? "";
|
|
1045
|
+
for (const line of parts) {
|
|
1046
|
+
const trimmed = line.trim();
|
|
1047
|
+
if (trimmed) {
|
|
1048
|
+
try {
|
|
1049
|
+
onLine(JSON.parse(trimmed));
|
|
1050
|
+
} catch {
|
|
1051
|
+
/* ignore non-JSON */
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
});
|
|
1056
|
+
child.stderr.on("data", (d) => {
|
|
1057
|
+
stderr += d;
|
|
1058
|
+
});
|
|
1059
|
+
child.on("error", (e) => {
|
|
1060
|
+
closed = true;
|
|
1061
|
+
timerLive = false;
|
|
1062
|
+
clearTimeout(timer);
|
|
1063
|
+
reject(e);
|
|
1064
|
+
});
|
|
1065
|
+
child.on("close", (code) => {
|
|
1066
|
+
closed = true;
|
|
1067
|
+
timerLive = false;
|
|
1068
|
+
clearTimeout(timer);
|
|
1069
|
+
if (abortSignal) abortSignal.removeEventListener?.("abort", onAbort);
|
|
1070
|
+
// Drain any remaining buffered line
|
|
1071
|
+
const remaining = lineBuffer.trim();
|
|
1072
|
+
if (remaining) {
|
|
1073
|
+
try {
|
|
1074
|
+
onLine(JSON.parse(remaining));
|
|
1075
|
+
} catch {
|
|
1076
|
+
/* ignore */
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
if (aborted) {
|
|
1080
|
+
resolve(); // cancelled by the user — a clean stop, not an error
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
if (timedOut) {
|
|
1084
|
+
reject(new Error(`agent chat timed out after ${timeoutMs} ms`));
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
if (code !== 0) {
|
|
1088
|
+
reject(
|
|
1089
|
+
new Error(
|
|
1090
|
+
`agent chat exited ${code}${stderr.trim() ? `: ${stderr.trim().slice(0, 500)}` : ""}`
|
|
1091
|
+
)
|
|
1092
|
+
);
|
|
1093
|
+
return;
|
|
1094
|
+
}
|
|
1095
|
+
resolve();
|
|
1096
|
+
});
|
|
1097
|
+
child.stdin.write(stdinStr);
|
|
1098
|
+
child.stdin.end();
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/** Run a child process, feed it stdin, resolve with stdout (rejects on non-zero/timeout). */
|
|
1103
|
+
function runProcessWithStdin(cmd, cmdArgs, stdinStr, timeoutMs) {
|
|
1104
|
+
return new Promise((resolve, reject) => {
|
|
1105
|
+
const child = spawn(cmd, cmdArgs, { stdio: ["pipe", "pipe", "pipe"] });
|
|
1106
|
+
let stdout = "";
|
|
1107
|
+
let stderr = "";
|
|
1108
|
+
let timedOut = false;
|
|
1109
|
+
const timer = setTimeout(() => {
|
|
1110
|
+
timedOut = true;
|
|
1111
|
+
child.kill("SIGKILL");
|
|
1112
|
+
}, timeoutMs);
|
|
1113
|
+
child.stdout.on("data", (d) => {
|
|
1114
|
+
stdout += d;
|
|
1115
|
+
});
|
|
1116
|
+
child.stderr.on("data", (d) => {
|
|
1117
|
+
stderr += d;
|
|
1118
|
+
});
|
|
1119
|
+
child.on("error", (e) => {
|
|
1120
|
+
clearTimeout(timer);
|
|
1121
|
+
reject(e);
|
|
1122
|
+
});
|
|
1123
|
+
child.on("close", (code) => {
|
|
1124
|
+
clearTimeout(timer);
|
|
1125
|
+
if (timedOut) {
|
|
1126
|
+
reject(new Error(`probe agent timed out after ${timeoutMs} ms`));
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
if (code !== 0) {
|
|
1130
|
+
reject(
|
|
1131
|
+
new Error(
|
|
1132
|
+
`probe agent exited ${code}${stderr.trim() ? `: ${stderr.trim().slice(0, 500)}` : ""}`
|
|
1133
|
+
)
|
|
1134
|
+
);
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
resolve(stdout);
|
|
1138
|
+
});
|
|
1139
|
+
child.stdin.write(stdinStr);
|
|
1140
|
+
child.stdin.end();
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
/**
|
|
1145
|
+
* Agentic summary: drive a smolagents agent (on the selected local model) over
|
|
1146
|
+
* the worker's measurement. The agent does not re-fetch — the worker's Node
|
|
1147
|
+
* measurement stays the source of truth. Throws so callers can fall back.
|
|
1148
|
+
*/
|
|
1149
|
+
async function summarizeProbeAgentic(payload, result) {
|
|
1150
|
+
const python =
|
|
1151
|
+
(process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "")
|
|
1152
|
+
.trim() || "python3";
|
|
1153
|
+
const scriptPath = join(WORKER_DIR, "gonext_probe_agent.py");
|
|
1154
|
+
const input = JSON.stringify({
|
|
1155
|
+
request: result.request,
|
|
1156
|
+
measurement: result.response,
|
|
1157
|
+
category: result.category,
|
|
1158
|
+
error: result.error ?? null,
|
|
1159
|
+
agentBaseURL: payload?.agentBaseURL ?? "",
|
|
1160
|
+
agentApiKey: payload?.agentApiKey ?? "",
|
|
1161
|
+
agentModelId: payload?.agentModelId ?? "",
|
|
1162
|
+
});
|
|
1163
|
+
const timeoutMs =
|
|
1164
|
+
(Number.isFinite(payload?.timeoutMs) ? payload.timeoutMs : 15_000) + 120_000;
|
|
1165
|
+
const stdout = await runProcessWithStdin(python, [scriptPath], input, timeoutMs);
|
|
1166
|
+
const parsed = JSON.parse(stdout);
|
|
1167
|
+
const summary =
|
|
1168
|
+
typeof parsed?.agentSummary === "string" ? parsed.agentSummary.trim() : "";
|
|
1169
|
+
if (!summary) {
|
|
1170
|
+
throw new Error(parsed?.error || "agent produced no summary");
|
|
1171
|
+
}
|
|
1172
|
+
return summary;
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
async function runHttpProbeJob(job) {
|
|
1176
|
+
const { jobId, payload } = job;
|
|
1177
|
+
const start = Date.now();
|
|
1178
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1179
|
+
method: "PATCH",
|
|
1180
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
1181
|
+
});
|
|
1182
|
+
await ensureWorkerOk(runRes, `mark running http_probe jobId=${jobId}`);
|
|
1183
|
+
try {
|
|
1184
|
+
const method = String(payload?.method || "GET").toUpperCase();
|
|
1185
|
+
const url = String(payload?.url || "");
|
|
1186
|
+
if (!url) throw new Error("http_probe payload missing url.");
|
|
1187
|
+
const headers =
|
|
1188
|
+
payload?.headers && typeof payload.headers === "object" ? payload.headers : {};
|
|
1189
|
+
const sendBody =
|
|
1190
|
+
!["GET", "HEAD"].includes(method) &&
|
|
1191
|
+
typeof payload?.body === "string" &&
|
|
1192
|
+
payload.body.length > 0;
|
|
1193
|
+
const timeoutMs =
|
|
1194
|
+
Number.isFinite(payload?.timeoutMs) && payload.timeoutMs > 0
|
|
1195
|
+
? payload.timeoutMs
|
|
1196
|
+
: 15_000;
|
|
1197
|
+
const requestBodyBytes = sendBody ? Buffer.byteLength(payload.body) : 0;
|
|
1198
|
+
const measureParams = { method, url, headers, sendBody, body: payload?.body, timeoutMs };
|
|
1199
|
+
const agentMode = payload?.agentMode === "agentic" ? "agentic" : "tool_only";
|
|
1200
|
+
|
|
1201
|
+
// The worker's Node fetch is the authoritative measurement for both modes.
|
|
1202
|
+
const measurement = await performHttpMeasurement(measureParams);
|
|
1203
|
+
const { response, category, errorMessage } = measurement;
|
|
1204
|
+
|
|
1205
|
+
const status = response?.status ?? 0;
|
|
1206
|
+
const verdict =
|
|
1207
|
+
category === "timeout" || category === "network_error"
|
|
1208
|
+
? "fail"
|
|
1209
|
+
: probeVerdict(status, payload?.expectedStatus);
|
|
1210
|
+
|
|
1211
|
+
const result = {
|
|
1212
|
+
request: { method, url, headers, bodyBytes: requestBodyBytes },
|
|
1213
|
+
response,
|
|
1214
|
+
category,
|
|
1215
|
+
verdict,
|
|
1216
|
+
agentMode,
|
|
1217
|
+
agentModel: payload?.agentModel || payload?.agentModelId || "",
|
|
1218
|
+
...(errorMessage ? { error: errorMessage } : {}),
|
|
1219
|
+
};
|
|
1220
|
+
|
|
1221
|
+
if (agentMode === "agentic") {
|
|
1222
|
+
try {
|
|
1223
|
+
result.agentSummary = await summarizeProbeAgentic(payload, result);
|
|
1224
|
+
} catch (e) {
|
|
1225
|
+
// smolagents/Python unavailable: fall back to a direct model summary.
|
|
1226
|
+
try {
|
|
1227
|
+
result.agentSummary = `${await summarizeProbeWithModel(payload, result)} (agentic fallback)`;
|
|
1228
|
+
} catch {
|
|
1229
|
+
result.agentSummary = `(Agentic summary unavailable: ${
|
|
1230
|
+
e instanceof Error ? e.message : String(e)
|
|
1231
|
+
})`;
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
} else {
|
|
1235
|
+
try {
|
|
1236
|
+
result.agentSummary = await summarizeProbeWithModel(payload, result);
|
|
1237
|
+
} catch (e) {
|
|
1238
|
+
result.agentSummary = `(Model summary unavailable: ${
|
|
1239
|
+
e instanceof Error ? e.message : String(e)
|
|
1240
|
+
})`;
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
1245
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1246
|
+
method: "PATCH",
|
|
1247
|
+
body: JSON.stringify({
|
|
1248
|
+
jobStatus: "completed",
|
|
1249
|
+
resultText: JSON.stringify(result),
|
|
1250
|
+
tokenCount: 1,
|
|
1251
|
+
totalTimeSeconds,
|
|
1252
|
+
}),
|
|
1253
|
+
});
|
|
1254
|
+
await ensureWorkerOk(doneRes, `complete http_probe jobId=${jobId}`);
|
|
1255
|
+
console.log(
|
|
1256
|
+
`[gonext-worker] completed http_probe ${jobId} (${totalTimeSeconds.toFixed(
|
|
1257
|
+
1
|
|
1258
|
+
)}s) ${method} ${url} -> ${category}/${verdict}`
|
|
1259
|
+
);
|
|
1260
|
+
} catch (e) {
|
|
1261
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
1262
|
+
const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1263
|
+
method: "PATCH",
|
|
1264
|
+
body: JSON.stringify({
|
|
1265
|
+
jobStatus: "failed",
|
|
1266
|
+
errorMessage: message,
|
|
1267
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
1268
|
+
}),
|
|
1269
|
+
});
|
|
1270
|
+
if (!failRes.ok) {
|
|
1271
|
+
const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
|
|
1272
|
+
console.error(
|
|
1273
|
+
`[gonext-worker] http_probe fail PATCH also failed ${failRes.status} jobId=${jobId}` +
|
|
1274
|
+
(snippet ? ` response=${snippet}` : "")
|
|
1275
|
+
);
|
|
1276
|
+
}
|
|
1277
|
+
console.error(`[gonext-worker] failed http_probe ${jobId}:`, message);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
function resolveImageExtension(mimeType, fileName) {
|
|
1282
|
+
const byMime = {
|
|
1283
|
+
"image/png": ".png",
|
|
1284
|
+
"image/jpeg": ".jpg",
|
|
1285
|
+
"image/jpg": ".jpg",
|
|
1286
|
+
"image/webp": ".webp",
|
|
1287
|
+
"image/gif": ".gif",
|
|
1288
|
+
"image/bmp": ".bmp",
|
|
1289
|
+
"image/tiff": ".tiff",
|
|
1290
|
+
"image/heic": ".heic",
|
|
1291
|
+
"image/heif": ".heif",
|
|
1292
|
+
};
|
|
1293
|
+
const byMimeExt = byMime[String(mimeType ?? "").toLowerCase()];
|
|
1294
|
+
if (byMimeExt) return byMimeExt;
|
|
1295
|
+
const ext = fileName ? extname(String(fileName)).toLowerCase() : "";
|
|
1296
|
+
return ext || ".png";
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
function resolveOcrModelPath(modelFromJob) {
|
|
1300
|
+
const fromJob = String(modelFromJob ?? "").trim();
|
|
1301
|
+
if (fromJob) {
|
|
1302
|
+
const expanded = fromJob.replace(/^~(?=\/)/, homedir());
|
|
1303
|
+
if (expanded.startsWith("/")) {
|
|
1304
|
+
return expanded;
|
|
1305
|
+
}
|
|
1306
|
+
return join(homedir(), "mlx-models", expanded);
|
|
1307
|
+
}
|
|
1308
|
+
const fromEnv = String(process.env.GONEXT_OCR_MODEL_PATH ?? "").trim();
|
|
1309
|
+
if (fromEnv) {
|
|
1310
|
+
return fromEnv.replace(/^~(?=\/)/, homedir());
|
|
1311
|
+
}
|
|
1312
|
+
return join(homedir(), "mlx-models", "GLM-OCR-bf16");
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
function normalizeOcrOutput(output) {
|
|
1316
|
+
let text = String(output ?? "").replace(/\r\n/g, "\n");
|
|
1317
|
+
const afterAssistant = text.includes("<|assistant|>")
|
|
1318
|
+
? text.split("<|assistant|>").pop()
|
|
1319
|
+
: "";
|
|
1320
|
+
if (afterAssistant && afterAssistant.trim()) {
|
|
1321
|
+
text = afterAssistant;
|
|
1322
|
+
}
|
|
1323
|
+
const escapedPrompt = OCR_PROMPT.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1324
|
+
text = text
|
|
1325
|
+
.replace(/<\|[^|>]+\|>/g, " ")
|
|
1326
|
+
.replace(/<think>[\s\S]*?<\/think>/gi, " ")
|
|
1327
|
+
.replace(/<think>|<\/think>/gi, " ")
|
|
1328
|
+
.replace(new RegExp(escapedPrompt, "gi"), " ")
|
|
1329
|
+
.replace(/(?:^|[\s/\\])nothink\b/gi, " ")
|
|
1330
|
+
.replace(/\\n/g, "\n");
|
|
1331
|
+
const lines = text
|
|
1332
|
+
.split(/\r?\n/)
|
|
1333
|
+
.map((line) => line.trim())
|
|
1334
|
+
.filter((line) => line.length > 0)
|
|
1335
|
+
.filter((line) => !line.startsWith("<frozen runpy>:"))
|
|
1336
|
+
.filter(
|
|
1337
|
+
(line) =>
|
|
1338
|
+
!line.startsWith("Files:") &&
|
|
1339
|
+
!line.startsWith("Prompt:") &&
|
|
1340
|
+
!line.startsWith("Generation:") &&
|
|
1341
|
+
!line.startsWith("Calling `python -m mlx_vlm.generate") &&
|
|
1342
|
+
!line.startsWith("Peak memory:") &&
|
|
1343
|
+
!line.startsWith("=======") &&
|
|
1344
|
+
line !== "<think>" &&
|
|
1345
|
+
line !== "</think>" &&
|
|
1346
|
+
line !== "<think></think>" &&
|
|
1347
|
+
line !== "No text generated for this prompt"
|
|
1348
|
+
);
|
|
1349
|
+
return lines.join("\n").trim();
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
function normalizeCorrection(raw, originalText) {
|
|
1353
|
+
let text = String(raw ?? "").replace(/\r\n/g, "\n");
|
|
1354
|
+
// Strip common mlx_lm.generate noise lines
|
|
1355
|
+
const lines = text
|
|
1356
|
+
.split(/\r?\n/)
|
|
1357
|
+
.map((l) => l.trim())
|
|
1358
|
+
.filter((l) => l.length > 0)
|
|
1359
|
+
.filter((l) =>
|
|
1360
|
+
!l.startsWith("=======") &&
|
|
1361
|
+
!l.startsWith("Prompt:") &&
|
|
1362
|
+
!l.startsWith("Generation:") &&
|
|
1363
|
+
!l.startsWith("Peak memory:") &&
|
|
1364
|
+
!l.startsWith("Files:") &&
|
|
1365
|
+
!l.startsWith("<frozen runpy>:")
|
|
1366
|
+
);
|
|
1367
|
+
const result = lines.join("\n").trim();
|
|
1368
|
+
// Safety: if correction came back empty or too short, keep original
|
|
1369
|
+
if (!result || result.length < originalText.length * 0.3) {
|
|
1370
|
+
return originalText;
|
|
1371
|
+
}
|
|
1372
|
+
return result;
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* Call an Ollama host's native chat endpoint (POST {base}/api/chat) for text
|
|
1377
|
+
* correction/translation, e.g. base "https://ollama1.gomarsic.cc" model "qwen3:14b".
|
|
1378
|
+
* `think:false` suppresses reasoning models' <think> preamble so we get clean text.
|
|
1379
|
+
*/
|
|
1380
|
+
const OCR_CORRECT_SYSTEM_PROMPT =
|
|
1381
|
+
"You are a text correction assistant. Fix grammar and language errors in the " +
|
|
1382
|
+
"provided text while preserving the original meaning and language. " +
|
|
1383
|
+
"Return only the corrected text without any explanation.";
|
|
1384
|
+
|
|
1385
|
+
/**
|
|
1386
|
+
* Call an OpenAI-compatible chat endpoint (POST {base}/v1/chat/completions) for text
|
|
1387
|
+
* correction — e.g. an MLX `mlx_lm.server` (the agent model server) at
|
|
1388
|
+
* http://127.0.0.1:8080/v1 serving qwen3:14b. Strips any <think> preamble.
|
|
1389
|
+
*/
|
|
1390
|
+
async function correctOcrTextViaOpenAI(extractedText, base, model) {
|
|
1391
|
+
const b = base.replace(/\/+$/, "");
|
|
1392
|
+
const url = /\/v1$/.test(b) ? `${b}/chat/completions` : `${b}/v1/chat/completions`;
|
|
1393
|
+
const res = await fetch(url, {
|
|
1394
|
+
method: "POST",
|
|
1395
|
+
headers: { "Content-Type": "application/json" },
|
|
1396
|
+
body: JSON.stringify({
|
|
1397
|
+
model,
|
|
1398
|
+
stream: false,
|
|
1399
|
+
temperature: 0,
|
|
1400
|
+
// Qwen3 (and other reasoning models) emit a <think> trace by default. Over the
|
|
1401
|
+
// OpenAI /v1 API there's no `think:false` flag, so use Qwen3's `/no_think` SOFT
|
|
1402
|
+
// SWITCH (honored by its chat template, same as the agent loop) to suppress it —
|
|
1403
|
+
// otherwise the model spends its whole token budget reasoning and returns no
|
|
1404
|
+
// corrected text (seen live: correction was a no-op, chars unchanged). Generous
|
|
1405
|
+
// max_tokens so a long OCR page isn't truncated.
|
|
1406
|
+
max_tokens: 4096,
|
|
1407
|
+
messages: [
|
|
1408
|
+
{ role: "system", content: `${OCR_CORRECT_SYSTEM_PROMPT}\n/no_think` },
|
|
1409
|
+
{ role: "user", content: `Text:\n${extractedText}` },
|
|
1410
|
+
],
|
|
1411
|
+
}),
|
|
1412
|
+
signal: AbortSignal.timeout(OCR_CORRECT_TIMEOUT_MS),
|
|
1413
|
+
});
|
|
1414
|
+
if (!res.ok) {
|
|
1415
|
+
throw new Error(`MLX ${b} returned ${res.status}`);
|
|
1416
|
+
}
|
|
1417
|
+
const json = await res.json();
|
|
1418
|
+
let corrected = json?.choices?.[0]?.message?.content?.trim() || "";
|
|
1419
|
+
// Strip a <think> trace if `/no_think` wasn't honored: closed pairs, a template-opened
|
|
1420
|
+
// leading …</think>, and a trailing UNCLOSED <think> (truncated by max_tokens).
|
|
1421
|
+
corrected = corrected
|
|
1422
|
+
.replace(/<think>[\s\S]*?<\/think>/gi, "")
|
|
1423
|
+
.replace(/^[\s\S]*?<\/think>/i, "")
|
|
1424
|
+
.replace(/<think>[\s\S]*$/i, "")
|
|
1425
|
+
.trim();
|
|
1426
|
+
return normalizeCorrection(corrected, extractedText);
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
async function correctOcrTextViaOllama(extractedText, base, model) {
|
|
1430
|
+
const systemPrompt = OCR_CORRECT_SYSTEM_PROMPT;
|
|
1431
|
+
const res = await fetch(`${base.replace(/\/+$/, "")}/api/chat`, {
|
|
1432
|
+
method: "POST",
|
|
1433
|
+
headers: { "Content-Type": "application/json" },
|
|
1434
|
+
body: JSON.stringify({
|
|
1435
|
+
model,
|
|
1436
|
+
stream: false,
|
|
1437
|
+
think: false,
|
|
1438
|
+
options: { temperature: 0, num_ctx: OCR_TRANSLATE_NUM_CTX },
|
|
1439
|
+
messages: [
|
|
1440
|
+
{ role: "system", content: systemPrompt },
|
|
1441
|
+
{ role: "user", content: `Text:\n${extractedText}` },
|
|
1442
|
+
],
|
|
1443
|
+
}),
|
|
1444
|
+
signal: AbortSignal.timeout(OCR_CORRECT_TIMEOUT_MS),
|
|
1445
|
+
});
|
|
1446
|
+
if (!res.ok) {
|
|
1447
|
+
throw new Error(`Ollama ${base} returned ${res.status}`);
|
|
1448
|
+
}
|
|
1449
|
+
const json = await res.json();
|
|
1450
|
+
let corrected = json?.message?.content?.trim() || "";
|
|
1451
|
+
// Safety net for reasoning models (qwen3) if `think:false` isn't honored:
|
|
1452
|
+
// drop any <think>…</think> preamble before normalizing.
|
|
1453
|
+
corrected = corrected.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
1454
|
+
return normalizeCorrection(corrected, extractedText);
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
async function correctOcrText(extractedText, correction) {
|
|
1458
|
+
if (OCR_SKIP_CORRECTION) {
|
|
1459
|
+
console.log("[gonext-worker] OCR correction skipped (GONEXT_OCR_SKIP_CORRECTION=1)");
|
|
1460
|
+
return extractedText;
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
// The API resolves the backend from the user's settings and bakes {backend,url,model}
|
|
1464
|
+
// into the job payload — DEFAULT is "mlx" (the agent MLX model server, OpenAI /v1).
|
|
1465
|
+
// Absent (older jobs / no settings) → fall back to the legacy managed-Ollama default.
|
|
1466
|
+
const backend = correction?.backend === "mlx" ? "mlx" : "ollama";
|
|
1467
|
+
const url = (correction?.url || "").trim() || OCR_TRANSLATE_OLLAMA_URL;
|
|
1468
|
+
const model = (correction?.model || "").trim() || OCR_TRANSLATE_OLLAMA_MODEL;
|
|
1469
|
+
console.log(
|
|
1470
|
+
`[gonext-worker] OCR correction via ${backend} ${url} model=${model}`
|
|
1471
|
+
);
|
|
1472
|
+
console.log(`[gonext-worker] OCR correction input: ${extractedText.slice(0, 300)}`);
|
|
1473
|
+
try {
|
|
1474
|
+
const corrected =
|
|
1475
|
+
backend === "mlx"
|
|
1476
|
+
? await correctOcrTextViaOpenAI(extractedText, url, model)
|
|
1477
|
+
: await correctOcrTextViaOllama(extractedText, url, model);
|
|
1478
|
+
console.log(`[gonext-worker] OCR correction output: ${corrected.slice(0, 300)}`);
|
|
1479
|
+
console.log(
|
|
1480
|
+
`[gonext-worker] OCR correction done: ${extractedText.length} → ${corrected.length} chars`
|
|
1481
|
+
);
|
|
1482
|
+
return corrected;
|
|
1483
|
+
} catch (e) {
|
|
1484
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
1485
|
+
console.warn(
|
|
1486
|
+
`[gonext-worker] OCR correction (${backend}) failed (using raw OCR text): ${msg.slice(0, 200)}`
|
|
1487
|
+
);
|
|
1488
|
+
return extractedText;
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
async function runMlxVlmGenerate(modelPath, imagePath) {
|
|
1493
|
+
const sharedArgs = [
|
|
1494
|
+
"--model",
|
|
1495
|
+
modelPath,
|
|
1496
|
+
"--prompt",
|
|
1497
|
+
OCR_PROMPT,
|
|
1498
|
+
"--image",
|
|
1499
|
+
imagePath,
|
|
1500
|
+
"--temperature",
|
|
1501
|
+
"0.0",
|
|
1502
|
+
"--max-tokens",
|
|
1503
|
+
String(OCR_MAX_TOKENS),
|
|
1504
|
+
];
|
|
1505
|
+
try {
|
|
1506
|
+
const result = await execFile(
|
|
1507
|
+
"python3",
|
|
1508
|
+
["-m", "mlx_vlm.generate", ...sharedArgs],
|
|
1509
|
+
{
|
|
1510
|
+
timeout: OCR_TIMEOUT_MS,
|
|
1511
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
1512
|
+
}
|
|
1513
|
+
);
|
|
1514
|
+
return { ...result, invocation: "python3 -m mlx_vlm.generate" };
|
|
1515
|
+
} catch (primaryError) {
|
|
1516
|
+
const stderr =
|
|
1517
|
+
primaryError && typeof primaryError === "object" && "stderr" in primaryError
|
|
1518
|
+
? String(primaryError.stderr ?? "").toLowerCase()
|
|
1519
|
+
: "";
|
|
1520
|
+
const message =
|
|
1521
|
+
primaryError instanceof Error ? primaryError.message.toLowerCase() : "";
|
|
1522
|
+
const missingLegacyModule =
|
|
1523
|
+
stderr.includes("no module named mlx_vlm.generate") ||
|
|
1524
|
+
message.includes("no module named mlx_vlm.generate");
|
|
1525
|
+
if (!missingLegacyModule) {
|
|
1526
|
+
throw primaryError;
|
|
1527
|
+
}
|
|
1528
|
+
const result = await execFile(
|
|
1529
|
+
"python3",
|
|
1530
|
+
["-m", "mlx_vlm", "generate", ...sharedArgs],
|
|
1531
|
+
{
|
|
1532
|
+
timeout: OCR_TIMEOUT_MS,
|
|
1533
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
1534
|
+
}
|
|
1535
|
+
);
|
|
1536
|
+
return { ...result, invocation: "python3 -m mlx_vlm generate" };
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
function compactPreview(text, max = 800) {
|
|
1541
|
+
const normalized = String(text ?? "")
|
|
1542
|
+
.replace(/\r\n/g, "\n")
|
|
1543
|
+
.replace(/\s+/g, " ")
|
|
1544
|
+
.trim();
|
|
1545
|
+
if (!normalized) {
|
|
1546
|
+
return "(empty)";
|
|
1547
|
+
}
|
|
1548
|
+
if (normalized.length <= max) {
|
|
1549
|
+
return normalized;
|
|
1550
|
+
}
|
|
1551
|
+
return `${normalized.slice(0, max)} …[truncated ${normalized.length - max} chars]`;
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
async function runOcrJob(job) {
|
|
1555
|
+
const { jobId, payload } = job;
|
|
1556
|
+
const start = Date.now();
|
|
1557
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1558
|
+
method: "PATCH",
|
|
1559
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
1560
|
+
});
|
|
1561
|
+
await ensureWorkerOk(runRes, `mark running OCR jobId=${jobId}`);
|
|
1562
|
+
try {
|
|
1563
|
+
if (!payload || typeof payload !== "object") {
|
|
1564
|
+
throw new Error("Invalid OCR payload.");
|
|
1565
|
+
}
|
|
1566
|
+
const attachment = payload.attachment;
|
|
1567
|
+
const mimeType = typeof attachment?.mimeType === "string" ? attachment.mimeType : "";
|
|
1568
|
+
const data = typeof attachment?.data === "string" ? attachment.data : "";
|
|
1569
|
+
const s3Object =
|
|
1570
|
+
attachment && typeof attachment === "object" ? attachment.s3Object : undefined;
|
|
1571
|
+
const s3GetUrl =
|
|
1572
|
+
s3Object && typeof s3Object.getUrl === "string" ? s3Object.getUrl.trim() : "";
|
|
1573
|
+
const sourceLabel = s3GetUrl
|
|
1574
|
+
? `s3://${s3Object?.bucket ?? "unknown"}/${s3Object?.key ?? "unknown"}`
|
|
1575
|
+
: "inline_base64";
|
|
1576
|
+
const ocrModel =
|
|
1577
|
+
typeof payload.model === "string" && payload.model.trim()
|
|
1578
|
+
? payload.model.trim()
|
|
1579
|
+
: "GLM-OCR-bf16";
|
|
1580
|
+
const name = typeof attachment?.name === "string" ? attachment.name : "";
|
|
1581
|
+
if (!mimeType.startsWith("image/")) {
|
|
1582
|
+
throw new Error("OCR job attachment must be an image.");
|
|
1583
|
+
}
|
|
1584
|
+
if (!data && !s3GetUrl) {
|
|
1585
|
+
throw new Error("OCR job attachment is empty.");
|
|
1586
|
+
}
|
|
1587
|
+
let bytes;
|
|
1588
|
+
if (s3GetUrl) {
|
|
1589
|
+
const dlRes = await fetch(s3GetUrl, { method: "GET" });
|
|
1590
|
+
if (!dlRes.ok) {
|
|
1591
|
+
throw new Error(
|
|
1592
|
+
`Failed to download OCR image from S3 (${dlRes.status}).`
|
|
1593
|
+
);
|
|
1594
|
+
}
|
|
1595
|
+
const arr = await dlRes.arrayBuffer();
|
|
1596
|
+
bytes = Buffer.from(arr);
|
|
1597
|
+
} else {
|
|
1598
|
+
bytes = Buffer.from(data, "base64");
|
|
1599
|
+
}
|
|
1600
|
+
if (!bytes.length) {
|
|
1601
|
+
throw new Error("OCR job attachment data is not valid.");
|
|
1602
|
+
}
|
|
1603
|
+
console.log(`[gonext-worker] OCR source jobId=${jobId} ${sourceLabel}`);
|
|
1604
|
+
|
|
1605
|
+
const tempDir = await mkdtemp(join(tmpdir(), "gonext-ocr-worker-"));
|
|
1606
|
+
let extractedText = "";
|
|
1607
|
+
try {
|
|
1608
|
+
const imagePath = join(
|
|
1609
|
+
tempDir,
|
|
1610
|
+
`input${resolveImageExtension(mimeType, name)}`
|
|
1611
|
+
);
|
|
1612
|
+
await writeFile(imagePath, bytes);
|
|
1613
|
+
const modelPath = resolveOcrModelPath(ocrModel);
|
|
1614
|
+
const { stdout, stderr, invocation } = await runMlxVlmGenerate(
|
|
1615
|
+
modelPath,
|
|
1616
|
+
imagePath
|
|
1617
|
+
);
|
|
1618
|
+
if (OCR_DEBUG) {
|
|
1619
|
+
console.log(
|
|
1620
|
+
`[gonext-worker][ocr-debug] jobId=${jobId} invocation=${invocation} model=${modelPath} image=${imagePath} source=${
|
|
1621
|
+
sourceLabel
|
|
1622
|
+
} stdout=${compactPreview(stdout)} stderr=${compactPreview(stderr)}`
|
|
1623
|
+
);
|
|
1624
|
+
}
|
|
1625
|
+
extractedText = normalizeOcrOutput(stdout);
|
|
1626
|
+
if (!extractedText) {
|
|
1627
|
+
throw new Error(
|
|
1628
|
+
`OCR returned empty text for this image. invocation=${invocation} stdout=${compactPreview(stdout)} stderr=${compactPreview(stderr)}`
|
|
1629
|
+
);
|
|
1630
|
+
}
|
|
1631
|
+
} finally {
|
|
1632
|
+
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
|
1633
|
+
}
|
|
1634
|
+
// Post-process: correct grammar/language via the backend the API resolved from the
|
|
1635
|
+
// user's settings (payload.correction = {backend,url,model}; default MLX = the agent
|
|
1636
|
+
// model server). Falls back to raw OCR text on any error so the job never fails here.
|
|
1637
|
+
extractedText = await correctOcrText(extractedText, payload.correction);
|
|
1638
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
1639
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1640
|
+
method: "PATCH",
|
|
1641
|
+
body: JSON.stringify({
|
|
1642
|
+
jobStatus: "completed",
|
|
1643
|
+
resultText: extractedText,
|
|
1644
|
+
tokenCount: Math.max(1, Math.ceil(extractedText.length / 4)),
|
|
1645
|
+
totalTimeSeconds,
|
|
1646
|
+
}),
|
|
1647
|
+
});
|
|
1648
|
+
await ensureWorkerOk(doneRes, `complete OCR jobId=${jobId}`);
|
|
1649
|
+
console.log(
|
|
1650
|
+
`[gonext-worker] completed OCR ${jobId} (${totalTimeSeconds.toFixed(1)}s) model=${ocrModel}`
|
|
1651
|
+
);
|
|
1652
|
+
} catch (e) {
|
|
1653
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
1654
|
+
const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1655
|
+
method: "PATCH",
|
|
1656
|
+
body: JSON.stringify({
|
|
1657
|
+
jobStatus: "failed",
|
|
1658
|
+
errorMessage: message,
|
|
1659
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
1660
|
+
}),
|
|
1661
|
+
});
|
|
1662
|
+
if (!failRes.ok) {
|
|
1663
|
+
const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
|
|
1664
|
+
console.error(
|
|
1665
|
+
`[gonext-worker] OCR fail PATCH also failed ${failRes.status} jobId=${jobId}` +
|
|
1666
|
+
(snippet ? ` response=${snippet}` : "")
|
|
1667
|
+
);
|
|
1668
|
+
}
|
|
1669
|
+
console.error(`[gonext-worker] failed OCR ${jobId}:`, message);
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
const TRANSCRIBE_TIMEOUT_MS =
|
|
1674
|
+
Number(process.env.GONEXT_TRANSCRIBE_TIMEOUT_MS ?? "300000") || 300000;
|
|
1675
|
+
|
|
1676
|
+
function resolveWorkerPython() {
|
|
1677
|
+
return (
|
|
1678
|
+
(process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "").trim() ||
|
|
1679
|
+
"python3"
|
|
1680
|
+
);
|
|
1681
|
+
}
|
|
1682
|
+
|
|
1683
|
+
/** Spawn a Python script, write `input` to stdin, capture stdout/stderr. */
|
|
1684
|
+
function runPythonCapture(python, args, input, timeoutMs) {
|
|
1685
|
+
return new Promise((resolve) => {
|
|
1686
|
+
const child = spawn(python, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
1687
|
+
let stdout = "";
|
|
1688
|
+
let stderr = "";
|
|
1689
|
+
let settled = false;
|
|
1690
|
+
const timer = setTimeout(() => {
|
|
1691
|
+
if (settled) return;
|
|
1692
|
+
settled = true;
|
|
1693
|
+
try {
|
|
1694
|
+
child.kill("SIGKILL");
|
|
1695
|
+
} catch {
|
|
1696
|
+
/* ignore */
|
|
1697
|
+
}
|
|
1698
|
+
resolve({ code: null, stdout, stderr, timedOut: true });
|
|
1699
|
+
}, timeoutMs);
|
|
1700
|
+
child.stdout.on("data", (d) => {
|
|
1701
|
+
stdout += d.toString();
|
|
1702
|
+
});
|
|
1703
|
+
child.stderr.on("data", (d) => {
|
|
1704
|
+
stderr += d.toString();
|
|
1705
|
+
});
|
|
1706
|
+
child.on("error", (e) => {
|
|
1707
|
+
if (settled) return;
|
|
1708
|
+
settled = true;
|
|
1709
|
+
clearTimeout(timer);
|
|
1710
|
+
resolve({ code: null, stdout, stderr: stderr + String(e), timedOut: false });
|
|
1711
|
+
});
|
|
1712
|
+
child.on("close", (code) => {
|
|
1713
|
+
if (settled) return;
|
|
1714
|
+
settled = true;
|
|
1715
|
+
clearTimeout(timer);
|
|
1716
|
+
resolve({ code, stdout, stderr, timedOut: false });
|
|
1717
|
+
});
|
|
1718
|
+
try {
|
|
1719
|
+
child.stdin.write(input);
|
|
1720
|
+
child.stdin.end();
|
|
1721
|
+
} catch {
|
|
1722
|
+
/* the error/close handler will settle */
|
|
1723
|
+
}
|
|
1724
|
+
});
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
async function runTranscribeJob(job) {
|
|
1728
|
+
const { jobId, payload } = job;
|
|
1729
|
+
const start = Date.now();
|
|
1730
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1731
|
+
method: "PATCH",
|
|
1732
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
1733
|
+
});
|
|
1734
|
+
await ensureWorkerOk(runRes, `mark running transcribe jobId=${jobId}`);
|
|
1735
|
+
const tempDir = await mkdtemp(join(tmpdir(), "gonext-transcribe-"));
|
|
1736
|
+
try {
|
|
1737
|
+
if (!payload || typeof payload !== "object") {
|
|
1738
|
+
throw new Error("Invalid transcribe payload.");
|
|
1739
|
+
}
|
|
1740
|
+
const audio = payload.audio;
|
|
1741
|
+
const mimeType = typeof audio?.mimeType === "string" ? audio.mimeType : "";
|
|
1742
|
+
const s3Object = audio && typeof audio === "object" ? audio.s3Object : undefined;
|
|
1743
|
+
const s3GetUrl =
|
|
1744
|
+
s3Object && typeof s3Object.getUrl === "string" ? s3Object.getUrl.trim() : "";
|
|
1745
|
+
const model =
|
|
1746
|
+
typeof payload.model === "string" && payload.model.trim()
|
|
1747
|
+
? payload.model.trim()
|
|
1748
|
+
: "whisper-large-v3-turbo";
|
|
1749
|
+
const language =
|
|
1750
|
+
typeof payload.language === "string" ? payload.language.trim() : "";
|
|
1751
|
+
if (!mimeType.startsWith("audio/")) {
|
|
1752
|
+
throw new Error("Transcribe job attachment must be audio.");
|
|
1753
|
+
}
|
|
1754
|
+
if (!s3GetUrl) {
|
|
1755
|
+
throw new Error("Transcribe job is missing the audio download URL.");
|
|
1756
|
+
}
|
|
1757
|
+
const dlRes = await fetch(s3GetUrl, { method: "GET" });
|
|
1758
|
+
if (!dlRes.ok) {
|
|
1759
|
+
throw new Error(`Failed to download audio from S3 (${dlRes.status}).`);
|
|
1760
|
+
}
|
|
1761
|
+
const bytes = Buffer.from(await dlRes.arrayBuffer());
|
|
1762
|
+
if (!bytes.length) {
|
|
1763
|
+
throw new Error("Downloaded audio is empty.");
|
|
1764
|
+
}
|
|
1765
|
+
const audioPath = join(tempDir, "input.wav");
|
|
1766
|
+
await writeFile(audioPath, bytes);
|
|
1767
|
+
|
|
1768
|
+
const python = resolveWorkerPython();
|
|
1769
|
+
const scriptPath = join(WORKER_DIR, "gonext_transcribe.py");
|
|
1770
|
+
const { code, stdout, stderr, timedOut } = await runPythonCapture(
|
|
1771
|
+
python,
|
|
1772
|
+
[scriptPath],
|
|
1773
|
+
JSON.stringify({ audioPath, model, language: language || undefined }),
|
|
1774
|
+
TRANSCRIBE_TIMEOUT_MS
|
|
1775
|
+
);
|
|
1776
|
+
if (timedOut) {
|
|
1777
|
+
throw new Error("Transcription timed out.");
|
|
1778
|
+
}
|
|
1779
|
+
if (code !== 0) {
|
|
1780
|
+
const reason = String(stderr ?? "").split("\n")[0]?.trim() || "error";
|
|
1781
|
+
const hint =
|
|
1782
|
+
reason === "lib-missing"
|
|
1783
|
+
? "Whisper library not installed. Run: pip install mlx-whisper"
|
|
1784
|
+
: reason === "model-missing"
|
|
1785
|
+
? `Whisper model not downloaded. Run: hf download mlx-community/${model}`
|
|
1786
|
+
: reason === "audio-error"
|
|
1787
|
+
? "Could not decode the recorded audio."
|
|
1788
|
+
: "Transcription failed.";
|
|
1789
|
+
throw new Error(`${hint} (${compactPreview(stderr)})`);
|
|
1790
|
+
}
|
|
1791
|
+
let text = "";
|
|
1792
|
+
try {
|
|
1793
|
+
const parsed = JSON.parse(String(stdout ?? "").trim() || "{}");
|
|
1794
|
+
text = typeof parsed.text === "string" ? parsed.text.trim() : "";
|
|
1795
|
+
} catch {
|
|
1796
|
+
throw new Error(`Transcription returned malformed output: ${compactPreview(stdout)}`);
|
|
1797
|
+
}
|
|
1798
|
+
if (!text) {
|
|
1799
|
+
throw new Error("Transcription returned empty text.");
|
|
1800
|
+
}
|
|
1801
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
1802
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1803
|
+
method: "PATCH",
|
|
1804
|
+
body: JSON.stringify({
|
|
1805
|
+
jobStatus: "completed",
|
|
1806
|
+
resultText: text,
|
|
1807
|
+
tokenCount: Math.max(1, Math.ceil(text.length / 4)),
|
|
1808
|
+
totalTimeSeconds,
|
|
1809
|
+
}),
|
|
1810
|
+
});
|
|
1811
|
+
await ensureWorkerOk(doneRes, `complete transcribe jobId=${jobId}`);
|
|
1812
|
+
console.log(
|
|
1813
|
+
`[gonext-worker] completed transcribe ${jobId} (${totalTimeSeconds.toFixed(1)}s) model=${model}`
|
|
1814
|
+
);
|
|
1815
|
+
} catch (e) {
|
|
1816
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
1817
|
+
const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1818
|
+
method: "PATCH",
|
|
1819
|
+
body: JSON.stringify({
|
|
1820
|
+
jobStatus: "failed",
|
|
1821
|
+
errorMessage: message,
|
|
1822
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
1823
|
+
}),
|
|
1824
|
+
});
|
|
1825
|
+
if (!failRes.ok) {
|
|
1826
|
+
const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
|
|
1827
|
+
console.error(
|
|
1828
|
+
`[gonext-worker] transcribe fail PATCH also failed ${failRes.status} jobId=${jobId}` +
|
|
1829
|
+
(snippet ? ` response=${snippet}` : "")
|
|
1830
|
+
);
|
|
1831
|
+
}
|
|
1832
|
+
console.error(`[gonext-worker] failed transcribe ${jobId}:`, message);
|
|
1833
|
+
} finally {
|
|
1834
|
+
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
// —— Ollama coding-model keep-warm ——————————————————————————————————————————————
|
|
1839
|
+
// Ollama evicts idle models (default OLLAMA_KEEP_ALIVE=5m), so a cold agent step
|
|
1840
|
+
// blocks ~20s+ while llama-server pages the weights into VRAM. Instead of warming on
|
|
1841
|
+
// EVERY question (noisy, redundant), we remember the coding model from agent jobs and
|
|
1842
|
+
// re-assert a keep_alive:-1 load on a BACKGROUND INTERVAL so it stays resident between
|
|
1843
|
+
// questions. The agent talks to Ollama over /v1 (which ignores keep_alive), so we hit
|
|
1844
|
+
// the NATIVE /api/generate (no prompt = load-only, returns once the model is resident).
|
|
1845
|
+
// Best-effort throughout: a non-Ollama coding server just 404s and it's swallowed.
|
|
1846
|
+
let _warmCoding = null; // { root, model }
|
|
1847
|
+
let _warmTimer = null;
|
|
1848
|
+
let _warmLoggedOk = false;
|
|
1849
|
+
|
|
1850
|
+
function _warmupPing(quiet) {
|
|
1851
|
+
if (!_warmCoding) return;
|
|
1852
|
+
const { root, model } = _warmCoding;
|
|
1853
|
+
const url = `${root}/api/generate`;
|
|
1854
|
+
fetch(url, {
|
|
1855
|
+
method: "POST",
|
|
1856
|
+
headers: { "Content-Type": "application/json" },
|
|
1857
|
+
body: JSON.stringify({ model, keep_alive: -1 }),
|
|
1858
|
+
signal: AbortSignal.timeout(AGENT_WARMUP_TIMEOUT_MS),
|
|
1859
|
+
})
|
|
1860
|
+
.then((res) => {
|
|
1861
|
+
if (!res.ok) {
|
|
1862
|
+
console.log(`[gonext-worker] agent warm-up HTTP ${res.status} → ${url} model=${model}`);
|
|
1863
|
+
} else if (!quiet || !_warmLoggedOk) {
|
|
1864
|
+
// Quiet interval pings only log the first success per model; a fresh/changed
|
|
1865
|
+
// model always logs so you can see it warm.
|
|
1866
|
+
console.log(`[gonext-worker] agent warm-up ok (model loading/resident) → ${url} model=${model}`);
|
|
1867
|
+
_warmLoggedOk = true;
|
|
1868
|
+
}
|
|
1869
|
+
})
|
|
1870
|
+
.catch((err) => {
|
|
1871
|
+
const reason =
|
|
1872
|
+
err?.name === "TimeoutError" ? "still loading in background" : err?.message || "unreachable";
|
|
1873
|
+
console.log(`[gonext-worker] agent warm-up skipped (${reason}) → ${url}`);
|
|
1874
|
+
});
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
// ─── Coding-backend resolution (task #117, folded into #123 Part B) ──────────────────
|
|
1878
|
+
// ONE answer to "which coding backend is this job using?", for the two decisions THIS
|
|
1879
|
+
// process makes before python starts: keep an Ollama model warm, and fetch the
|
|
1880
|
+
// OpenAI-compatible API key. The answer is passed down as `codingBackend` so python
|
|
1881
|
+
// doesn't re-derive it and can't disagree.
|
|
1882
|
+
//
|
|
1883
|
+
// The reference implementation is _resolve_coding_backend() in gonext_agent_chat.py —
|
|
1884
|
+
// it is pure and carries the offline resolution table (tests/test_backend_resolution.py).
|
|
1885
|
+
// Keep this rule in step with it: explicit kind wins → Ollama probe → remote ⇒ openai →
|
|
1886
|
+
// local. The probe runs BEFORE the remote rule so a remote Ollama box stays "ollama".
|
|
1887
|
+
|
|
1888
|
+
// root → { value, until }. A YES is cached long (a box doesn't stop being Ollama), a NO
|
|
1889
|
+
// only briefly: "not Ollama" and "the probe failed" are indistinguishable here, so caching
|
|
1890
|
+
// a failure would strand a box that was merely booting/unreachable for one job.
|
|
1891
|
+
const _ollamaProbeCache = new Map();
|
|
1892
|
+
const _PROBE_TTL_YES_MS = 30 * 60_000;
|
|
1893
|
+
const _PROBE_TTL_NO_MS = 60_000;
|
|
1894
|
+
|
|
1895
|
+
async function _probeIsOllama(baseURL) {
|
|
1896
|
+
const root = String(baseURL || "").replace(/\/+$/, "").replace(/\/v1$/i, "");
|
|
1897
|
+
if (!root) return false;
|
|
1898
|
+
const hit = _ollamaProbeCache.get(root);
|
|
1899
|
+
if (hit && hit.until > Date.now()) return hit.value;
|
|
1900
|
+
let isOllama = false;
|
|
1901
|
+
try {
|
|
1902
|
+
// Ollama answers GET / with the plain-text banner "Ollama is running".
|
|
1903
|
+
const res = await fetch(root, {
|
|
1904
|
+
headers: { "User-Agent": "gonext-worker/1.0" },
|
|
1905
|
+
signal: AbortSignal.timeout(6000),
|
|
1906
|
+
});
|
|
1907
|
+
isOllama = res.ok && /ollama/i.test((await res.text()).slice(0, 200));
|
|
1908
|
+
} catch {
|
|
1909
|
+
isOllama = false; // unreachable/TLS/404 → not provably Ollama
|
|
1910
|
+
}
|
|
1911
|
+
_ollamaProbeCache.set(root, {
|
|
1912
|
+
value: isOllama,
|
|
1913
|
+
until: Date.now() + (isOllama ? _PROBE_TTL_YES_MS : _PROBE_TTL_NO_MS),
|
|
1914
|
+
});
|
|
1915
|
+
return isOllama;
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
function _isLocalCodingUrl(baseURL) {
|
|
1919
|
+
let host = "";
|
|
1920
|
+
try {
|
|
1921
|
+
host = new URL(String(baseURL || "")).hostname.toLowerCase().replace(/\.$/, "");
|
|
1922
|
+
} catch {
|
|
1923
|
+
return true; // unparseable → we are certainly not talking to a paid remote
|
|
1924
|
+
}
|
|
1925
|
+
if (!host) return true;
|
|
1926
|
+
if (host.startsWith("[") && host.endsWith("]")) host = host.slice(1, -1);
|
|
1927
|
+
if (["localhost", "0.0.0.0", "::", "::1"].includes(host)) return true;
|
|
1928
|
+
if (/^127(\.\d{1,3}){3}$/.test(host)) return true;
|
|
1929
|
+
if (/^(10(\.\d{1,3}){3}|192\.168(\.\d{1,3}){2}|172\.(1[6-9]|2\d|3[01])(\.\d{1,3}){2}|169\.254(\.\d{1,3}){2})$/.test(host))
|
|
1930
|
+
return true;
|
|
1931
|
+
if (host.includes(":") && (host.startsWith("fe80:") || /^f[cd]/.test(host))) return true;
|
|
1932
|
+
if (/\.(local|localdomain|lan|home|internal)$/.test(host)) return true;
|
|
1933
|
+
// A dotless hostname ("mac-studio", "ollama1") is a LAN box: a public host always has a
|
|
1934
|
+
// dot, and erring this way only ever means "don't expect an API key here".
|
|
1935
|
+
return !host.includes(".") && !host.includes(":");
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1938
|
+
/** Resolve to "ollama" | "openai" | "local". `kind` is the user's Settings choice
|
|
1939
|
+
* ("ollama" | "openai" | "" = Auto). Only Auto costs a probe. */
|
|
1940
|
+
async function resolveCodingBackend(kind, baseURL) {
|
|
1941
|
+
if (kind === "ollama" || kind === "openai" || kind === "local") return kind;
|
|
1942
|
+
if (await _probeIsOllama(baseURL)) return "ollama";
|
|
1943
|
+
return _isLocalCodingUrl(baseURL) ? "local" : "openai";
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
/**
|
|
1947
|
+
* Remember the agent's Ollama coding model (from a claimed agent_chat job) and keep it
|
|
1948
|
+
* resident via a background interval. Warms IMMEDIATELY only when the model is first
|
|
1949
|
+
* seen or CHANGES — not on every question — so repeat questions don't re-warm/re-log.
|
|
1950
|
+
*/
|
|
1951
|
+
function trackCodingModelForWarmup(codingBaseURL, codingModelId) {
|
|
1952
|
+
if (!codingBaseURL || !codingModelId) return;
|
|
1953
|
+
// codingBaseURL is normalized to …/v1 for OpenAI-compat; strip it to reach the
|
|
1954
|
+
// Ollama root where /api/generate lives.
|
|
1955
|
+
const root = String(codingBaseURL).replace(/\/+$/, "").replace(/\/v1$/i, "");
|
|
1956
|
+
const changed = !_warmCoding || _warmCoding.root !== root || _warmCoding.model !== codingModelId;
|
|
1957
|
+
_warmCoding = { root, model: codingModelId };
|
|
1958
|
+
if (changed) {
|
|
1959
|
+
_warmLoggedOk = false;
|
|
1960
|
+
_warmupPing(false); // warm the newly-seen/changed model now so the next question isn't cold
|
|
1961
|
+
}
|
|
1962
|
+
if (!_warmTimer) {
|
|
1963
|
+
// 4 min — under Ollama's 5-min default unload, so the model stays resident even
|
|
1964
|
+
// if a proxy strips keep_alive. unref() so this timer never blocks worker exit.
|
|
1965
|
+
_warmTimer = setInterval(() => _warmupPing(true), AGENT_WARMUP_INTERVAL_MS);
|
|
1966
|
+
if (typeof _warmTimer.unref === "function") _warmTimer.unref();
|
|
1967
|
+
}
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
async function runAgentChatJob(job) {
|
|
1971
|
+
const { jobId, payload } = job;
|
|
1972
|
+
const start = Date.now();
|
|
1973
|
+
// User-requested cancel (gonext Ctrl+C): the REPL marks the job "cancelled" via the
|
|
1974
|
+
// API; a poll below (and the job-chunk 409 path) trips this controller, which kills
|
|
1975
|
+
// the Python agent. `cancelled` is checked at the finish so we DON'T overwrite the
|
|
1976
|
+
// "cancelled" status with completed/failed.
|
|
1977
|
+
const cancelAc = new AbortController();
|
|
1978
|
+
let cancelled = false;
|
|
1979
|
+
// Task #108: the user's Coding backend setting ("ollama" | "openai" | "" = Auto).
|
|
1980
|
+
const codingKind = ["ollama", "openai", "local"].includes(payload?.codingKind)
|
|
1981
|
+
? payload.codingKind
|
|
1982
|
+
: "";
|
|
1983
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
1984
|
+
method: "PATCH",
|
|
1985
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
1986
|
+
});
|
|
1987
|
+
await ensureWorkerOk(runRes, `mark running agent_chat jobId=${jobId}`);
|
|
1988
|
+
// Task #117/#123: `codingKind` above is the SETTING, not the answer — resolve the actual
|
|
1989
|
+
// backend ONCE here and let both decisions below, plus python, read that one value.
|
|
1990
|
+
// AFTER the "running" PATCH on purpose: an Auto config costs a probe (~6s worst case,
|
|
1991
|
+
// then cached), and the REPL shouldn't watch a claimed job sit in "queued" for it.
|
|
1992
|
+
const codingBackend = await resolveCodingBackend(codingKind, payload?.codingBaseURL ?? "");
|
|
1993
|
+
console.log(
|
|
1994
|
+
`[gonext-worker] agent job ${jobId}: coding backend ${codingBackend}` +
|
|
1995
|
+
` (kind=${codingKind || "auto"}, ${payload?.codingBaseURL || "no coding url"})`
|
|
1996
|
+
);
|
|
1997
|
+
// Remember this job's Ollama coding model and keep it resident via a background
|
|
1998
|
+
// interval. Only warms now if the model is new/changed — repeat questions rely on
|
|
1999
|
+
// the interval, so we don't re-warm (or re-log) on every question. The warm-up uses
|
|
2000
|
+
// Ollama's native /api/generate, so it runs ONLY for an actual Ollama backend: on
|
|
2001
|
+
// anything else that endpoint doesn't exist, and on a cloud one we'd be pinging a paid
|
|
2002
|
+
// API every 4 minutes (which an Auto+cloud config did until #117 was fixed — the old
|
|
2003
|
+
// gate only skipped an EXPLICIT openai kind).
|
|
2004
|
+
if (codingBackend === "ollama") {
|
|
2005
|
+
trackCodingModelForWarmup(payload?.codingBaseURL ?? "", payload?.codingModelId ?? "");
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
let buf = "";
|
|
2009
|
+
let flushTimer = null;
|
|
2010
|
+
let fullText = "";
|
|
2011
|
+
let flushTail = Promise.resolve();
|
|
2012
|
+
const CHUNK_DEBOUNCE_MS = 80;
|
|
2013
|
+
const CHUNK_MAX_BUF = 6144;
|
|
2014
|
+
|
|
2015
|
+
const flushChunks = async () => {
|
|
2016
|
+
const t = buf;
|
|
2017
|
+
buf = "";
|
|
2018
|
+
if (!t) return;
|
|
2019
|
+
const res = await workerFetch(CHUNK_PATH, {
|
|
2020
|
+
method: "POST",
|
|
2021
|
+
body: JSON.stringify({ jobId, text: t }),
|
|
2022
|
+
});
|
|
2023
|
+
if (!res.ok && res.status !== 204) {
|
|
2024
|
+
const snippet = (await res.text().catch(() => "")).trim().slice(0, 400);
|
|
2025
|
+
const cancelledNow =
|
|
2026
|
+
res.status === 409 && snippet.includes('"jobStatus":"cancelled"');
|
|
2027
|
+
if (cancelledNow) cancelAc.abort(); // REPL cancelled mid-run → stop the Python agent
|
|
2028
|
+
const benign409 =
|
|
2029
|
+
res.status === 409 &&
|
|
2030
|
+
(snippet.includes('"jobStatus":"completed"') || cancelledNow);
|
|
2031
|
+
if (!benign409) {
|
|
2032
|
+
console.error(
|
|
2033
|
+
`[gonext-worker] agent_chat job-chunk POST failed status=${res.status} jobId=${jobId}` +
|
|
2034
|
+
(snippet ? ` response=${snippet}` : "")
|
|
2035
|
+
);
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
};
|
|
2039
|
+
|
|
2040
|
+
const enqueueText = (s) => {
|
|
2041
|
+
if (!s) return;
|
|
2042
|
+
fullText += s;
|
|
2043
|
+
buf += s;
|
|
2044
|
+
if (buf.length >= CHUNK_MAX_BUF) {
|
|
2045
|
+
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
|
|
2046
|
+
flushTail = flushTail.then(() => flushChunks()).catch((err) => {
|
|
2047
|
+
console.error("[gonext-worker] agent_chat chunk flush error:", err);
|
|
2048
|
+
});
|
|
2049
|
+
return;
|
|
2050
|
+
}
|
|
2051
|
+
if (!flushTimer) {
|
|
2052
|
+
flushTimer = setTimeout(() => {
|
|
2053
|
+
flushTimer = null;
|
|
2054
|
+
flushTail = flushTail.then(() => flushChunks()).catch((err) => {
|
|
2055
|
+
console.error("[gonext-worker] agent_chat chunk flush error:", err);
|
|
2056
|
+
});
|
|
2057
|
+
}, CHUNK_DEBOUNCE_MS);
|
|
2058
|
+
}
|
|
2059
|
+
};
|
|
2060
|
+
|
|
2061
|
+
console.log(
|
|
2062
|
+
`[gonext-worker] agent_chat ${jobId} baseURL=${payload?.agentBaseURL ?? "(none)"} modelId=${payload?.agentModelId ?? "(none)"}`
|
|
2063
|
+
);
|
|
2064
|
+
try {
|
|
2065
|
+
const python =
|
|
2066
|
+
(process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "")
|
|
2067
|
+
.trim() || "python3";
|
|
2068
|
+
const scriptPath = join(WORKER_DIR, "gonext_agent_chat.py");
|
|
2069
|
+
|
|
2070
|
+
// Workspace scope depends on WHERE the job came from (task #75). The `gonext`
|
|
2071
|
+
// terminal always sends its launch cwd as activeWorkspace; the web app never does —
|
|
2072
|
+
// that's the discriminator.
|
|
2073
|
+
// • Terminal: the real `gonext-cli workspace add` registry + that cwd.
|
|
2074
|
+
// • Web agent chat: ONE isolated temp scratch folder — NOT the Mac's registered
|
|
2075
|
+
// project folders. Inheriting them leaked their paths into the web prompt and
|
|
2076
|
+
// let terminal state drive web routing. run_command stays off (not a code repo).
|
|
2077
|
+
const terminalActiveWs = String(payload?.activeWorkspace ?? "").trim();
|
|
2078
|
+
let jobWorkspaces;
|
|
2079
|
+
let jobActiveWorkspace;
|
|
2080
|
+
if (terminalActiveWs) {
|
|
2081
|
+
jobWorkspaces = await readFile(join(homedir(), ".gonext", "workspaces.json"), "utf8")
|
|
2082
|
+
.then((raw) => {
|
|
2083
|
+
const parsed = JSON.parse(raw);
|
|
2084
|
+
return Array.isArray(parsed?.workspaces) ? parsed.workspaces : [];
|
|
2085
|
+
})
|
|
2086
|
+
.catch(() => []);
|
|
2087
|
+
jobActiveWorkspace = terminalActiveWs;
|
|
2088
|
+
} else {
|
|
2089
|
+
const webWork = join(homedir(), ".gonext", "web-agent-work");
|
|
2090
|
+
await mkdir(webWork, { recursive: true }).catch(() => {});
|
|
2091
|
+
jobWorkspaces = [{ name: "web", path: webWork, allowRun: false }];
|
|
2092
|
+
jobActiveWorkspace = webWork;
|
|
2093
|
+
}
|
|
2094
|
+
|
|
2095
|
+
// Task #108: for an OpenAI-compatible coder, fetch the API key from a dedicated
|
|
2096
|
+
// worker-authed endpoint (NOT the job payload/Mongo doc — secret hygiene) and inject
|
|
2097
|
+
// it into python stdin only. Keyed off the RESOLVED backend (#117), so an Auto config
|
|
2098
|
+
// pointing at a cloud endpoint now gets its key: it used to require the explicit
|
|
2099
|
+
// "openai" kind, and python's `coding_api_key or agent_api_key` fallback then
|
|
2100
|
+
// authenticated a cloud provider with the LOCAL agent key → 401.
|
|
2101
|
+
let codingApiKey = "";
|
|
2102
|
+
if (codingBackend === "openai") {
|
|
2103
|
+
try {
|
|
2104
|
+
const kr = await workerFetch("/api/worker/coding-key");
|
|
2105
|
+
if (kr.ok) {
|
|
2106
|
+
const kj = await kr.json().catch(() => ({}));
|
|
2107
|
+
if (typeof kj?.apiKey === "string") codingApiKey = kj.apiKey;
|
|
2108
|
+
}
|
|
2109
|
+
} catch {
|
|
2110
|
+
/* best-effort — python falls back to a keyless call, which the endpoint may 401 */
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
|
|
2114
|
+
const input = JSON.stringify({
|
|
2115
|
+
messages: payload?.messages ?? [],
|
|
2116
|
+
agentBaseURL: payload?.agentBaseURL ?? "",
|
|
2117
|
+
agentApiKey: payload?.agentApiKey ?? "",
|
|
2118
|
+
agentModelId: payload?.agentModelId ?? "",
|
|
2119
|
+
codingBaseURL: payload?.codingBaseURL ?? "",
|
|
2120
|
+
codingModelId: payload?.codingModelId ?? "",
|
|
2121
|
+
codingKind,
|
|
2122
|
+
// The resolved backend (#117): python trusts this instead of probing again, so the
|
|
2123
|
+
// key fetch above and python's streaming / repair / step-budget branches can't
|
|
2124
|
+
// disagree about what this endpoint is. Python re-resolves only if it ends up on a
|
|
2125
|
+
// DIFFERENT url than the one judged here (its no-model-id fallback to the chat model).
|
|
2126
|
+
codingBackend,
|
|
2127
|
+
codingApiKey,
|
|
2128
|
+
// Task #107: when true, python emits a {type:code_response} event per code-model
|
|
2129
|
+
// call so the worker persists the full raw text to Mongo.
|
|
2130
|
+
saveFullResponse: payload?.saveFullResponse === true,
|
|
2131
|
+
// Dedicated SEARCH model (#105): when set, web_search synthesizes its results into a
|
|
2132
|
+
// cited answer with this model instead of returning raw read pages. Empty = no synth.
|
|
2133
|
+
searchBaseURL: payload?.searchBaseURL ?? "",
|
|
2134
|
+
searchModelId: payload?.searchModelId ?? "",
|
|
2135
|
+
// SearXNG metasearch URL ("Agent Search Server", web Settings). Empty = python falls
|
|
2136
|
+
// back to the GONEXT_SEARXNG_URL env (or keyless DDG only).
|
|
2137
|
+
searxngUrl: payload?.searxngUrl ?? "",
|
|
2138
|
+
tools: payload?.tools ?? ["http_request"],
|
|
2139
|
+
// 0 = "not explicitly set": python defaults to 5, but may auto-raise to 12 when
|
|
2140
|
+
// workspaces are registered. Passing a literal 5 here would look user-set and
|
|
2141
|
+
// block that bump.
|
|
2142
|
+
maxSteps: payload?.maxSteps ?? 0,
|
|
2143
|
+
// run_command policy (web Settings → Agent). Both optional: empty = allow every
|
|
2144
|
+
// command except the always-blocked sudo/su. runAllowlist (non-empty) = opt-in
|
|
2145
|
+
// lockdown to exactly those runners; runDenylist = extra blocks. Accept whatever
|
|
2146
|
+
// the API sends (list or comma/space string); python re-parses either shape.
|
|
2147
|
+
runAllowlist: payload?.runAllowlist ?? [],
|
|
2148
|
+
runDenylist: payload?.runDenylist ?? [],
|
|
2149
|
+
// Interactive command-approval gate: the jobId lets python register a pending
|
|
2150
|
+
// approval + poll for the user's Yes/No via the API; interactiveApproval is the
|
|
2151
|
+
// REPL's "I can show a picker" signal (else python keeps hard-blocking, as on web).
|
|
2152
|
+
jobId,
|
|
2153
|
+
interactiveApproval: payload?.interactiveApproval === true,
|
|
2154
|
+
// Session opt-in (task #80): auto-extend past the step budget without re-prompting
|
|
2155
|
+
// (the REPL sets this once the user picks "yes, don't ask again").
|
|
2156
|
+
alwaysExtendOnMaxStep: payload?.alwaysExtendOnMaxStep === true,
|
|
2157
|
+
// Auto-test mode (/test-auto): the agent verifies its work and fixes until it passes.
|
|
2158
|
+
autoTest: payload?.autoTest === true,
|
|
2159
|
+
// Deploy target chosen via the terminal /server picker (task #69). Host/user only.
|
|
2160
|
+
deployServer: payload?.deployServer ?? null,
|
|
2161
|
+
// Max web_search + fetch_url calls the agent may make per task (user-configurable
|
|
2162
|
+
// in web Settings → Agent). Default 10; past it the retrieval tools refuse.
|
|
2163
|
+
researchBudget: payload?.researchBudget ?? 10,
|
|
2164
|
+
// Tool-invocation mode for the agent: "code" (default, python CodeAgent) or
|
|
2165
|
+
// "toolcall" (structured JSON tool calls). Passed through only; the API/web can
|
|
2166
|
+
// set payload.agentToolMode to A/B — default keeps current behavior.
|
|
2167
|
+
agentToolMode: payload?.agentToolMode ?? "code",
|
|
2168
|
+
// For the create_pdf tool: the worker key lets the Python tool ask the API
|
|
2169
|
+
// to presign an S3 upload, so no AWS creds ever live on the worker machine.
|
|
2170
|
+
apiBaseURL: apiBase,
|
|
2171
|
+
workerKey,
|
|
2172
|
+
// send_email tool: the user configures their own email API (e.g. a Lambda) in
|
|
2173
|
+
// web Settings. The tool is only registered when emailEnabled + URL + body
|
|
2174
|
+
// template are set, and always previews before sending (confirm required).
|
|
2175
|
+
emailEnabled: payload?.emailEnabled ?? false,
|
|
2176
|
+
emailApiUrl: payload?.emailApiUrl ?? "",
|
|
2177
|
+
emailApiMethod: payload?.emailApiMethod ?? "POST",
|
|
2178
|
+
emailApiHeaders: payload?.emailApiHeaders ?? "",
|
|
2179
|
+
emailBodyTemplate: payload?.emailBodyTemplate ?? "",
|
|
2180
|
+
emailFrom: payload?.emailFrom ?? "",
|
|
2181
|
+
emailAllowList: payload?.emailAllowList ?? "",
|
|
2182
|
+
// RAG tools (download/unzip/index/search over a user's ZIP-at-URL). Only enabled
|
|
2183
|
+
// + registered when ragEnabled + AWS creds present. The worker talks to the user's
|
|
2184
|
+
// OWN S3 bucket directly with these creds (boto3), NOT via presigned URLs.
|
|
2185
|
+
ragEnabled: payload?.ragEnabled ?? false,
|
|
2186
|
+
ragS3Location: payload?.ragS3Location ?? "",
|
|
2187
|
+
ragAwsRegion: payload?.ragAwsRegion ?? "",
|
|
2188
|
+
ragAwsAccessKeyId: payload?.ragAwsAccessKeyId ?? "",
|
|
2189
|
+
ragAwsSecretAccessKey: payload?.ragAwsSecretAccessKey ?? "",
|
|
2190
|
+
ragEmbedModel: payload?.ragEmbedModel ?? "",
|
|
2191
|
+
ragEmbedUrl: payload?.ragEmbedUrl ?? "",
|
|
2192
|
+
ragTopK: payload?.ragTopK ?? 6,
|
|
2193
|
+
// RAG storage backend for this folder (task #97): "local" = on disk under ~/.gonext
|
|
2194
|
+
// (per-folder, no S3 creds), else "cloud" (the user's S3). Terminal sends the folder's
|
|
2195
|
+
// choice; web never sets it → "cloud". python re-validates and defaults to cloud.
|
|
2196
|
+
ragMode: payload?.ragMode === "local" ? "local" : "cloud",
|
|
2197
|
+
// Workspaces the agent may read/edit/test code in (see the terminal-vs-web split
|
|
2198
|
+
// computed above): the real registry for the terminal, a lone temp folder for web.
|
|
2199
|
+
workspaces: jobWorkspaces,
|
|
2200
|
+
// download_file/unzip_file output lands here (a registered workspace; python
|
|
2201
|
+
// re-validates). Terminal = its launch cwd; web = the temp scratch folder.
|
|
2202
|
+
activeWorkspace: jobActiveWorkspace,
|
|
2203
|
+
});
|
|
2204
|
+
// Agent run time budget (task #81). Default 60 min; user-configurable in web Settings
|
|
2205
|
+
// (agentTimeoutMinutes → payload.agentTimeoutMs). Multi-step ReAct on a 14B/31B with a
|
|
2206
|
+
// cold prompt cache — or a remote Ollama box slow-cold-loading a big model — can run
|
|
2207
|
+
// for many minutes per step, so a generous budget is normal. On expiry the TERMINAL
|
|
2208
|
+
// (interactive) asks the user whether to keep waiting (onAgentTimeout below); "yes"
|
|
2209
|
+
// ignores the budget for the rest of the run. WEB has no picker AND a hard WS deadline
|
|
2210
|
+
// (1_860_000 ms), so it's capped at 30 min and just aborts — our clean timeout error
|
|
2211
|
+
// then reaches the user instead of a generic WS timeout. On a hard stop the python
|
|
2212
|
+
// child is SIGKILLed (shows as a BrokenPipeError in the model server log — expected).
|
|
2213
|
+
// Liveness during long model calls comes from the python heartbeat (a keepalive step
|
|
2214
|
+
// chunk every 45s), not from this cap.
|
|
2215
|
+
const DEFAULT_AGENT_TIMEOUT_MS = 3_600_000; // 1 hour
|
|
2216
|
+
const WEB_AGENT_TIMEOUT_CAP_MS = 1_800_000; // stay under the web WS deadline
|
|
2217
|
+
const isInteractiveTimeout = payload?.interactiveApproval === true;
|
|
2218
|
+
const requestedTimeoutMs =
|
|
2219
|
+
Number.isFinite(payload?.agentTimeoutMs) && payload.agentTimeoutMs > 0
|
|
2220
|
+
? payload.agentTimeoutMs
|
|
2221
|
+
: DEFAULT_AGENT_TIMEOUT_MS;
|
|
2222
|
+
const timeoutMs = isInteractiveTimeout
|
|
2223
|
+
? requestedTimeoutMs
|
|
2224
|
+
: Math.min(requestedTimeoutMs, WEB_AGENT_TIMEOUT_CAP_MS);
|
|
2225
|
+
|
|
2226
|
+
let inThink = false;
|
|
2227
|
+
let finalText = "";
|
|
2228
|
+
let answerStreamed = false; // an answer_stream delta already delivered the visible text
|
|
2229
|
+
// Task #83: running total of prompt tokens sent to the agent CODE model. Updated on
|
|
2230
|
+
// each python "tokens" event, persisted to the job doc (Mongo) so the REPL can show
|
|
2231
|
+
// it live and it survives on the completed job.
|
|
2232
|
+
let latestCodeTokens = 0;
|
|
2233
|
+
// Task #84: peak single-request prompt-token count seen this turn (not the sum).
|
|
2234
|
+
let latestMaxCodeTokens = 0;
|
|
2235
|
+
// Task #104: running total of OUTPUT (completion) tokens the CODE model generated this
|
|
2236
|
+
// turn. Persisted to the job doc; the REPL folds this turn's total into the lifetime
|
|
2237
|
+
// per-workspace + per-(user, coding-URL) counters at completion.
|
|
2238
|
+
let latestCodeOutputTokens = 0;
|
|
2239
|
+
// Raw model-stream tokens (event.type === "stream") are Python code + `Thought:`
|
|
2240
|
+
// prose + <code>/<|"|> fragments. The web Thought panel renders reasoning as
|
|
2241
|
+
// markdown, which turns `#`-prefixed code comments into <h1> headings and
|
|
2242
|
+
// swallows the tag fragments. Wrap only the raw stream in a fenced code block so
|
|
2243
|
+
// it renders verbatim; `step` labels stay outside the fence as normal prose.
|
|
2244
|
+
// Tilde fences (~~~) avoid colliding with any ``` the model itself emits.
|
|
2245
|
+
let inStreamFence = false;
|
|
2246
|
+
const closeStreamFence = () => {
|
|
2247
|
+
if (inStreamFence) {
|
|
2248
|
+
inStreamFence = false;
|
|
2249
|
+
enqueueText("\n~~~\n\n");
|
|
2250
|
+
}
|
|
2251
|
+
};
|
|
2252
|
+
|
|
2253
|
+
// The create_pdf tool uses WeasyPrint, whose Homebrew libs (pango/cairo/gobject)
|
|
2254
|
+
// live in /opt/homebrew/lib (Apple Silicon) or /usr/local/lib (Intel). macOS doesn't
|
|
2255
|
+
// search those by default, so ctypes can't dlopen them. Inject the path for the child
|
|
2256
|
+
// so users don't have to set DYLD_FALLBACK_LIBRARY_PATH themselves.
|
|
2257
|
+
let pdfEnv;
|
|
2258
|
+
if (process.platform === "darwin") {
|
|
2259
|
+
const existing = process.env.DYLD_FALLBACK_LIBRARY_PATH ?? "";
|
|
2260
|
+
const brewLibs = ["/opt/homebrew/lib", "/usr/local/lib"];
|
|
2261
|
+
const merged = [existing, ...brewLibs].filter(Boolean).join(":");
|
|
2262
|
+
pdfEnv = { DYLD_FALLBACK_LIBRARY_PATH: merged };
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
// Poll our own job status during the run so a REPL Ctrl+C (which marks the job
|
|
2266
|
+
// "cancelled" via the API) actually stops the Python agent within ~1.5s, instead of
|
|
2267
|
+
// only being noticed on the next chunk flush (which can be 45s apart during a long
|
|
2268
|
+
// model call). The job-chunk 409 path (flushChunks) is the backup signal.
|
|
2269
|
+
const cancelPoll = setInterval(async () => {
|
|
2270
|
+
if (cancelAc.signal.aborted) return;
|
|
2271
|
+
try {
|
|
2272
|
+
const r = await workerFetch(`/api/worker/jobs/${jobId}`);
|
|
2273
|
+
if (r.ok) {
|
|
2274
|
+
const j = await r.json().catch(() => ({}));
|
|
2275
|
+
if (j?.jobStatus === "cancelled") {
|
|
2276
|
+
cancelled = true;
|
|
2277
|
+
cancelAc.abort();
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
} catch {
|
|
2281
|
+
/* transient poll error — try again next tick */
|
|
2282
|
+
}
|
|
2283
|
+
}, 1500);
|
|
2284
|
+
cancelAc.signal.addEventListener("abort", () => { cancelled = true; }, { once: true });
|
|
2285
|
+
|
|
2286
|
+
// Task #81: when the time budget is reached in the TERMINAL, ask the user whether to
|
|
2287
|
+
// keep waiting instead of just aborting. Reuses the #67 approval plumbing
|
|
2288
|
+
// (approval-request → poll approvalDecision) with a __TIMEOUT__:: sentinel the REPL
|
|
2289
|
+
// recognizes and renders as a Yes/No picker (auto-No at 60s). Returns true = keep
|
|
2290
|
+
// waiting (disable the cap), false = stop. Only wired for interactive jobs (web never
|
|
2291
|
+
// reaches here — it isn't passed as onTimeout).
|
|
2292
|
+
const onAgentTimeout = async () => {
|
|
2293
|
+
if (!isInteractiveTimeout || !jobId) return false;
|
|
2294
|
+
const approvalId = `timeout-${Date.now()}`;
|
|
2295
|
+
const mins = Math.round(timeoutMs / 60000);
|
|
2296
|
+
const command = `__TIMEOUT__::The agent has run for ${mins} min (the time budget). Keep waiting?`;
|
|
2297
|
+
try {
|
|
2298
|
+
const r = await workerFetch(`/api/worker/jobs/${jobId}/approval-request`, {
|
|
2299
|
+
method: "POST",
|
|
2300
|
+
body: JSON.stringify({ id: approvalId, command }),
|
|
2301
|
+
});
|
|
2302
|
+
if (!r.ok) return false;
|
|
2303
|
+
} catch {
|
|
2304
|
+
return false;
|
|
2305
|
+
}
|
|
2306
|
+
// Poll for the user's Yes/No up to ~65s (the REPL auto-answers No at 60s).
|
|
2307
|
+
const deadline = Date.now() + 65_000;
|
|
2308
|
+
while (Date.now() < deadline) {
|
|
2309
|
+
if (cancelAc.signal.aborted) return false;
|
|
2310
|
+
await new Promise((r) => setTimeout(r, 1500));
|
|
2311
|
+
try {
|
|
2312
|
+
const r = await workerFetch(`/api/worker/jobs/${jobId}`);
|
|
2313
|
+
if (r.ok) {
|
|
2314
|
+
const j = await r.json().catch(() => ({}));
|
|
2315
|
+
const dec = j?.approvalDecision;
|
|
2316
|
+
if (dec && dec.id === approvalId) return dec.allow === true;
|
|
2317
|
+
}
|
|
2318
|
+
} catch {
|
|
2319
|
+
/* transient poll error — keep trying */
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
return false; // no answer in time → treat as No (stop)
|
|
2323
|
+
};
|
|
2324
|
+
|
|
2325
|
+
try {
|
|
2326
|
+
await runProcessWithStreamingStdout(python, [scriptPath], input, timeoutMs, (event) => {
|
|
2327
|
+
if (event.type === "log" && typeof event.text === "string") {
|
|
2328
|
+
console.log(`[gonext-agent] ${event.text}`);
|
|
2329
|
+
} else if (event.type === "stream" && typeof event.text === "string") {
|
|
2330
|
+
// Live model token stream (the agent's Thought / reasoning as it generates).
|
|
2331
|
+
// Goes inside the same <think> block as the step summaries, but RAW — no
|
|
2332
|
+
// "\n\n" separator, so the tokens concatenate into flowing text. Wrapped in
|
|
2333
|
+
// a code fence (opened lazily on the first stream token) so the web viewer
|
|
2334
|
+
// renders it verbatim instead of as markdown.
|
|
2335
|
+
if (!inThink) {
|
|
2336
|
+
inThink = true;
|
|
2337
|
+
enqueueText("<think>");
|
|
2338
|
+
}
|
|
2339
|
+
if (!inStreamFence) {
|
|
2340
|
+
inStreamFence = true;
|
|
2341
|
+
enqueueText("~~~\n");
|
|
2342
|
+
}
|
|
2343
|
+
enqueueText(event.text);
|
|
2344
|
+
} else if (event.type === "step" && typeof event.text === "string") {
|
|
2345
|
+
closeStreamFence();
|
|
2346
|
+
if (!inThink) {
|
|
2347
|
+
inThink = true;
|
|
2348
|
+
enqueueText("<think>");
|
|
2349
|
+
}
|
|
2350
|
+
enqueueText(event.text + "\n\n");
|
|
2351
|
+
} else if (event.type === "answer_stream" && typeof event.text === "string") {
|
|
2352
|
+
// Live delta of the FINAL user-facing answer (the plain-chat fast path streams
|
|
2353
|
+
// its reply as it generates, instead of blocking for the whole thing) — this is
|
|
2354
|
+
// the actual answer, NOT reasoning, so it must stay OUTSIDE <think> (wrapping it
|
|
2355
|
+
// would hide it in the web's collapsed Thinking panel instead of the live main
|
|
2356
|
+
// answer bubble). Close any open think/stream fence first in case a prior step
|
|
2357
|
+
// left one dangling.
|
|
2358
|
+
closeStreamFence();
|
|
2359
|
+
if (inThink) {
|
|
2360
|
+
inThink = false;
|
|
2361
|
+
enqueueText("</think>");
|
|
2362
|
+
}
|
|
2363
|
+
enqueueText(event.text);
|
|
2364
|
+
answerStreamed = true;
|
|
2365
|
+
} else if (event.type === "tokens" && Number.isFinite(event.codeInput)) {
|
|
2366
|
+
// Prompt tokens sent to the agent CODE model (tasks #83 accumulated + #84 peak).
|
|
2367
|
+
// Persist both to the job doc so they're in Mongo and the REPL can render them
|
|
2368
|
+
// live. Steps are seconds apart, so a fire-and-forget PATCH per event is cheap;
|
|
2369
|
+
// it only touches agentCodeTokens/agentMaxCodeTokens (never resultText/jobStatus),
|
|
2370
|
+
// so it can't race the chunk/completed writes.
|
|
2371
|
+
const nextTotal = Math.max(latestCodeTokens, event.codeInput);
|
|
2372
|
+
const nextMax = Number.isFinite(event.codeMax)
|
|
2373
|
+
? Math.max(latestMaxCodeTokens, event.codeMax)
|
|
2374
|
+
: latestMaxCodeTokens;
|
|
2375
|
+
// Task #104: output tokens climb monotonically across the turn too.
|
|
2376
|
+
const nextOut = Number.isFinite(event.codeOutput)
|
|
2377
|
+
? Math.max(latestCodeOutputTokens, event.codeOutput)
|
|
2378
|
+
: latestCodeOutputTokens;
|
|
2379
|
+
if (
|
|
2380
|
+
nextTotal !== latestCodeTokens ||
|
|
2381
|
+
nextMax !== latestMaxCodeTokens ||
|
|
2382
|
+
nextOut !== latestCodeOutputTokens
|
|
2383
|
+
) {
|
|
2384
|
+
latestCodeTokens = nextTotal;
|
|
2385
|
+
latestMaxCodeTokens = nextMax;
|
|
2386
|
+
latestCodeOutputTokens = nextOut;
|
|
2387
|
+
workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2388
|
+
method: "PATCH",
|
|
2389
|
+
body: JSON.stringify({
|
|
2390
|
+
agentCodeTokens: latestCodeTokens,
|
|
2391
|
+
agentMaxCodeTokens: latestMaxCodeTokens,
|
|
2392
|
+
agentOutputTokens: latestCodeOutputTokens,
|
|
2393
|
+
}),
|
|
2394
|
+
}).catch(() => {});
|
|
2395
|
+
}
|
|
2396
|
+
} else if (event.type === "code_response" && typeof event.text === "string") {
|
|
2397
|
+
// Task #107: the user's Save-Full-Response toggle is on — persist this code-model
|
|
2398
|
+
// response's full raw text to Mongo. Fire-and-forget; never blocks the turn.
|
|
2399
|
+
workerFetch(`/api/worker/agent-response`, {
|
|
2400
|
+
method: "POST",
|
|
2401
|
+
body: JSON.stringify({
|
|
2402
|
+
jobId,
|
|
2403
|
+
step: Number.isFinite(event.step) ? event.step : 0,
|
|
2404
|
+
text: event.text,
|
|
2405
|
+
inputTokens: Number.isFinite(event.inputTokens) ? event.inputTokens : 0,
|
|
2406
|
+
outputTokens: Number.isFinite(event.outputTokens) ? event.outputTokens : 0,
|
|
2407
|
+
codingUrl: payload?.codingBaseURL ?? "",
|
|
2408
|
+
}),
|
|
2409
|
+
}).catch(() => {});
|
|
2410
|
+
} else if (event.type === "final" && typeof event.text === "string") {
|
|
2411
|
+
closeStreamFence();
|
|
2412
|
+
if (inThink) {
|
|
2413
|
+
inThink = false;
|
|
2414
|
+
enqueueText("</think>");
|
|
2415
|
+
}
|
|
2416
|
+
finalText = event.text;
|
|
2417
|
+
// If the answer was already delivered live via answer_stream deltas, its text is
|
|
2418
|
+
// ALREADY in resultText — appending it again here would duplicate the whole reply
|
|
2419
|
+
// (streamed once, then dumped again in full). Only append when nothing streamed it.
|
|
2420
|
+
if (!answerStreamed) {
|
|
2421
|
+
enqueueText(event.text);
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
}, pdfEnv, cancelAc.signal, isInteractiveTimeout ? onAgentTimeout : undefined);
|
|
2425
|
+
} finally {
|
|
2426
|
+
clearInterval(cancelPoll);
|
|
2427
|
+
}
|
|
2428
|
+
|
|
2429
|
+
// User cancelled mid-run: the Python child was SIGKILLed and the job is already
|
|
2430
|
+
// "cancelled" in the store. Leave it there — do NOT PATCH completed/failed (which
|
|
2431
|
+
// would resurrect a finished status) and skip the final chunk flush (it'd 409).
|
|
2432
|
+
if (cancelled || cancelAc.signal.aborted) {
|
|
2433
|
+
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
|
|
2434
|
+
console.log(`[gonext-worker] agent_chat ${jobId} cancelled by user`);
|
|
2435
|
+
return;
|
|
2436
|
+
}
|
|
2437
|
+
|
|
2438
|
+
closeStreamFence();
|
|
2439
|
+
if (inThink) {
|
|
2440
|
+
enqueueText("</think>");
|
|
2441
|
+
}
|
|
2442
|
+
|
|
2443
|
+
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
|
|
2444
|
+
await flushTail;
|
|
2445
|
+
await flushChunks();
|
|
2446
|
+
|
|
2447
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
2448
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2449
|
+
method: "PATCH",
|
|
2450
|
+
body: JSON.stringify({
|
|
2451
|
+
jobStatus: "completed",
|
|
2452
|
+
// When the answer streamed live (answer_stream), completed resultText must stay
|
|
2453
|
+
// an EXACT EXTENSION of the chunk accumulation: the REPL renders the final poll
|
|
2454
|
+
// by flushing resultText.slice(<chars already shown>), so replacing the stream
|
|
2455
|
+
// with the shorter bare answer here made short plain replies render cut
|
|
2456
|
+
// mid-word ("Hello! How"). Only when nothing streamed is the deduped finalText
|
|
2457
|
+
// the whole story.
|
|
2458
|
+
resultText: answerStreamed ? fullText : finalText || fullText,
|
|
2459
|
+
tokenCount: Math.max(1, Math.ceil((finalText || fullText).length / 4)),
|
|
2460
|
+
...(latestCodeTokens > 0 ? { agentCodeTokens: latestCodeTokens } : {}),
|
|
2461
|
+
...(latestMaxCodeTokens > 0 ? { agentMaxCodeTokens: latestMaxCodeTokens } : {}),
|
|
2462
|
+
...(latestCodeOutputTokens > 0 ? { agentOutputTokens: latestCodeOutputTokens } : {}),
|
|
2463
|
+
totalTimeSeconds,
|
|
2464
|
+
}),
|
|
2465
|
+
});
|
|
2466
|
+
await ensureWorkerOk(doneRes, `complete agent_chat PATCH jobId=${jobId}`);
|
|
2467
|
+
console.log(`[gonext-worker] completed agent_chat ${jobId} (${totalTimeSeconds.toFixed(1)}s)`);
|
|
2468
|
+
} catch (e) {
|
|
2469
|
+
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
|
|
2470
|
+
// A cancel that raced into the catch (e.g. the kill surfaced as an error before the
|
|
2471
|
+
// aborted-close path): the job is already "cancelled" — don't overwrite with failed.
|
|
2472
|
+
if (cancelled || cancelAc.signal.aborted) {
|
|
2473
|
+
console.log(`[gonext-worker] agent_chat ${jobId} cancelled by user`);
|
|
2474
|
+
return;
|
|
2475
|
+
}
|
|
2476
|
+
await flushTail;
|
|
2477
|
+
await flushChunks().catch(() => {});
|
|
2478
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
2479
|
+
const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2480
|
+
method: "PATCH",
|
|
2481
|
+
body: JSON.stringify({
|
|
2482
|
+
jobStatus: "failed",
|
|
2483
|
+
errorMessage: message,
|
|
2484
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
2485
|
+
}),
|
|
2486
|
+
});
|
|
2487
|
+
if (!failRes.ok) {
|
|
2488
|
+
const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
|
|
2489
|
+
console.error(
|
|
2490
|
+
`[gonext-worker] agent_chat fail PATCH also failed ${failRes.status} jobId=${jobId}` +
|
|
2491
|
+
(snippet ? ` response=${snippet}` : "")
|
|
2492
|
+
);
|
|
2493
|
+
}
|
|
2494
|
+
console.error(`[gonext-worker] failed agent_chat ${jobId}:`, message);
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
|
|
2498
|
+
function normalizeBaseUrl(raw) {
|
|
2499
|
+
return typeof raw === "string" ? raw.trim().replace(/\/+$/, "") : "";
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
/**
|
|
2503
|
+
* "Load Models" (task #44): probe the agent coding-model server's model list. Tries the
|
|
2504
|
+
* OpenAI-compatible /v1/models (MLX + Ollama both serve it) and falls back to Ollama's
|
|
2505
|
+
* /api/tags. The server is on the user's network, so only this worker can reach it. The
|
|
2506
|
+
* job's resultText is JSON {models:[...]} the web parses into checkboxes.
|
|
2507
|
+
*/
|
|
2508
|
+
async function runListModelsJob(job) {
|
|
2509
|
+
const { jobId, payload } = job;
|
|
2510
|
+
const start = Date.now();
|
|
2511
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2512
|
+
method: "PATCH",
|
|
2513
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
2514
|
+
});
|
|
2515
|
+
await ensureWorkerOk(runRes, `mark running list_models jobId=${jobId}`);
|
|
2516
|
+
try {
|
|
2517
|
+
const base = normalizeBaseUrl(payload?.url).replace(/\/v1$/i, "");
|
|
2518
|
+
if (!base) throw new Error("No coding-model URL provided.");
|
|
2519
|
+
const models = new Set();
|
|
2520
|
+
// 1) OpenAI-compatible /v1/models → { data: [{ id }] }
|
|
2521
|
+
try {
|
|
2522
|
+
const r = await fetch(`${base}/v1/models`, { method: "GET" });
|
|
2523
|
+
if (r.ok) {
|
|
2524
|
+
const j = await r.json().catch(() => ({}));
|
|
2525
|
+
for (const m of Array.isArray(j?.data) ? j.data : []) {
|
|
2526
|
+
if (typeof m?.id === "string" && m.id.trim()) models.add(m.id.trim());
|
|
2527
|
+
}
|
|
2528
|
+
}
|
|
2529
|
+
} catch {
|
|
2530
|
+
/* try the Ollama shape next */
|
|
2531
|
+
}
|
|
2532
|
+
// 2) Ollama /api/tags → { models: [{ name }] }
|
|
2533
|
+
if (models.size === 0) {
|
|
2534
|
+
const r = await fetch(`${base}/api/tags`, { method: "GET" });
|
|
2535
|
+
if (r.ok) {
|
|
2536
|
+
const j = await r.json().catch(() => ({}));
|
|
2537
|
+
for (const m of Array.isArray(j?.models) ? j.models : []) {
|
|
2538
|
+
if (typeof m?.name === "string" && m.name.trim()) models.add(m.name.trim());
|
|
2539
|
+
}
|
|
2540
|
+
}
|
|
2541
|
+
}
|
|
2542
|
+
const list = Array.from(models).sort();
|
|
2543
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2544
|
+
method: "PATCH",
|
|
2545
|
+
body: JSON.stringify({
|
|
2546
|
+
jobStatus: "completed",
|
|
2547
|
+
resultText: JSON.stringify({ models: list }),
|
|
2548
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
2549
|
+
}),
|
|
2550
|
+
});
|
|
2551
|
+
await ensureWorkerOk(doneRes, `complete list_models jobId=${jobId}`);
|
|
2552
|
+
console.log(`[gonext-worker] list_models ${jobId}: ${list.length} model(s) from ${base}`);
|
|
2553
|
+
} catch (e) {
|
|
2554
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
2555
|
+
await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2556
|
+
method: "PATCH",
|
|
2557
|
+
body: JSON.stringify({
|
|
2558
|
+
jobStatus: "failed",
|
|
2559
|
+
errorMessage: `Could not list models: ${message}`,
|
|
2560
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
2561
|
+
}),
|
|
2562
|
+
}).catch(() => {});
|
|
2563
|
+
console.error(`[gonext-worker] failed list_models ${jobId}:`, message);
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
|
|
2567
|
+
function normalizeOpenAiV1Root(raw) {
|
|
2568
|
+
const base = normalizeBaseUrl(raw);
|
|
2569
|
+
if (!base) return "";
|
|
2570
|
+
return /\/v1$/i.test(base) ? base : `${base}/v1`;
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2573
|
+
function sourceLabelFromBase(base) {
|
|
2574
|
+
try {
|
|
2575
|
+
return new URL(base).host || base;
|
|
2576
|
+
} catch {
|
|
2577
|
+
return base;
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
|
|
2581
|
+
function expandHomePath(rawPath) {
|
|
2582
|
+
const v = String(rawPath ?? "").trim();
|
|
2583
|
+
if (!v) return "";
|
|
2584
|
+
return v.replace(/^~(?=\/)/, homedir());
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2587
|
+
async function checkModelPathExists(rawPath) {
|
|
2588
|
+
const inputPath = String(rawPath ?? "").trim();
|
|
2589
|
+
const resolvedPath = expandHomePath(inputPath);
|
|
2590
|
+
if (!resolvedPath) {
|
|
2591
|
+
return { path: inputPath, resolvedPath: "", exists: false };
|
|
2592
|
+
}
|
|
2593
|
+
try {
|
|
2594
|
+
const st = await stat(resolvedPath);
|
|
2595
|
+
return { path: inputPath, resolvedPath, exists: st.isDirectory() || st.isFile() };
|
|
2596
|
+
} catch {
|
|
2597
|
+
return { path: inputPath, resolvedPath, exists: false };
|
|
2598
|
+
}
|
|
2599
|
+
}
|
|
2600
|
+
|
|
2601
|
+
/** Log assistant text to stdout; cap size so huge replies do not flood the terminal. */
|
|
2602
|
+
function logModelResponseToWorker(jobId, modelId, text) {
|
|
2603
|
+
const max = 12000;
|
|
2604
|
+
const n = text.length;
|
|
2605
|
+
const body = n <= max ? text : `${text.slice(0, max)}\n… [log truncated: ${n - max} more chars]`;
|
|
2606
|
+
console.log(
|
|
2607
|
+
`[gonext-worker] model reply job=${jobId} model=${modelId} chars=${n}:\n${body}`
|
|
2608
|
+
);
|
|
2609
|
+
}
|
|
2610
|
+
|
|
2611
|
+
async function checkOllamaTags(rawBase) {
|
|
2612
|
+
// Tolerate a base that already includes the endpoint (users sometimes paste the
|
|
2613
|
+
// full `.../api/tags` URL) so we don't fetch `.../api/tags/api/tags` → 404.
|
|
2614
|
+
const base = normalizeBaseUrl(rawBase).replace(/\/(?:api\/tags|api|v1)$/i, "");
|
|
2615
|
+
const endpoint = `${base}/api/tags`;
|
|
2616
|
+
try {
|
|
2617
|
+
// Bounded so an unreachable/slow host fails fast (a hung fetch would exceed
|
|
2618
|
+
// the client's local-health wait and leave the status dot stuck "pending").
|
|
2619
|
+
const res = await fetch(endpoint, {
|
|
2620
|
+
method: "GET",
|
|
2621
|
+
signal: AbortSignal.timeout(4000),
|
|
2622
|
+
});
|
|
2623
|
+
if (!res.ok) return { online: false, endpoint, models: [] };
|
|
2624
|
+
const j = await res.json();
|
|
2625
|
+
const source = sourceLabelFromBase(base);
|
|
2626
|
+
const models = (j.models ?? []).map((m) => {
|
|
2627
|
+
const name = m.name ?? m.model ?? "model";
|
|
2628
|
+
return {
|
|
2629
|
+
id: `${name}@@${source}`,
|
|
2630
|
+
name: `${name} (${source})`,
|
|
2631
|
+
value: `ollama:${name}@@${encodeURIComponent(base)}`,
|
|
2632
|
+
};
|
|
2633
|
+
});
|
|
2634
|
+
return { online: true, endpoint, models };
|
|
2635
|
+
} catch {
|
|
2636
|
+
return { online: false, endpoint, models: [] };
|
|
2637
|
+
}
|
|
2638
|
+
}
|
|
2639
|
+
|
|
2640
|
+
/**
|
|
2641
|
+
* Discover the mlx_lm.server processes actually running on this machine by
|
|
2642
|
+
* parsing `ps` output. Returns deduped `[{ port, modelPath }]` so the model
|
|
2643
|
+
* dropdown reflects live servers instead of a hand-maintained port list.
|
|
2644
|
+
* mlx_lm.server defaults to port 8080 when `--port` is omitted. macOS only.
|
|
2645
|
+
*/
|
|
2646
|
+
async function discoverMlxServers() {
|
|
2647
|
+
if (platform() !== "darwin") return [];
|
|
2648
|
+
let stdout = "";
|
|
2649
|
+
try {
|
|
2650
|
+
({ stdout } = await execFile("ps", ["-Ao", "args="], {
|
|
2651
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
2652
|
+
}));
|
|
2653
|
+
} catch {
|
|
2654
|
+
return [];
|
|
2655
|
+
}
|
|
2656
|
+
const byPort = new Map();
|
|
2657
|
+
for (const raw of stdout.split("\n")) {
|
|
2658
|
+
const line = raw.trim();
|
|
2659
|
+
if (!line) continue;
|
|
2660
|
+
// Match `mlx_lm.server` or `mlx_lm server` (python -m mlx_lm server).
|
|
2661
|
+
if (!/mlx_lm[.\s]server\b/.test(line)) continue;
|
|
2662
|
+
// Ignore our own grep/ps invocations that happen to mention the string.
|
|
2663
|
+
if (/\bgrep\b/.test(line)) continue;
|
|
2664
|
+
const portMatch = line.match(/--port(?:=|\s+)(\d{2,5})/);
|
|
2665
|
+
const port = portMatch ? parseInt(portMatch[1], 10) : 8080;
|
|
2666
|
+
if (!(port > 0)) continue;
|
|
2667
|
+
const modelMatch = line.match(/--model(?:=|\s+)(\S+)/);
|
|
2668
|
+
const modelPath = modelMatch ? modelMatch[1] : "";
|
|
2669
|
+
if (!byPort.has(port)) byPort.set(port, { port, modelPath });
|
|
2670
|
+
}
|
|
2671
|
+
return [...byPort.values()];
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2674
|
+
async function checkOpenAiModels(base, apiKey, source) {
|
|
2675
|
+
const endpoint = `${base}/models`;
|
|
2676
|
+
const headers = {};
|
|
2677
|
+
if (apiKey?.trim()) {
|
|
2678
|
+
headers.Authorization = `Bearer ${apiKey.trim()}`;
|
|
2679
|
+
}
|
|
2680
|
+
const hostSourceId =
|
|
2681
|
+
typeof source?.workerHostId === "string" ? source.workerHostId.trim() : "";
|
|
2682
|
+
const hostSourceLabel =
|
|
2683
|
+
typeof source?.hostName === "string" ? source.hostName.trim() : "";
|
|
2684
|
+
const sourcePort = typeof source?.port === "number" ? source.port : null;
|
|
2685
|
+
try {
|
|
2686
|
+
const res = await fetch(endpoint, { method: "GET", headers });
|
|
2687
|
+
if (!res.ok) return { online: false, endpoint, models: [] };
|
|
2688
|
+
const j = await res.json();
|
|
2689
|
+
const hostSuffix = hostSourceId ? `@@host:${hostSourceId}` : "";
|
|
2690
|
+
const portSuffix = sourcePort ? `@@port:${sourcePort}` : "";
|
|
2691
|
+
const modelSuffix = `${hostSuffix}${portSuffix}`;
|
|
2692
|
+
const portLabel = sourcePort ? `:${sourcePort}` : "";
|
|
2693
|
+
const displaySuffix = hostSourceLabel ? ` (${hostSourceLabel}${portLabel})` : portLabel ? ` (${portLabel})` : "";
|
|
2694
|
+
let rawIds = (j.data ?? []).map((d) => d.id).filter(Boolean);
|
|
2695
|
+
// mlx_lm.server registers both the local path and the HF model ID.
|
|
2696
|
+
// Prefer local absolute paths so each server shows exactly one model entry.
|
|
2697
|
+
const localPaths = rawIds.filter((id) => id.startsWith("/") || id.startsWith("~"));
|
|
2698
|
+
if (localPaths.length > 0) rawIds = localPaths;
|
|
2699
|
+
const models = rawIds.map((id) => ({
|
|
2700
|
+
id: hostSourceId ? `${id}@@${hostSourceId}` : id,
|
|
2701
|
+
name: `${id}${displaySuffix}`,
|
|
2702
|
+
value: `mlx:${id}${modelSuffix}`,
|
|
2703
|
+
}));
|
|
2704
|
+
return { online: true, endpoint, models };
|
|
2705
|
+
} catch {
|
|
2706
|
+
return { online: false, endpoint, models: [] };
|
|
2707
|
+
}
|
|
2708
|
+
}
|
|
2709
|
+
|
|
2710
|
+
/** True MLX LM check: import mlx_lm in Python (macOS). Not the OpenAI HTTP surface. */
|
|
2711
|
+
async function checkMlxLmNativeImport() {
|
|
2712
|
+
const preferred = (process.env.GONEXT_MLX_LM_PYTHON ?? "").trim() || "python3";
|
|
2713
|
+
const code = [
|
|
2714
|
+
"import sys",
|
|
2715
|
+
"try:",
|
|
2716
|
+
" import mlx_lm",
|
|
2717
|
+
" v = getattr(mlx_lm, '__version__', None)",
|
|
2718
|
+
" print(v or 'ok')",
|
|
2719
|
+
"except Exception:",
|
|
2720
|
+
" sys.exit(1)",
|
|
2721
|
+
].join("\n");
|
|
2722
|
+
|
|
2723
|
+
const candidates = [preferred];
|
|
2724
|
+
if (preferred === "python3") candidates.push("python");
|
|
2725
|
+
|
|
2726
|
+
const tried = [];
|
|
2727
|
+
for (const exe of [...new Set(candidates)]) {
|
|
2728
|
+
tried.push(exe);
|
|
2729
|
+
try {
|
|
2730
|
+
const { stdout } = await execFile(exe, ["-c", code], {
|
|
2731
|
+
timeout: 15000,
|
|
2732
|
+
maxBuffer: 65536,
|
|
2733
|
+
windowsHide: true,
|
|
2734
|
+
});
|
|
2735
|
+
const version = String(stdout ?? "").trim();
|
|
2736
|
+
return {
|
|
2737
|
+
available: true,
|
|
2738
|
+
python: exe,
|
|
2739
|
+
version: version || undefined,
|
|
2740
|
+
method: "python_import_mlx_lm",
|
|
2741
|
+
};
|
|
2742
|
+
} catch {
|
|
2743
|
+
/* try next */
|
|
2744
|
+
}
|
|
2745
|
+
}
|
|
2746
|
+
return {
|
|
2747
|
+
available: false,
|
|
2748
|
+
python: preferred,
|
|
2749
|
+
method: "python_import_mlx_lm",
|
|
2750
|
+
error: `Could not import mlx_lm (tried: ${tried.join(", ")})`,
|
|
2751
|
+
};
|
|
2752
|
+
}
|
|
2753
|
+
|
|
2754
|
+
// Probe whether voice transcription is ready (Audio support, task #21):
|
|
2755
|
+
// mlx-whisper importable AND the requested model present in the HF hub cache
|
|
2756
|
+
// (~/.cache/huggingface/hub, honoring HF_HOME/HF_HUB_CACHE). Returns
|
|
2757
|
+
// { available, model, reason } where reason is "ok" | "lib-missing" | "model-missing".
|
|
2758
|
+
async function checkWhisperAvailability(model) {
|
|
2759
|
+
const preferred =
|
|
2760
|
+
(process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "").trim() ||
|
|
2761
|
+
"python3";
|
|
2762
|
+
const repoName = `models--mlx-community--${model}`;
|
|
2763
|
+
const code = [
|
|
2764
|
+
"import os, sys, json, glob",
|
|
2765
|
+
"model = sys.argv[1] if len(sys.argv) > 1 else 'whisper-large-v3-turbo'",
|
|
2766
|
+
"try:",
|
|
2767
|
+
" import mlx_whisper # noqa: F401",
|
|
2768
|
+
"except Exception:",
|
|
2769
|
+
" print(json.dumps({'available': False, 'reason': 'lib-missing'})); sys.exit(0)",
|
|
2770
|
+
"home = os.environ.get('HF_HOME') or os.path.join(os.path.expanduser('~'), '.cache', 'huggingface')",
|
|
2771
|
+
"hub = os.environ.get('HF_HUB_CACHE') or os.path.join(home, 'hub')",
|
|
2772
|
+
`repo = os.path.join(hub, ${JSON.stringify(repoName)})`,
|
|
2773
|
+
"snaps = glob.glob(os.path.join(repo, 'snapshots', '*'))",
|
|
2774
|
+
"ok = False",
|
|
2775
|
+
"for s in snaps:",
|
|
2776
|
+
" if glob.glob(os.path.join(s, '*.safetensors')) or glob.glob(os.path.join(s, '*.npz')) or os.path.exists(os.path.join(s, 'weights.npz')):",
|
|
2777
|
+
" ok = True; break",
|
|
2778
|
+
"print(json.dumps({'available': ok, 'reason': 'ok' if ok else 'model-missing', 'cache': hub}))",
|
|
2779
|
+
].join("\n");
|
|
2780
|
+
const candidates = [preferred];
|
|
2781
|
+
if (preferred === "python3") candidates.push("python");
|
|
2782
|
+
for (const exe of [...new Set(candidates)]) {
|
|
2783
|
+
try {
|
|
2784
|
+
const { stdout } = await execFile(exe, ["-c", code, model], {
|
|
2785
|
+
timeout: 15000,
|
|
2786
|
+
maxBuffer: 65536,
|
|
2787
|
+
windowsHide: true,
|
|
2788
|
+
});
|
|
2789
|
+
const parsed = JSON.parse(String(stdout ?? "").trim() || "{}");
|
|
2790
|
+
return {
|
|
2791
|
+
available: parsed.available === true,
|
|
2792
|
+
model,
|
|
2793
|
+
reason:
|
|
2794
|
+
typeof parsed.reason === "string" ? parsed.reason : "model-missing",
|
|
2795
|
+
};
|
|
2796
|
+
} catch {
|
|
2797
|
+
/* try next candidate */
|
|
2798
|
+
}
|
|
2799
|
+
}
|
|
2800
|
+
return { available: false, model, reason: "lib-missing" };
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
async function runLocalHealthJob(job) {
|
|
2804
|
+
const { jobId, payload } = job;
|
|
2805
|
+
const start = Date.now();
|
|
2806
|
+
const ollamaPayloadCount = Array.isArray(payload?.ollamaBaseUrls)
|
|
2807
|
+
? payload.ollamaBaseUrls.length
|
|
2808
|
+
: 0;
|
|
2809
|
+
const mlxUrlCount = Array.isArray(payload?.mlxOpenAiBaseUrls)
|
|
2810
|
+
? payload.mlxOpenAiBaseUrls.length
|
|
2811
|
+
: payload?.mlxOpenAiBaseUrl ? 1 : 0;
|
|
2812
|
+
console.log(
|
|
2813
|
+
`[gonext-worker] local_health ${jobId} start (ollamaUrls=${ollamaPayloadCount}, mlxUrls=${mlxUrlCount})`
|
|
2814
|
+
);
|
|
2815
|
+
const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
2816
|
+
method: "PATCH",
|
|
2817
|
+
body: JSON.stringify({ jobStatus: "running" }),
|
|
2818
|
+
});
|
|
2819
|
+
await ensureWorkerOk(runRes, `mark running local_health jobId=${jobId}`);
|
|
2820
|
+
try {
|
|
2821
|
+
const ollamaBases = Array.isArray(payload?.ollamaBaseUrls)
|
|
2822
|
+
? payload.ollamaBaseUrls.map(normalizeBaseUrl).filter(Boolean)
|
|
2823
|
+
: [];
|
|
2824
|
+
const dedup = new Map();
|
|
2825
|
+
let ollamaOnline = false;
|
|
2826
|
+
let ollamaEndpoint = "";
|
|
2827
|
+
const ollamaSources = [];
|
|
2828
|
+
for (const base of ollamaBases) {
|
|
2829
|
+
const baseStart = Date.now();
|
|
2830
|
+
console.log(`[gonext-worker] local_health ${jobId} check ollama ${base}`);
|
|
2831
|
+
const r = await checkOllamaTags(base);
|
|
2832
|
+
console.log(
|
|
2833
|
+
`[gonext-worker] local_health ${jobId} ollama result ${base} online=${r.online} models=${r.models.length} took=${((Date.now() - baseStart) / 1000).toFixed(2)}s`
|
|
2834
|
+
);
|
|
2835
|
+
ollamaOnline = ollamaOnline || r.online;
|
|
2836
|
+
if (!ollamaEndpoint) ollamaEndpoint = r.endpoint;
|
|
2837
|
+
ollamaSources.push({
|
|
2838
|
+
base,
|
|
2839
|
+
label: sourceLabelFromBase(base),
|
|
2840
|
+
endpoint: r.endpoint,
|
|
2841
|
+
online: r.online,
|
|
2842
|
+
});
|
|
2843
|
+
for (const m of r.models) {
|
|
2844
|
+
if (!dedup.has(m.value)) dedup.set(m.value, m);
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
const targetWorkerHostId =
|
|
2848
|
+
typeof payload?.targetWorkerHostId === "string"
|
|
2849
|
+
? payload.targetWorkerHostId.trim()
|
|
2850
|
+
: "";
|
|
2851
|
+
const targetWorkerHostName =
|
|
2852
|
+
typeof payload?.targetWorkerHostName === "string"
|
|
2853
|
+
? payload.targetWorkerHostName.trim()
|
|
2854
|
+
: "";
|
|
2855
|
+
// MLX servers are discovered dynamically from running mlx_lm.server
|
|
2856
|
+
// processes (macOS), so the dropdown reflects reality with no saved port
|
|
2857
|
+
// list. Only discover when a worker host is targeted so the encoded models
|
|
2858
|
+
// are host-scoped (routable). Each server is probed at 127.0.0.1:<port>/v1.
|
|
2859
|
+
const discoveredMlx = targetWorkerHostId ? await discoverMlxServers() : [];
|
|
2860
|
+
const discoveredByPort = new Map(discoveredMlx.map((s) => [s.port, s]));
|
|
2861
|
+
const mlxRootsDiscovered = discoveredMlx.map(
|
|
2862
|
+
(s) => `http://127.0.0.1:${s.port}/v1`
|
|
2863
|
+
);
|
|
2864
|
+
// Fall back to any explicitly provided base (e.g. a remote host) so
|
|
2865
|
+
// non-local MLX endpoints still work when no local process is found.
|
|
2866
|
+
const mlxRootSingle = normalizeOpenAiV1Root(payload?.mlxOpenAiBaseUrl);
|
|
2867
|
+
const mlxRootsMulti = Array.isArray(payload?.mlxOpenAiBaseUrls)
|
|
2868
|
+
? payload.mlxOpenAiBaseUrls.map(normalizeOpenAiV1Root).filter(Boolean)
|
|
2869
|
+
: [];
|
|
2870
|
+
const mlxRoots =
|
|
2871
|
+
mlxRootsDiscovered.length > 0
|
|
2872
|
+
? mlxRootsDiscovered
|
|
2873
|
+
: mlxRootsMulti.length > 0
|
|
2874
|
+
? mlxRootsMulti
|
|
2875
|
+
: mlxRootSingle
|
|
2876
|
+
? [mlxRootSingle]
|
|
2877
|
+
: [];
|
|
2878
|
+
const mlxRoot = mlxRoots[0] || null;
|
|
2879
|
+
let mlxNative = null;
|
|
2880
|
+
const modelPathChecksRaw = Array.isArray(payload?.modelPathChecks)
|
|
2881
|
+
? payload.modelPathChecks
|
|
2882
|
+
: [];
|
|
2883
|
+
const modelPathChecks = [...new Set(modelPathChecksRaw)]
|
|
2884
|
+
.filter((v) => typeof v === "string" && v.trim())
|
|
2885
|
+
.map((v) => String(v).trim())
|
|
2886
|
+
.slice(0, 8);
|
|
2887
|
+
const modelPathStatus = [];
|
|
2888
|
+
|
|
2889
|
+
// Collect results from all MLX endpoints (one per port)
|
|
2890
|
+
const mlxHttpResults = [];
|
|
2891
|
+
for (const root of mlxRoots) {
|
|
2892
|
+
// Extract port from URL for model value encoding
|
|
2893
|
+
let port = null;
|
|
2894
|
+
try {
|
|
2895
|
+
const u = new URL(root.replace(/\/v1\/?$/i, ""));
|
|
2896
|
+
const p = parseInt(u.port, 10);
|
|
2897
|
+
if (p > 0) port = p;
|
|
2898
|
+
} catch { /* ignore */ }
|
|
2899
|
+
const mlxStart = Date.now();
|
|
2900
|
+
console.log(`[gonext-worker] local_health ${jobId} check mlx HTTP ${root} port=${port ?? "default"}`);
|
|
2901
|
+
const result = await checkOpenAiModels(root, payload?.mlxApiKey ?? "", {
|
|
2902
|
+
workerHostId: targetWorkerHostId,
|
|
2903
|
+
hostName: targetWorkerHostName || sourceLabelFromBase(root),
|
|
2904
|
+
port,
|
|
2905
|
+
});
|
|
2906
|
+
console.log(
|
|
2907
|
+
`[gonext-worker] local_health ${jobId} mlx HTTP online=${result.online} models=${result.models.length} took=${((Date.now() - mlxStart) / 1000).toFixed(2)}s`
|
|
2908
|
+
);
|
|
2909
|
+
mlxHttpResults.push(result);
|
|
2910
|
+
}
|
|
2911
|
+
// Merge all per-port results
|
|
2912
|
+
const mlxModelDedup = new Map();
|
|
2913
|
+
let mlxHttpOnline = false;
|
|
2914
|
+
const portsWithModels = new Set();
|
|
2915
|
+
for (const r of mlxHttpResults) {
|
|
2916
|
+
if (r.online) mlxHttpOnline = true;
|
|
2917
|
+
for (const m of r.models) {
|
|
2918
|
+
const pm = String(m.value ?? "").match(/@@port:(\d+)/);
|
|
2919
|
+
if (pm) portsWithModels.add(parseInt(pm[1], 10));
|
|
2920
|
+
if (!mlxModelDedup.has(m.value)) mlxModelDedup.set(m.value, m);
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2923
|
+
// A discovered server may be booting (process up, /v1/models not ready).
|
|
2924
|
+
// Synthesize an entry from its `--model` path so it is still visible.
|
|
2925
|
+
for (const { port, modelPath } of discoveredByPort.values()) {
|
|
2926
|
+
if (portsWithModels.has(port) || !modelPath) continue;
|
|
2927
|
+
const hostSuffix = targetWorkerHostId ? `@@host:${targetWorkerHostId}` : "";
|
|
2928
|
+
const value = `mlx:${modelPath}${hostSuffix}@@port:${port}`;
|
|
2929
|
+
if (mlxModelDedup.has(value)) continue;
|
|
2930
|
+
const label = targetWorkerHostName || sourceLabelFromBase(`http://127.0.0.1:${port}`);
|
|
2931
|
+
mlxModelDedup.set(value, {
|
|
2932
|
+
id: targetWorkerHostId ? `${modelPath}@@${targetWorkerHostId}` : modelPath,
|
|
2933
|
+
name: `${modelPath} (${label}:${port}) — starting…`,
|
|
2934
|
+
value,
|
|
2935
|
+
});
|
|
2936
|
+
}
|
|
2937
|
+
const mlxHttp = mlxRoots.length > 0 ? {
|
|
2938
|
+
online: mlxHttpOnline,
|
|
2939
|
+
endpoint: mlxHttpResults[0]?.endpoint,
|
|
2940
|
+
models: [...mlxModelDedup.values()],
|
|
2941
|
+
} : null;
|
|
2942
|
+
|
|
2943
|
+
const wantNativeFallback =
|
|
2944
|
+
(targetWorkerHostId || mlxRoot) &&
|
|
2945
|
+
payload?.mlxNativeFallback !== false &&
|
|
2946
|
+
platform() === "darwin" &&
|
|
2947
|
+
(!mlxHttp?.online || (mlxHttp?.models?.length ?? 0) === 0);
|
|
2948
|
+
|
|
2949
|
+
if (wantNativeFallback) {
|
|
2950
|
+
const t0 = Date.now();
|
|
2951
|
+
console.log(
|
|
2952
|
+
`[gonext-worker] local_health ${jobId} mlx native probe (Python mlx_lm import)`
|
|
2953
|
+
);
|
|
2954
|
+
mlxNative = await checkMlxLmNativeImport();
|
|
2955
|
+
console.log(
|
|
2956
|
+
`[gonext-worker] local_health ${jobId} mlx native available=${mlxNative.available} took=${((Date.now() - t0) / 1000).toFixed(2)}s`
|
|
2957
|
+
);
|
|
2958
|
+
}
|
|
2959
|
+
|
|
2960
|
+
for (const rawPath of modelPathChecks) {
|
|
2961
|
+
const check = await checkModelPathExists(rawPath);
|
|
2962
|
+
modelPathStatus.push(check);
|
|
2963
|
+
}
|
|
2964
|
+
|
|
2965
|
+
const whisperModel =
|
|
2966
|
+
typeof payload?.voiceModel === "string" && payload.voiceModel.trim()
|
|
2967
|
+
? payload.voiceModel.trim()
|
|
2968
|
+
: "whisper-large-v3-turbo";
|
|
2969
|
+
let whisper = null;
|
|
2970
|
+
if (platform() === "darwin") {
|
|
2971
|
+
const tW = Date.now();
|
|
2972
|
+
whisper = await checkWhisperAvailability(whisperModel);
|
|
2973
|
+
console.log(
|
|
2974
|
+
`[gonext-worker] local_health ${jobId} whisper available=${whisper.available} reason=${whisper.reason} took=${((Date.now() - tW) / 1000).toFixed(2)}s`
|
|
2975
|
+
);
|
|
2976
|
+
}
|
|
2977
|
+
|
|
2978
|
+
let mlx = null;
|
|
2979
|
+
if (mlxRoot || mlxNative?.available) {
|
|
2980
|
+
const httpOk = Boolean(mlxHttp?.online && (mlxHttp?.models?.length ?? 0) > 0);
|
|
2981
|
+
const nativeOk = mlxNative?.available === true;
|
|
2982
|
+
const hostName = targetWorkerHostName || sourceLabelFromBase(mlxRoot || "mlx");
|
|
2983
|
+
const sources = [
|
|
2984
|
+
{
|
|
2985
|
+
workerHostId: targetWorkerHostId || undefined,
|
|
2986
|
+
hostName,
|
|
2987
|
+
endpoint: mlxHttp?.endpoint,
|
|
2988
|
+
online: httpOk || nativeOk,
|
|
2989
|
+
modelCount: httpOk ? mlxHttp.models.length : nativeOk ? 1 : 0,
|
|
2990
|
+
},
|
|
2991
|
+
];
|
|
2992
|
+
mlx = {
|
|
2993
|
+
configured: httpOk || nativeOk,
|
|
2994
|
+
online: httpOk || nativeOk,
|
|
2995
|
+
models: httpOk
|
|
2996
|
+
? mlxHttp.models
|
|
2997
|
+
: nativeOk
|
|
2998
|
+
? [
|
|
2999
|
+
{
|
|
3000
|
+
id: "mlx_lm_native",
|
|
3001
|
+
name: mlxNative.version
|
|
3002
|
+
? `MLX LM (${mlxNative.version})`
|
|
3003
|
+
: "MLX LM (Python import OK)",
|
|
3004
|
+
value: "mlx:mlx_lm_native",
|
|
3005
|
+
},
|
|
3006
|
+
]
|
|
3007
|
+
: [],
|
|
3008
|
+
endpoint: mlxHttp?.endpoint,
|
|
3009
|
+
http: mlxHttp
|
|
3010
|
+
? {
|
|
3011
|
+
online: mlxHttp.online,
|
|
3012
|
+
endpoint: mlxHttp.endpoint,
|
|
3013
|
+
modelCount: mlxHttp.models.length,
|
|
3014
|
+
}
|
|
3015
|
+
: undefined,
|
|
3016
|
+
native: mlxNative ?? undefined,
|
|
3017
|
+
sources,
|
|
3018
|
+
};
|
|
3019
|
+
}
|
|
3020
|
+
const result = {
|
|
3021
|
+
ollama:
|
|
3022
|
+
ollamaBases.length > 0
|
|
3023
|
+
? {
|
|
3024
|
+
configured: dedup.size > 0,
|
|
3025
|
+
online: ollamaOnline,
|
|
3026
|
+
models: [...dedup.values()],
|
|
3027
|
+
endpoint: ollamaEndpoint,
|
|
3028
|
+
sources: ollamaSources,
|
|
3029
|
+
}
|
|
3030
|
+
: undefined,
|
|
3031
|
+
mlx,
|
|
3032
|
+
whisper: whisper ?? undefined,
|
|
3033
|
+
setup:
|
|
3034
|
+
modelPathStatus.length > 0
|
|
3035
|
+
? {
|
|
3036
|
+
modelPaths: modelPathStatus,
|
|
3037
|
+
}
|
|
3038
|
+
: undefined,
|
|
3039
|
+
// Registered coding workspaces (names/paths only — no secrets) so the web can
|
|
3040
|
+
// show which folders the agent may read/edit on this Mac.
|
|
3041
|
+
workspaces: await readFile(join(homedir(), ".gonext", "workspaces.json"), "utf8")
|
|
3042
|
+
.then((raw) => {
|
|
3043
|
+
const parsed = JSON.parse(raw);
|
|
3044
|
+
return Array.isArray(parsed?.workspaces)
|
|
3045
|
+
? parsed.workspaces.map((w) => ({
|
|
3046
|
+
name: String(w.name ?? ""),
|
|
3047
|
+
path: String(w.path ?? ""),
|
|
3048
|
+
allowRun: Boolean(w.allowRun),
|
|
3049
|
+
}))
|
|
3050
|
+
: undefined;
|
|
3051
|
+
})
|
|
3052
|
+
.catch(() => undefined),
|
|
3053
|
+
};
|
|
3054
|
+
const totalTimeSeconds = (Date.now() - start) / 1000;
|
|
3055
|
+
const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
3056
|
+
method: "PATCH",
|
|
3057
|
+
body: JSON.stringify({
|
|
3058
|
+
jobStatus: "completed",
|
|
3059
|
+
resultText: JSON.stringify(result),
|
|
3060
|
+
tokenCount: 1,
|
|
3061
|
+
totalTimeSeconds,
|
|
3062
|
+
}),
|
|
3063
|
+
});
|
|
3064
|
+
if (doneRes.status === 404) {
|
|
3065
|
+
console.warn(
|
|
3066
|
+
`[gonext-worker] local_health ${jobId} disappeared before completion PATCH (404 Job not found). Skipping.`
|
|
3067
|
+
);
|
|
3068
|
+
return;
|
|
3069
|
+
}
|
|
3070
|
+
await ensureWorkerOk(doneRes, `complete local_health jobId=${jobId}`);
|
|
3071
|
+
const onlineCount = ollamaSources.filter((s) => s.online).length;
|
|
3072
|
+
console.log(
|
|
3073
|
+
`[gonext-worker] completed local_health ${jobId} (${totalTimeSeconds.toFixed(1)}s) summary: ollamaOnline=${onlineCount}/${ollamaSources.length}, mlx=${mlx ? (mlx.online ? "online" : "offline") : "n/a"}`
|
|
3074
|
+
);
|
|
3075
|
+
} catch (e) {
|
|
3076
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
3077
|
+
if (/failed 404/i.test(message) || /job not found/i.test(message)) {
|
|
3078
|
+
console.warn(
|
|
3079
|
+
`[gonext-worker] local_health ${jobId} no longer exists (404). Skipping fail PATCH.`
|
|
3080
|
+
);
|
|
3081
|
+
return;
|
|
3082
|
+
}
|
|
3083
|
+
const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
|
|
3084
|
+
method: "PATCH",
|
|
3085
|
+
body: JSON.stringify({
|
|
3086
|
+
jobStatus: "failed",
|
|
3087
|
+
errorMessage: message,
|
|
3088
|
+
totalTimeSeconds: (Date.now() - start) / 1000,
|
|
3089
|
+
}),
|
|
3090
|
+
});
|
|
3091
|
+
if (!failRes.ok) {
|
|
3092
|
+
const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
|
|
3093
|
+
console.error(
|
|
3094
|
+
`[gonext-worker] local_health fail PATCH also failed ${failRes.status} jobId=${jobId}` +
|
|
3095
|
+
(snippet ? ` response=${snippet}` : "")
|
|
3096
|
+
);
|
|
3097
|
+
}
|
|
3098
|
+
console.error(`[gonext-worker] failed local_health ${jobId}:`, message);
|
|
3099
|
+
}
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
const localHealthInFlight = new Set();
|
|
3103
|
+
|
|
3104
|
+
function trackLocalHealthTask(task) {
|
|
3105
|
+
localHealthInFlight.add(task);
|
|
3106
|
+
task.finally(() => {
|
|
3107
|
+
localHealthInFlight.delete(task);
|
|
3108
|
+
});
|
|
3109
|
+
}
|
|
3110
|
+
|
|
3111
|
+
async function pollOnce() {
|
|
3112
|
+
while (localHealthInFlight.size < localHealthConcurrency) {
|
|
3113
|
+
const res = await workerFetch("/api/worker/jobs/next", { method: "POST" });
|
|
3114
|
+
if (res.status === 204) {
|
|
3115
|
+
return;
|
|
3116
|
+
}
|
|
3117
|
+
if (!res.ok) {
|
|
3118
|
+
const t = await res.text().catch(() => "");
|
|
3119
|
+
throw new Error(`next failed ${res.status}: ${t}`);
|
|
3120
|
+
}
|
|
3121
|
+
const job = await res.json();
|
|
3122
|
+
if (!job?.jobId) {
|
|
3123
|
+
return;
|
|
3124
|
+
}
|
|
3125
|
+
console.log(
|
|
3126
|
+
`[gonext-worker] claimed ${job.jobId} type=${job.jobType ?? "unknown"} modelKey=${job.modelKey ?? "unknown"}`
|
|
3127
|
+
);
|
|
3128
|
+
const isLocalHealthByType = job.jobType === "local_health";
|
|
3129
|
+
const isLocalHealthByModelKey = job.modelKey === "local_health";
|
|
3130
|
+
const isLocalHealthByPayload =
|
|
3131
|
+
Array.isArray(job.payload?.ollamaBaseUrls) ||
|
|
3132
|
+
!!job.payload?.mlxOpenAiBaseUrl ||
|
|
3133
|
+
Array.isArray(job.payload?.mlxOpenAiBaseUrls);
|
|
3134
|
+
if (isLocalHealthByType || isLocalHealthByModelKey || isLocalHealthByPayload) {
|
|
3135
|
+
const task = runLocalHealthJob(job).catch((e) => {
|
|
3136
|
+
console.error("[gonext-worker] local_health task error:", e);
|
|
3137
|
+
});
|
|
3138
|
+
trackLocalHealthTask(task);
|
|
3139
|
+
console.log(
|
|
3140
|
+
`[gonext-worker] local_health in-flight ${localHealthInFlight.size}/${localHealthConcurrency}`
|
|
3141
|
+
);
|
|
3142
|
+
continue;
|
|
3143
|
+
}
|
|
3144
|
+
if (job.jobType === "http_probe") {
|
|
3145
|
+
await runHttpProbeJob(job);
|
|
3146
|
+
return;
|
|
3147
|
+
}
|
|
3148
|
+
if (job.jobType === "list_models") {
|
|
3149
|
+
await runListModelsJob(job);
|
|
3150
|
+
return;
|
|
3151
|
+
}
|
|
3152
|
+
if (job.jobType === "agent_chat") {
|
|
3153
|
+
await runAgentChatJob(job);
|
|
3154
|
+
return;
|
|
3155
|
+
}
|
|
3156
|
+
if (job.jobType === "transcribe") {
|
|
3157
|
+
await runTranscribeJob(job);
|
|
3158
|
+
return;
|
|
3159
|
+
}
|
|
3160
|
+
const isOcrByType = job.jobType === "ocr";
|
|
3161
|
+
const isOcrByPayload =
|
|
3162
|
+
job.payload &&
|
|
3163
|
+
typeof job.payload === "object" &&
|
|
3164
|
+
typeof job.payload.ocrId === "string" &&
|
|
3165
|
+
(typeof job.payload.attachment?.data === "string" ||
|
|
3166
|
+
typeof job.payload.attachment?.s3Object?.getUrl === "string");
|
|
3167
|
+
if (isOcrByType || isOcrByPayload) {
|
|
3168
|
+
await runOcrJob(job);
|
|
3169
|
+
return;
|
|
3170
|
+
}
|
|
3171
|
+
await runChatJob(job);
|
|
3172
|
+
return;
|
|
3173
|
+
}
|
|
3174
|
+
}
|
|
3175
|
+
|
|
3176
|
+
async function main() {
|
|
3177
|
+
// Task #127, Phase 2: record our own pidfile so `status`/`stop` find us whether we were
|
|
3178
|
+
// launched by `start` (detached), by the REPL's auto-ensure, or run in the foreground.
|
|
3179
|
+
// Best-effort; a failure here must not stop the worker from polling.
|
|
3180
|
+
try {
|
|
3181
|
+
const { PID_FILE } = await import("./daemon-control.mjs");
|
|
3182
|
+
const { unlinkSync } = await import("node:fs");
|
|
3183
|
+
await writeFile(PID_FILE, String(process.pid), "utf8");
|
|
3184
|
+
const cleanup = () => {
|
|
3185
|
+
try {
|
|
3186
|
+
unlinkSync(PID_FILE);
|
|
3187
|
+
} catch {
|
|
3188
|
+
/* already gone */
|
|
3189
|
+
}
|
|
3190
|
+
};
|
|
3191
|
+
process.on("SIGTERM", () => {
|
|
3192
|
+
cleanup();
|
|
3193
|
+
process.exit(0);
|
|
3194
|
+
});
|
|
3195
|
+
process.on("SIGINT", () => {
|
|
3196
|
+
cleanup();
|
|
3197
|
+
process.exit(0);
|
|
3198
|
+
});
|
|
3199
|
+
process.on("exit", cleanup);
|
|
3200
|
+
} catch {
|
|
3201
|
+
/* pidfile is a convenience; polling works without it */
|
|
3202
|
+
}
|
|
3203
|
+
console.log(
|
|
3204
|
+
`[gonext-worker] polling ${apiBase} every ${pollMs}ms (local_health concurrency=${localHealthConcurrency})`
|
|
3205
|
+
);
|
|
3206
|
+
for (;;) {
|
|
3207
|
+
try {
|
|
3208
|
+
await pollOnce();
|
|
3209
|
+
} catch (e) {
|
|
3210
|
+
console.error("[gonext-worker] poll error:", e);
|
|
3211
|
+
}
|
|
3212
|
+
await new Promise((r) => setTimeout(r, pollMs));
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
|
|
3216
|
+
main();
|