@tiens.nguyen/gonext-local-worker 1.0.41 → 1.0.44

package/gonext-local-worker.mjs

@@ -9,6 +9,7 @@
  */
 import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { execFile as execFileCallback } from "node:child_process";
+import { createHash } from "node:crypto";
 import { homedir, platform, tmpdir } from "node:os";
 import { extname, join } from "node:path";
 import { promisify } from "node:util";
@@ -24,6 +25,12 @@ dotenv.config();
 
 const args = process.argv.slice(2);
 
+function workerKeyFingerprint(secret) {
+  const v = String(secret ?? "").trim();
+  if (!v) return "";
+  return createHash("sha256").update(v, "utf8").digest("hex").slice(0, 12);
+}
+
 function printHelp() {
   console.log(`
 gonext-local-worker
@@ -44,6 +51,7 @@ Env (optional):
   GONEXT_SIMULATE_TEXT   default body for simulate-chat when no args
   GONEXT_MLX_LM_PYTHON   Python executable for MLX LM native probe (default: python3)
   GONEXT_OCR_MODEL_PATH  Local GLM-OCR-bf16 directory (default: ~/mlx-models/GLM-OCR-bf16)
+  GONEXT_OCR_DEBUG=1     Print raw OCR stdout/stderr snippets for troubleshooting
 `);
 }
 
@@ -96,16 +104,18 @@ if (args[0] === "set") {
   process.exit(0);
 }
 
-const apiBase = (process.env.GONEXT_API_BASE ?? "").replace(/\/+$/, "");
-const workerKey = process.env.GONEXT_WORKER_KEY ?? "";
+const apiBase = String(process.env.GONEXT_API_BASE ?? "")
+  .trim()
+  .replace(/\/+$/, "");
+const workerKey = String(process.env.GONEXT_WORKER_KEY ?? "").trim();
 const pollMs = 500;
 const localHealthConcurrency = 4;
 
 const CHUNK_PATH = "/api/worker/job-chunk";
-const OCR_PROMPT =
-  "Extract all readable text from this image. Return plain text only and preserve line breaks.";
+const OCR_PROMPT = "Text Recognition:";
 const OCR_MAX_TOKENS = 2048;
 const OCR_TIMEOUT_MS = 180_000;
+const OCR_DEBUG = String(process.env.GONEXT_OCR_DEBUG ?? "").trim() === "1";
 
 async function workerFetch(path, init = {}) {
   const url = `${apiBase}${path.startsWith("/") ? path : `/${path}`}`;
@@ -125,13 +135,23 @@ async function ensureWorkerOk(res, context) {
   );
 }
 
-if (args[0] === "ws-ping-test") {
-  if (!apiBase || !workerKey) {
+function assertWorkerRuntimeConfig(mode) {
+  if (!workerKey) {
+    console.error(
+      `[gonext-worker] ${mode}: missing GONEXT_WORKER_KEY. Run: gonext-local-worker set <workerKey> --api-base <url>`
+    );
+    process.exit(1);
+  }
+  if (!apiBase) {
     console.error(
-      "ws-ping-test needs GONEXT_API_BASE and GONEXT_WORKER_KEY (run: gonext-local-worker set <key> --api-base <url>)"
+      `[gonext-worker] ${mode}: missing GONEXT_API_BASE. Run: gonext-local-worker set <workerKey> --api-base <url>`
     );
     process.exit(1);
   }
+}
+
+if (args[0] === "ws-ping-test") {
+  assertWorkerRuntimeConfig("ws-ping-test");
   const url = `${apiBase}/api/worker/ws-ping-test`;
   const res = await fetch(url, {
     method: "POST",
@@ -151,17 +171,21 @@ if (args[0] === "ws-ping-test") {
     }
     process.exit(1);
   }
-  console.log("[gonext-worker] ws-ping-test OK:", text);
+  try {
+    const parsed = JSON.parse(text);
+    console.log(
+      `[gonext-worker] ws-ping-test OK user=${parsed.userId ?? "unknown"} host=${
+        parsed.workerHostId ?? "legacy"
+      } keyfp=${parsed.workerKeyFingerprint ?? workerKeyFingerprint(workerKey)}`
+    );
+  } catch {
+    console.log("[gonext-worker] ws-ping-test OK:", text);
+  }
   process.exit(0);
 }
 
 if (args[0] === "simulate-chat") {
-  if (!apiBase || !workerKey) {
-    console.error(
-      "simulate-chat needs GONEXT_API_BASE and GONEXT_WORKER_KEY (run: gonext-local-worker set <key> --api-base <url>)"
-    );
-    process.exit(1);
-  }
+  assertWorkerRuntimeConfig("simulate-chat");
   const reply =
     args.slice(1).join(" ").trim() ||
     String(process.env.GONEXT_SIMULATE_TEXT ?? "").trim() ||
@@ -263,15 +287,12 @@ if (args[0] === "simulate-chat") {
   process.exit(0);
 }
 
-if (!apiBase || !workerKey) {
-  console.error(
-    "Set GONEXT_API_BASE (HTTP API origin, no /api suffix) and GONEXT_WORKER_KEY."
-  );
-  process.exit(1);
-}
+assertWorkerRuntimeConfig("worker");
 
 console.log(
-  `[gonext-worker] API base ${apiBase} — streaming chunks POST ${apiBase}${CHUNK_PATH}`
+  `[gonext-worker] API base ${apiBase} — streaming chunks POST ${apiBase}${CHUNK_PATH} — keyfp=${workerKeyFingerprint(
+    workerKey
+  )}`
 );
 
 function toOpenAIMessages(messages) {
@@ -522,10 +543,18 @@ function resolveImageExtension(mimeType, fileName) {
   return ext || ".png";
 }
 
-function resolveOcrModelPath() {
-  const raw = String(process.env.GONEXT_OCR_MODEL_PATH ?? "").trim();
-  if (raw) {
-    return raw.replace(/^~(?=\/)/, homedir());
+function resolveOcrModelPath(modelFromJob) {
+  const fromJob = String(modelFromJob ?? "").trim();
+  if (fromJob) {
+    const expanded = fromJob.replace(/^~(?=\/)/, homedir());
+    if (expanded.startsWith("/")) {
+      return expanded;
+    }
+    return join(homedir(), "mlx-models", expanded);
+  }
+  const fromEnv = String(process.env.GONEXT_OCR_MODEL_PATH ?? "").trim();
+  if (fromEnv) {
+    return fromEnv.replace(/^~(?=\/)/, homedir());
   }
   return join(homedir(), "mlx-models", "GLM-OCR-bf16");
 }
@@ -581,7 +610,7 @@ async function runMlxVlmGenerate(modelPath, imagePath) {
     String(OCR_MAX_TOKENS),
   ];
   try {
-    return await execFile(
+    const result = await execFile(
       "python3",
       ["-m", "mlx_vlm.generate", ...sharedArgs],
       {
@@ -589,6 +618,7 @@ async function runMlxVlmGenerate(modelPath, imagePath) {
         maxBuffer: 10 * 1024 * 1024,
       }
     );
+    return { ...result, invocation: "python3 -m mlx_vlm.generate" };
   } catch (primaryError) {
     const stderr =
       primaryError && typeof primaryError === "object" && "stderr" in primaryError
@@ -602,7 +632,7 @@ async function runMlxVlmGenerate(modelPath, imagePath) {
     if (!missingLegacyModule) {
       throw primaryError;
     }
-    return execFile(
+    const result = await execFile(
       "python3",
       ["-m", "mlx_vlm", "generate", ...sharedArgs],
       {
@@ -610,9 +640,24 @@ async function runMlxVlmGenerate(modelPath, imagePath) {
         maxBuffer: 10 * 1024 * 1024,
       }
     );
+    return { ...result, invocation: "python3 -m mlx_vlm generate" };
   }
 }
 
+function compactPreview(text, max = 800) {
+  const normalized = String(text ?? "")
+    .replace(/\r\n/g, "\n")
+    .replace(/\s+/g, " ")
+    .trim();
+  if (!normalized) {
+    return "(empty)";
+  }
+  if (normalized.length <= max) {
+    return normalized;
+  }
+  return `${normalized.slice(0, max)} …[truncated ${normalized.length - max} chars]`;
+}
+
 async function runOcrJob(job) {
   const { jobId, payload } = job;
   const start = Date.now();
@@ -628,17 +673,41 @@ async function runOcrJob(job) {
     const attachment = payload.attachment;
     const mimeType = typeof attachment?.mimeType === "string" ? attachment.mimeType : "";
     const data = typeof attachment?.data === "string" ? attachment.data : "";
+    const s3Object =
+      attachment && typeof attachment === "object" ? attachment.s3Object : undefined;
+    const s3GetUrl =
+      s3Object && typeof s3Object.getUrl === "string" ? s3Object.getUrl.trim() : "";
+    const sourceLabel = s3GetUrl
+      ? `s3://${s3Object?.bucket ?? "unknown"}/${s3Object?.key ?? "unknown"}`
+      : "inline_base64";
+    const ocrModel =
+      typeof payload.model === "string" && payload.model.trim()
+        ? payload.model.trim()
+        : "GLM-OCR-bf16";
     const name = typeof attachment?.name === "string" ? attachment.name : "";
     if (!mimeType.startsWith("image/")) {
       throw new Error("OCR job attachment must be an image.");
     }
-    if (!data) {
+    if (!data && !s3GetUrl) {
       throw new Error("OCR job attachment is empty.");
     }
-    const bytes = Buffer.from(data, "base64");
+    let bytes;
+    if (s3GetUrl) {
+      const dlRes = await fetch(s3GetUrl, { method: "GET" });
+      if (!dlRes.ok) {
+        throw new Error(
+          `Failed to download OCR image from S3 (${dlRes.status}).`
+        );
+      }
+      const arr = await dlRes.arrayBuffer();
+      bytes = Buffer.from(arr);
+    } else {
+      bytes = Buffer.from(data, "base64");
+    }
     if (!bytes.length) {
-      throw new Error("OCR job attachment data is not valid base64.");
+      throw new Error("OCR job attachment data is not valid.");
     }
+    console.log(`[gonext-worker] OCR source jobId=${jobId} ${sourceLabel}`);
 
     const tempDir = await mkdtemp(join(tmpdir(), "gonext-ocr-worker-"));
     let extractedText = "";
@@ -648,16 +717,27 @@ async function runOcrJob(job) {
         `input${resolveImageExtension(mimeType, name)}`
       );
       await writeFile(imagePath, bytes);
-      const modelPath = resolveOcrModelPath();
-      const { stdout } = await runMlxVlmGenerate(modelPath, imagePath);
+      const modelPath = resolveOcrModelPath(ocrModel);
+      const { stdout, stderr, invocation } = await runMlxVlmGenerate(
+        modelPath,
+        imagePath
+      );
+      if (OCR_DEBUG) {
+        console.log(
+          `[gonext-worker][ocr-debug] jobId=${jobId} invocation=${invocation} model=${modelPath} image=${imagePath} source=${
+            sourceLabel
+          } stdout=${compactPreview(stdout)} stderr=${compactPreview(stderr)}`
+        );
+      }
       extractedText = normalizeOcrOutput(stdout);
+      if (!extractedText) {
+        throw new Error(
+          `OCR returned empty text for this image. invocation=${invocation} stdout=${compactPreview(stdout)} stderr=${compactPreview(stderr)}`
+        );
+      }
     } finally {
       await rm(tempDir, { recursive: true, force: true }).catch(() => {});
     }
-
-    if (!extractedText) {
-      throw new Error("OCR returned empty text for this image.");
-    }
     const totalTimeSeconds = (Date.now() - start) / 1000;
     const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
       method: "PATCH",
@@ -670,7 +750,7 @@ async function runOcrJob(job) {
     });
     await ensureWorkerOk(doneRes, `complete OCR jobId=${jobId}`);
     console.log(
-      `[gonext-worker] completed OCR ${jobId} (${totalTimeSeconds.toFixed(1)}s)`
+      `[gonext-worker] completed OCR ${jobId} (${totalTimeSeconds.toFixed(1)}s) model=${ocrModel}`
     );
   } catch (e) {
     const message = e instanceof Error ? e.message : String(e);
@@ -742,20 +822,30 @@ async function checkOllamaTags(base) {
   }
 }
 
-async function checkOpenAiModels(base, apiKey) {
+async function checkOpenAiModels(base, apiKey, source) {
   const endpoint = `${base}/models`;
   const headers = {};
   if (apiKey?.trim()) {
     headers.Authorization = `Bearer ${apiKey.trim()}`;
   }
+  const hostSourceId =
+    typeof source?.workerHostId === "string" ? source.workerHostId.trim() : "";
+  const hostSourceLabel =
+    typeof source?.hostName === "string" ? source.hostName.trim() : "";
   try {
     const res = await fetch(endpoint, { method: "GET", headers });
     if (!res.ok) return { online: false, endpoint, models: [] };
     const j = await res.json();
+    const modelSuffix = hostSourceId ? `@@host:${hostSourceId}` : "";
+    const displaySuffix = hostSourceLabel ? ` (${hostSourceLabel})` : "";
     const models = (j.data ?? [])
       .map((d) => d.id)
       .filter(Boolean)
-      .map((id) => ({ id, name: id, value: `mlx:${id}` }));
+      .map((id) => ({
+        id: hostSourceId ? `${id}@@${hostSourceId}` : id,
+        name: `${id}${displaySuffix}`,
+        value: `mlx:${id}${modelSuffix}`,
+      }));
     return { online: true, endpoint, models };
   } catch {
     return { online: false, endpoint, models: [] };
@@ -848,13 +938,24 @@ async function runLocalHealthJob(job) {
       }
     }
     const mlxRoot = normalizeOpenAiV1Root(payload?.mlxOpenAiBaseUrl);
+    const targetWorkerHostId =
+      typeof payload?.targetWorkerHostId === "string"
+        ? payload.targetWorkerHostId.trim()
+        : "";
+    const targetWorkerHostName =
+      typeof payload?.targetWorkerHostName === "string"
+        ? payload.targetWorkerHostName.trim()
+        : "";
     let mlxHttp = null;
     let mlxNative = null;
 
     if (mlxRoot) {
       const mlxStart = Date.now();
       console.log(`[gonext-worker] local_health ${jobId} check mlx HTTP ${mlxRoot}`);
-      mlxHttp = await checkOpenAiModels(mlxRoot, payload?.mlxApiKey ?? "");
+      mlxHttp = await checkOpenAiModels(mlxRoot, payload?.mlxApiKey ?? "", {
+        workerHostId: targetWorkerHostId,
+        hostName: targetWorkerHostName || sourceLabelFromBase(mlxRoot),
+      });
       console.log(
         `[gonext-worker] local_health ${jobId} mlx HTTP online=${mlxHttp.online} models=${mlxHttp.models.length} took=${((Date.now() - mlxStart) / 1000).toFixed(2)}s`
       );
@@ -881,6 +982,16 @@ async function runLocalHealthJob(job) {
     if (mlxRoot || mlxNative?.available) {
       const httpOk = Boolean(mlxHttp?.online && (mlxHttp?.models?.length ?? 0) > 0);
       const nativeOk = mlxNative?.available === true;
+      const hostName = targetWorkerHostName || sourceLabelFromBase(mlxRoot || "mlx");
+      const sources = [
+        {
+          workerHostId: targetWorkerHostId || undefined,
+          hostName,
+          endpoint: mlxHttp?.endpoint,
+          online: httpOk || nativeOk,
+          modelCount: httpOk ? mlxHttp.models.length : nativeOk ? 1 : 0,
+        },
+      ];
       mlx = {
         configured: httpOk || nativeOk,
         online: httpOk || nativeOk,
@@ -906,6 +1017,7 @@ async function runLocalHealthJob(job) {
             }
           : undefined,
         native: mlxNative ?? undefined,
+        sources,
       };
     }
     const result = {
@@ -931,6 +1043,12 @@ async function runLocalHealthJob(job) {
         totalTimeSeconds,
       }),
     });
+    if (doneRes.status === 404) {
+      console.warn(
+        `[gonext-worker] local_health ${jobId} disappeared before completion PATCH (404 Job not found). Skipping.`
+      );
+      return;
+    }
     await ensureWorkerOk(doneRes, `complete local_health jobId=${jobId}`);
     const onlineCount = ollamaSources.filter((s) => s.online).length;
     console.log(
@@ -938,6 +1056,12 @@ async function runLocalHealthJob(job) {
     );
   } catch (e) {
     const message = e instanceof Error ? e.message : String(e);
+    if (/failed 404/i.test(message) || /job not found/i.test(message)) {
+      console.warn(
+        `[gonext-worker] local_health ${jobId} no longer exists (404). Skipping fail PATCH.`
+      );
+      return;
+    }
     const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
       method: "PATCH",
       body: JSON.stringify({
@@ -1002,7 +1126,8 @@ async function pollOnce() {
       job.payload &&
       typeof job.payload === "object" &&
       typeof job.payload.ocrId === "string" &&
-      typeof job.payload.attachment?.data === "string";
+      (typeof job.payload.attachment?.data === "string" ||
+        typeof job.payload.attachment?.s3Object?.getUrl === "string");
     if (isOcrByType || isOcrByPayload) {
       await runOcrJob(job);
       return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.41",
+  "version": "1.0.44",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",