@tiens.nguyen/gonext-local-worker 1.0.99 → 1.0.101

package/gonext-local-worker.mjs

@@ -1309,6 +1309,171 @@ async function runOcrJob(job) {
   }
 }
 
+const TRANSCRIBE_TIMEOUT_MS =
+  Number(process.env.GONEXT_TRANSCRIBE_TIMEOUT_MS ?? "300000") || 300000;
+
+function resolveWorkerPython() {
+  return (
+    (process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "").trim() ||
+    "python3"
+  );
+}
+
+/** Spawn a Python script, write `input` to stdin, capture stdout/stderr. */
+function runPythonCapture(python, args, input, timeoutMs) {
+  return new Promise((resolve) => {
+    const child = spawn(python, args, { stdio: ["pipe", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    let settled = false;
+    const timer = setTimeout(() => {
+      if (settled) return;
+      settled = true;
+      try {
+        child.kill("SIGKILL");
+      } catch {
+        /* ignore */
+      }
+      resolve({ code: null, stdout, stderr, timedOut: true });
+    }, timeoutMs);
+    child.stdout.on("data", (d) => {
+      stdout += d.toString();
+    });
+    child.stderr.on("data", (d) => {
+      stderr += d.toString();
+    });
+    child.on("error", (e) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve({ code: null, stdout, stderr: stderr + String(e), timedOut: false });
+    });
+    child.on("close", (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve({ code, stdout, stderr, timedOut: false });
+    });
+    try {
+      child.stdin.write(input);
+      child.stdin.end();
+    } catch {
+      /* the error/close handler will settle */
+    }
+  });
+}
+
+async function runTranscribeJob(job) {
+  const { jobId, payload } = job;
+  const start = Date.now();
+  const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
+    method: "PATCH",
+    body: JSON.stringify({ jobStatus: "running" }),
+  });
+  await ensureWorkerOk(runRes, `mark running transcribe jobId=${jobId}`);
+  const tempDir = await mkdtemp(join(tmpdir(), "gonext-transcribe-"));
+  try {
+    if (!payload || typeof payload !== "object") {
+      throw new Error("Invalid transcribe payload.");
+    }
+    const audio = payload.audio;
+    const mimeType = typeof audio?.mimeType === "string" ? audio.mimeType : "";
+    const s3Object = audio && typeof audio === "object" ? audio.s3Object : undefined;
+    const s3GetUrl =
+      s3Object && typeof s3Object.getUrl === "string" ? s3Object.getUrl.trim() : "";
+    const model =
+      typeof payload.model === "string" && payload.model.trim()
+        ? payload.model.trim()
+        : "whisper-large-v3-turbo";
+    const language =
+      typeof payload.language === "string" ? payload.language.trim() : "";
+    if (!mimeType.startsWith("audio/")) {
+      throw new Error("Transcribe job attachment must be audio.");
+    }
+    if (!s3GetUrl) {
+      throw new Error("Transcribe job is missing the audio download URL.");
+    }
+    const dlRes = await fetch(s3GetUrl, { method: "GET" });
+    if (!dlRes.ok) {
+      throw new Error(`Failed to download audio from S3 (${dlRes.status}).`);
+    }
+    const bytes = Buffer.from(await dlRes.arrayBuffer());
+    if (!bytes.length) {
+      throw new Error("Downloaded audio is empty.");
+    }
+    const audioPath = join(tempDir, "input.wav");
+    await writeFile(audioPath, bytes);
+
+    const python = resolveWorkerPython();
+    const scriptPath = join(WORKER_DIR, "gonext_transcribe.py");
+    const { code, stdout, stderr, timedOut } = await runPythonCapture(
+      python,
+      [scriptPath],
+      JSON.stringify({ audioPath, model, language: language || undefined }),
+      TRANSCRIBE_TIMEOUT_MS
+    );
+    if (timedOut) {
+      throw new Error("Transcription timed out.");
+    }
+    if (code !== 0) {
+      const reason = String(stderr ?? "").split("\n")[0]?.trim() || "error";
+      const hint =
+        reason === "lib-missing"
+          ? "Whisper library not installed. Run: pip install mlx-whisper"
+          : reason === "model-missing"
+            ? `Whisper model not downloaded. Run: hf download mlx-community/${model}`
+            : reason === "audio-error"
+              ? "Could not decode the recorded audio."
+              : "Transcription failed.";
+      throw new Error(`${hint} (${compactPreview(stderr)})`);
+    }
+    let text = "";
+    try {
+      const parsed = JSON.parse(String(stdout ?? "").trim() || "{}");
+      text = typeof parsed.text === "string" ? parsed.text.trim() : "";
+    } catch {
+      throw new Error(`Transcription returned malformed output: ${compactPreview(stdout)}`);
+    }
+    if (!text) {
+      throw new Error("Transcription returned empty text.");
+    }
+    const totalTimeSeconds = (Date.now() - start) / 1000;
+    const doneRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
+      method: "PATCH",
+      body: JSON.stringify({
+        jobStatus: "completed",
+        resultText: text,
+        tokenCount: Math.max(1, Math.ceil(text.length / 4)),
+        totalTimeSeconds,
+      }),
+    });
+    await ensureWorkerOk(doneRes, `complete transcribe jobId=${jobId}`);
+    console.log(
+      `[gonext-worker] completed transcribe ${jobId} (${totalTimeSeconds.toFixed(1)}s) model=${model}`
+    );
+  } catch (e) {
+    const message = e instanceof Error ? e.message : String(e);
+    const failRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
+      method: "PATCH",
+      body: JSON.stringify({
+        jobStatus: "failed",
+        errorMessage: message,
+        totalTimeSeconds: (Date.now() - start) / 1000,
+      }),
+    });
+    if (!failRes.ok) {
+      const snippet = (await failRes.text().catch(() => "")).trim().slice(0, 500);
+      console.error(
+        `[gonext-worker] transcribe fail PATCH also failed ${failRes.status} jobId=${jobId}` +
+          (snippet ? ` response=${snippet}` : "")
+      );
+    }
+    console.error(`[gonext-worker] failed transcribe ${jobId}:`, message);
+  } finally {
+    await rm(tempDir, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
 async function runAgentChatJob(job) {
   const { jobId, payload } = job;
   const start = Date.now();
@@ -1618,6 +1783,55 @@ async function checkMlxLmNativeImport() {
   };
 }
 
+// Probe whether voice transcription is ready (Audio support, task #21):
+// mlx-whisper importable AND the requested model present in the HF hub cache
+// (~/.cache/huggingface/hub, honoring HF_HOME/HF_HUB_CACHE). Returns
+// { available, model, reason } where reason is "ok" | "lib-missing" | "model-missing".
+async function checkWhisperAvailability(model) {
+  const preferred =
+    (process.env.GONEXT_PROBE_PYTHON ?? process.env.GONEXT_MLX_LM_PYTHON ?? "").trim() ||
+    "python3";
+  const repoName = `models--mlx-community--${model}`;
+  const code = [
+    "import os, sys, json, glob",
+    "model = sys.argv[1] if len(sys.argv) > 1 else 'whisper-large-v3-turbo'",
+    "try:",
+    "    import mlx_whisper  # noqa: F401",
+    "except Exception:",
+    "    print(json.dumps({'available': False, 'reason': 'lib-missing'})); sys.exit(0)",
+    "home = os.environ.get('HF_HOME') or os.path.join(os.path.expanduser('~'), '.cache', 'huggingface')",
+    "hub = os.environ.get('HF_HUB_CACHE') or os.path.join(home, 'hub')",
+    `repo = os.path.join(hub, ${JSON.stringify(repoName)})`,
+    "snaps = glob.glob(os.path.join(repo, 'snapshots', '*'))",
+    "ok = False",
+    "for s in snaps:",
+    "    if glob.glob(os.path.join(s, '*.safetensors')) or glob.glob(os.path.join(s, '*.npz')) or os.path.exists(os.path.join(s, 'weights.npz')):",
+    "        ok = True; break",
+    "print(json.dumps({'available': ok, 'reason': 'ok' if ok else 'model-missing', 'cache': hub}))",
+  ].join("\n");
+  const candidates = [preferred];
+  if (preferred === "python3") candidates.push("python");
+  for (const exe of [...new Set(candidates)]) {
+    try {
+      const { stdout } = await execFile(exe, ["-c", code, model], {
+        timeout: 15000,
+        maxBuffer: 65536,
+        windowsHide: true,
+      });
+      const parsed = JSON.parse(String(stdout ?? "").trim() || "{}");
+      return {
+        available: parsed.available === true,
+        model,
+        reason:
+          typeof parsed.reason === "string" ? parsed.reason : "model-missing",
+      };
+    } catch {
+      /* try next candidate */
+    }
+  }
+  return { available: false, model, reason: "lib-missing" };
+}
+
 async function runLocalHealthJob(job) {
   const { jobId, payload } = job;
   const start = Date.now();
@@ -1746,6 +1960,19 @@ async function runLocalHealthJob(job) {
       modelPathStatus.push(check);
     }
 
+    const whisperModel =
+      typeof payload?.voiceModel === "string" && payload.voiceModel.trim()
+        ? payload.voiceModel.trim()
+        : "whisper-large-v3-turbo";
+    let whisper = null;
+    if (platform() === "darwin") {
+      const tW = Date.now();
+      whisper = await checkWhisperAvailability(whisperModel);
+      console.log(
+        `[gonext-worker] local_health ${jobId} whisper available=${whisper.available} reason=${whisper.reason} took=${((Date.now() - tW) / 1000).toFixed(2)}s`
+      );
+    }
+
     let mlx = null;
     if (mlxRoot || mlxNative?.available) {
       const httpOk = Boolean(mlxHttp?.online && (mlxHttp?.models?.length ?? 0) > 0);
@@ -1800,6 +2027,7 @@ async function runLocalHealthJob(job) {
             }
           : undefined,
       mlx,
+      whisper: whisper ?? undefined,
       setup:
         modelPathStatus.length > 0
           ? {
@@ -1905,6 +2133,10 @@ async function pollOnce() {
       await runAgentChatJob(job);
       return;
     }
+    if (job.jobType === "transcribe") {
+      await runTranscribeJob(job);
+      return;
+    }
     const isOcrByType = job.jobType === "ocr";
     const isOcrByPayload =
       job.payload &&

package/gonext_transcribe.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Voice transcription for the GoNext local worker (Audio support, task #21).
+
+Reads a JSON job on stdin: {"audioPath": str, "model": str, "language": str?}.
+Transcribes with mlx-whisper and prints {"text": "..."} on stdout.
+
+Design notes:
+- The browser sends a 16 kHz mono 16-bit PCM WAV, so we decode it with the
+  stdlib `wave` module + numpy and hand mlx-whisper a float32 array. Passing an
+  array (not a file path) skips mlx-whisper's ffmpeg-based loader, so NO ffmpeg
+  install is required on the worker.
+- The model is resolved by HF repo id ("mlx-community/<model>") from the HF hub
+  cache (~/.cache/huggingface/hub, honoring HF_HOME/HF_HUB_CACHE). This is the
+  same location the wizard installs to and the readiness probe checks.
+- On failure we exit non-zero and print a machine-parseable reason token as the
+  first stderr line: "lib-missing" | "model-missing" | "audio-error" | "error".
+"""
+
+import json
+import sys
+import wave
+
+
+TARGET_SR = 16000
+
+
+def fail(reason: str, detail: str = "") -> None:
+    sys.stderr.write(reason + "\n")
+    if detail:
+        sys.stderr.write(detail + "\n")
+    sys.exit(1)
+
+
+def load_wav_mono_16k(path: str):
+    """Decode a PCM WAV to a float32 mono array at 16 kHz without ffmpeg."""
+    import numpy as np
+
+    with wave.open(path, "rb") as w:
+        n_channels = w.getnchannels()
+        sampwidth = w.getsampwidth()
+        framerate = w.getframerate()
+        n_frames = w.getnframes()
+        raw = w.readframes(n_frames)
+
+    if sampwidth == 2:
+        audio = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0
+    elif sampwidth == 4:
+        audio = np.frombuffer(raw, dtype=np.int32).astype(np.float32) / 2147483648.0
+    elif sampwidth == 1:
+        audio = (np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 128.0) / 128.0
+    else:
+        raise ValueError(f"unsupported WAV sample width: {sampwidth} bytes")
+
+    if n_channels > 1:
+        audio = audio.reshape(-1, n_channels).mean(axis=1)
+
+    if framerate != TARGET_SR and audio.size > 0:
+        # Linear resample; the browser already sends 16 kHz so this is a safety net.
+        duration = audio.size / float(framerate)
+        target_len = int(round(duration * TARGET_SR))
+        if target_len > 0:
+            src_x = np.linspace(0.0, 1.0, num=audio.size, endpoint=False)
+            dst_x = np.linspace(0.0, 1.0, num=target_len, endpoint=False)
+            audio = np.interp(dst_x, src_x, audio).astype(np.float32)
+
+    return np.ascontiguousarray(audio, dtype=np.float32)
+
+
+def main() -> None:
+    try:
+        payload = json.loads(sys.stdin.read() or "{}")
+    except Exception as e:  # noqa: BLE001
+        fail("error", f"invalid job json: {e}")
+        return
+
+    audio_path = str(payload.get("audioPath") or "").strip()
+    model = str(payload.get("model") or "whisper-large-v3-turbo").strip()
+    language = str(payload.get("language") or "").strip() or None
+    if not audio_path:
+        fail("error", "missing audioPath")
+        return
+
+    try:
+        import mlx_whisper  # noqa: F401
+    except Exception as e:  # noqa: BLE001
+        fail("lib-missing", str(e))
+        return
+
+    try:
+        audio = load_wav_mono_16k(audio_path)
+    except Exception as e:  # noqa: BLE001
+        fail("audio-error", str(e))
+        return
+
+    repo = f"mlx-community/{model}"
+    try:
+        result = mlx_whisper.transcribe(
+            audio,
+            path_or_hf_repo=repo,
+            task="transcribe",
+            language=language,
+        )
+    except Exception as e:  # noqa: BLE001
+        msg = str(e).lower()
+        if any(
+            token in msg
+            for token in (
+                "not found",
+                "repository",
+                "couldn't find",
+                "could not find",
+                "no such file",
+                "connection",
+                "offline",
+                "resolve",
+                "404",
+            )
+        ):
+            fail("model-missing", str(e))
+        else:
+            fail("error", str(e))
+        return
+
+    text = str(result.get("text", "")).strip() if isinstance(result, dict) else ""
+    sys.stdout.write(json.dumps({"text": text}))
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.99",
+  "version": "1.0.101",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",
@@ -24,6 +24,7 @@
     "gonext-local-worker.mjs",
     "gonext_probe_agent.py",
     "gonext_agent_chat.py",
+    "gonext_transcribe.py",
     "README.md",
     "launchd/"
   ],