@agentprojectcontext/apx 1.15.5 → 1.16.0

package/src/daemon/transcription.js

@@ -28,8 +28,9 @@
 
 import fs from "node:fs";
 import path from "node:path";
-import { spawn } from "node:child_process";
+import { spawn, exec } from "node:child_process";
 import { fileURLToPath } from "node:url";
+import { logInfo, logWarn, logError } from "../core/logging.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -43,8 +44,32 @@ const DEFAULT_LOCAL = {
   language: "auto",
   beam_size: 5,
   idle_minutes: 10,
+  // Max time we wait for /transcribe to return. Long audio files (Telegram
+  // voice notes > 10 min) can take several minutes on CPU; the previous
+  // hard-coded 5-minute cap silently truncated them. 20 minutes covers a
+  // ~60-minute voice note on a small int8 model. Override with
+  // transcription.local.timeout_ms in ~/.apx/config.json if needed.
+  timeout_ms: 20 * 60_000,
 };
 
+// ---------------------------------------------------------------------------
+// Config helpers (pure — exported for tests)
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve the effective transcription language.
+ * Priority: explicit local config → config.user.language → "auto" (whisper detects).
+ *
+ * @param {object} localCfg   merged transcription.local config
+ * @param {string} userLang   config.user.language ISO code (e.g. "es"), or ""
+ * @returns {string}          ISO code or "auto"
+ */
+export function resolveTranscriptionLanguage(localCfg, userLang) {
+  if (localCfg.language && localCfg.language !== "auto") return localCfg.language;
+  if (userLang) return userLang;
+  return "auto";
+}
+
 // ---------------------------------------------------------------------------
 // Config
 // ---------------------------------------------------------------------------
@@ -59,9 +84,7 @@ async function getConfig() {
     // Explicit transcription.local.language always wins; "auto" means fall back to user.language.
     const userLang = cfg.user?.language || "";
     const localBase = { ...DEFAULT_LOCAL, ...(t.local || {}) };
-    if ((!localBase.language || localBase.language === "auto") && userLang) {
-      localBase.language = userLang;
-    }
+    localBase.language = resolveTranscriptionLanguage(localBase, userLang);
     return {
       provider: t.provider || "auto",
       local: localBase,
@@ -98,6 +121,21 @@ async function _isServerHealthy() {
   }
 }
 
+// Check if the running whisper-server is using a specific model.
+// Returns the model name string, or null if not reachable.
+async function _serverModelName() {
+  try {
+    const res = await fetch(`http://127.0.0.1:${WHISPER_PORT}/health`, {
+      signal: AbortSignal.timeout(800),
+    });
+    if (!res.ok) return null;
+    const j = await res.json();
+    return j?.model || null;
+  } catch {
+    return null;
+  }
+}
+
 async function _waitForServer(maxMs = 15_000) {
   const deadline = Date.now() + maxMs;
   while (Date.now() < deadline) {
@@ -107,18 +145,62 @@ async function _waitForServer(maxMs = 15_000) {
   throw new Error(`whisper-server did not start within ${maxMs}ms`);
 }
 
+// Find the PID of the process LISTENing on the whisper port (server only,
+// not clients). Filtering by -sTCP:LISTEN is critical — without it, lsof
+// also returns clients with an open connection (including this daemon).
+async function _findListenerPid() {
+  return new Promise((resolve) => {
+    exec(`lsof -ti tcp:${WHISPER_PORT} -sTCP:LISTEN`, (err, stdout) => {
+      if (err || !stdout) return resolve(null);
+      const candidates = stdout.trim().split("\n")
+        .map(s => parseInt(s, 10))
+        .filter(n => Number.isFinite(n) && n !== process.pid);
+      resolve(candidates[0] || null);
+    });
+  });
+}
+
+async function _killOrphanWhisper() {
+  // First try graceful /shutdown on the whisper server.
+  try {
+    await fetch(`http://127.0.0.1:${WHISPER_PORT}/shutdown`, {
+      method: "POST", signal: AbortSignal.timeout(1000),
+    });
+    await _sleep(600);
+  } catch {}
+  // If still bound, force-kill the LISTENER pid only (never our own pid).
+  const pid = await _findListenerPid();
+  if (pid && pid !== process.pid) {
+    try { process.kill(pid, "SIGTERM"); } catch {}
+    await _sleep(400);
+    try { process.kill(pid, 0); try { process.kill(pid, "SIGKILL"); } catch {} } catch {}
+    await _sleep(300);
+  }
+}
+
 async function ensureWhisperServer(opts) {
   const model = opts.model || DEFAULT_LOCAL.model;
 
   // Already running with the right model — health-check to confirm still alive.
   if (_serverProcess && _serverModel === model) {
     if (await _isServerHealthy()) return;
-    // Process died (idle shutdown). Fall through to restart.
     _serverProcess = null;
     _serverModel = null;
   }
 
-  // Wrong model: kill old server and start fresh.
+  // Adopt an externally-running whisper-server (e.g. left over from prior daemon).
+  if (!_serverProcess) {
+    const existing = await _serverModelName();
+    if (existing === model) {
+      _serverModel = model;
+      return;
+    }
+    if (existing) {
+      // Wrong model: kick out the orphan so we can start the right one.
+      await _killOrphanWhisper();
+    }
+  }
+
   if (_serverProcess) {
     try { _serverProcess.kill(); } catch {}
     _serverProcess = null;
@@ -126,6 +208,10 @@ async function ensureWhisperServer(opts) {
     await _sleep(300);
   }
 
+  await _spawnWhisper(opts, model, /* retried */ false);
+}
+
+async function _spawnWhisper(opts, model, retried) {
   const args = [
     WHISPER_SERVER,
     "--port", String(WHISPER_PORT),
@@ -151,32 +237,44 @@ async function ensureWhisperServer(opts) {
   });
 
   // Wait for the "ready" line on stdout, then wait for HTTP to respond.
-  await new Promise((resolve, reject) => {
-    const timeout = setTimeout(
-      () => reject(new Error("whisper-server startup timed out (15s)")),
-      15_000
-    );
-    let buf = "";
-    proc.stdout.on("data", (chunk) => {
-      buf += chunk.toString();
-      const nl = buf.indexOf("\n");
-      if (nl === -1) return;
-      const line = buf.slice(0, nl).trim();
-      buf = buf.slice(nl + 1);
-      clearTimeout(timeout);
-      try {
-        const msg = JSON.parse(line);
-        if (msg.status === "error") return reject(new Error(msg.error || "whisper-server error"));
-        resolve(); // "ready"
-      } catch {
-        resolve(); // unexpected line but server is up
-      }
-    });
-    proc.on("exit", (code) => {
-      clearTimeout(timeout);
-      reject(new Error(`whisper-server exited (code ${code}) before becoming ready`));
+  try {
+    await new Promise((resolve, reject) => {
+      const timeout = setTimeout(
+        () => reject(new Error("whisper-server startup timed out (15s)")),
+        15_000
+      );
+      let buf = "";
+      proc.stdout.on("data", (chunk) => {
+        buf += chunk.toString();
+        const nl = buf.indexOf("\n");
+        if (nl === -1) return;
+        const line = buf.slice(0, nl).trim();
+        buf = buf.slice(nl + 1);
+        clearTimeout(timeout);
+        try {
+          const msg = JSON.parse(line);
+          if (msg.status === "error") return reject(new Error(msg.error || "whisper-server error"));
+          resolve(); // "ready"
+        } catch {
+          resolve(); // unexpected line but server is up
+        }
+      });
+      proc.on("exit", (code) => {
+        clearTimeout(timeout);
+        reject(new Error(`whisper-server exited (code ${code}) before becoming ready`));
+      });
     });
-  });
+  } catch (e) {
+    // Self-heal: if the port was already in use, kill the orphan and retry once.
+    const msg = e.message || "";
+    if (!retried && /address already in use|errno 48|eaddrinuse/i.test(msg)) {
+      _serverProcess = null;
+      _serverModel = null;
+      await _killOrphanWhisper();
+      return _spawnWhisper(opts, model, /* retried */ true);
+    }
+    throw e;
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -190,30 +288,74 @@ async function transcribeLocal(filePath, opts) {
     ? null
     : (opts.language || null);
 
-  const res = await fetch(`http://127.0.0.1:${WHISPER_PORT}/transcribe`, {
-    method: "POST",
-    headers: { "content-type": "application/json" },
-    body: JSON.stringify({
-      audio_path: filePath,
-      language,
-      beam_size: opts.beam_size || DEFAULT_LOCAL.beam_size,
-    }),
-    signal: AbortSignal.timeout(5 * 60_000),
-  });
+  const timeoutMs = Number(opts.timeout_ms) > 0
+    ? Number(opts.timeout_ms)
+    : DEFAULT_LOCAL.timeout_ms;
 
-  const json = await res.json();
-  if (!json.ok) throw new Error(json.error || "transcription failed");
+  const body = JSON.stringify({
+    audio_path: filePath,
+    language,
+    beam_size: opts.beam_size || DEFAULT_LOCAL.beam_size,
+  });
 
-  return {
-    ok: true,
-    backend: "local",
-    text: json.text || "",
-    language: json.language || null,
-    language_probability: json.language_probability ?? null,
-    duration: json.duration ?? null,
-    model: json.model,
-    compute_type: json.compute_type,
-  };
+  // Long transcriptions on CPU (small int8, 1-minute voice note) can take
+  // 30-45s. Under undici (Node fetch) we occasionally see "fetch failed"
+  // from the inbound Telegram path even though the whisper-server completes
+  // the request successfully — a keep-alive socket gets reset somewhere
+  // between the long whisper-server response and the daemon's other
+  // concurrent traffic. We retry once on a generic "fetch failed" so the
+  // user actually gets a reply.
+  const maxAttempts = 2;
+  let lastErr = null;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    const t0 = Date.now();
+    try {
+      logInfo("whisper", `transcribeLocal attempt ${attempt}/${maxAttempts}`, {
+        file: path.basename(filePath),
+        language: language || "auto",
+        timeout_ms: timeoutMs,
+      });
+      const res = await fetch(`http://127.0.0.1:${WHISPER_PORT}/transcribe`, {
+        method: "POST",
+        headers: { "content-type": "application/json", "connection": "close" },
+        body,
+        signal: AbortSignal.timeout(timeoutMs),
+      });
+      const json = await res.json();
+      if (!json.ok) throw new Error(json.error || "transcription failed");
+      logInfo("whisper", `transcribeLocal ok in ${Date.now() - t0}ms`, {
+        chars: (json.text || "").length,
+        language: json.language,
+        duration: json.duration,
+      });
+      return {
+        ok: true,
+        backend: "local",
+        text: json.text || "",
+        language: json.language || null,
+        language_probability: json.language_probability ?? null,
+        duration: json.duration ?? null,
+        model: json.model,
+        compute_type: json.compute_type,
+      };
+    } catch (e) {
+      lastErr = e;
+      const isRetriable =
+        /fetch failed|ECONNRESET|socket hang up|terminated/i.test(e.message || "");
+      const dt = Date.now() - t0;
+      logWarn("whisper", `transcribeLocal attempt ${attempt} failed in ${dt}ms`, {
+        error: e.message,
+        retriable: isRetriable,
+        will_retry: isRetriable && attempt < maxAttempts,
+      });
+      if (!isRetriable || attempt >= maxAttempts) break;
+      // Brief backoff before retry — gives the whisper-server.py thread time
+      // to flush its pending response and release the model lock.
+      await _sleep(500);
+    }
+  }
+  logError("whisper", `transcribeLocal exhausted retries`, { error: lastErr?.message });
+  throw lastErr || new Error("local transcription failed");
 }
 
 // ---------------------------------------------------------------------------
@@ -280,19 +422,80 @@ export async function transcribe(filePath, overrides = {}) {
     return transcribeOpenAI(filePath, cfg.openaiKey);
   }
   if (provider === "local") {
+    // Explicit local-only: bubble up the real error, do not mention OpenAI.
     return transcribeLocal(filePath, localOpts);
   }
 
-  // auto: local first, fall back to openai
+  // auto: local first, fall back to openai only if a key is configured
   try {
     return await transcribeLocal(filePath, localOpts);
   } catch (localErr) {
-    if (!cfg.openaiKey) {
-      throw new Error(
-        `local transcription failed and no OpenAI fallback available: ${localErr.message}`
-      );
+    if (cfg.openaiKey) {
+      return transcribeOpenAI(filePath, cfg.openaiKey);
     }
-    return transcribeOpenAI(filePath, cfg.openaiKey);
+    // No OpenAI configured — surface the real local error verbatim.
+    throw new Error(`local transcription failed: ${localErr.message}`);
+  }
+}
+
+/**
+ * Transcribe raw audio bytes (e.g. from a mic chunk or Telegram voice blob).
+ * Saves to a temp file, transcribes, cleans up.
+ *
+ * @param {Buffer} buf        raw audio data
+ * @param {string} format     file extension hint: "webm" | "ogg" | "wav" | "mp3" (default "webm")
+ * @param {object} overrides  same as transcribe() overrides
+ */
+export async function transcribeBuffer(buf, format = "webm", overrides = {}) {
+  if (!buf || !buf.length) throw new Error("transcribeBuffer: empty buffer");
+  const ext = format.replace(/^\./, "") || "webm";
+  const tmpFile = path.join(
+    (await import("node:os")).default.tmpdir(),
+    `apx-audio-${Date.now()}-${Math.random().toString(36).slice(2)}.${ext}`
+  );
+  try {
+    fs.writeFileSync(tmpFile, buf);
+    return await transcribe(tmpFile, overrides);
+  } finally {
+    try { fs.unlinkSync(tmpFile); } catch {}
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Lifecycle (preload on daemon start, shutdown on daemon stop)
+// ---------------------------------------------------------------------------
+
+/**
+ * Eagerly start the whisper server so the first transcription is fast.
+ * Safe to call multiple times. Never throws — logs and continues on failure.
+ */
+export async function preloadWhisperServer(log = console.log) {
+  try {
+    const cfg = await getConfig();
+    if (cfg.provider === "openai") return; // local backend not used
+    log(`whisper: preloading model "${cfg.local.model}" on port ${WHISPER_PORT}…`);
+    await ensureWhisperServer(cfg.local);
+    log(`whisper: ready on port ${WHISPER_PORT} (model: ${_serverModel})`);
+  } catch (e) {
+    log(`whisper: preload failed — ${e.message} (will retry lazily on first request)`);
+  }
+}
+
+/**
+ * Stop the whisper server we own (no-op if we adopted an external one).
+ */
+export async function shutdownWhisperServer() {
+  if (_serverProcess) {
+    try { _serverProcess.kill(); } catch {}
+    _serverProcess = null;
+    _serverModel = null;
+  } else {
+    // Try graceful shutdown of an adopted server
+    try {
+      await fetch(`http://127.0.0.1:${WHISPER_PORT}/shutdown`, {
+        method: "POST", signal: AbortSignal.timeout(500),
+      });
+    } catch {}
   }
 }
 

package/src/daemon/wakeup.js

@@ -5,33 +5,28 @@ import { resolveProvider, getAdapter } from "./engines/index.js";
 
 const WAKEUP_COOLDOWN_MS = 30 * 60 * 1000; // 30 min
 
-// Detect preferred language from identity, then fall back to system LANG env.
-function detectLanguage(identity) {
-  if (identity.language) return identity.language;
+const ISO_TO_LANGUAGE = {
+  es: "Spanish", en: "English", fr: "French", pt: "Portuguese",
+  de: "German", it: "Italian", nl: "Dutch", ru: "Russian",
+  ja: "Japanese", zh: "Chinese", ko: "Korean", ar: "Arabic",
+};
+
+// Exported for unit testing.
+// Priority: config.user.language (ISO 639-1) → identity.language (legacy) → system LANG env.
+export function detectLanguage(identity, config) {
+  const cfgLang = config?.user?.language;
+  if (cfgLang) return ISO_TO_LANGUAGE[cfgLang.toLowerCase()] || cfgLang;
+  if (identity?.language) return identity.language;
   const lang = process.env.LANG || process.env.LC_MESSAGES || process.env.LC_ALL || "";
   const code = lang.split(/[_\.]/)[0].toLowerCase();
-  const map = {
-    es: "Spanish",
-    en: "English",
-    fr: "French",
-    pt: "Portuguese",
-    de: "German",
-    it: "Italian",
-    nl: "Dutch",
-    ru: "Russian",
-    ja: "Japanese",
-    zh: "Chinese",
-    ko: "Korean",
-    ar: "Arabic",
-  };
-  return map[code] || "English";
+  return ISO_TO_LANGUAGE[code] || "English";
 }
 
 async function generateMessage(identity, engineConfig) {
   try {
     const { provider, model } = resolveProvider("ollama:qwen2.5:14b");
     const engine = getAdapter(provider);
-    const language = detectLanguage(identity);
+    const language = detectLanguage(identity, engineConfig);
     const result = await engine.chat({
       system: `You are ${identity.agent_name}, an AI agent assistant. Your personality: ${identity.personality || "direct, curious, helpful"}. Your owner is ${identity.owner_name}. Context: ${identity.owner_context || "AI developer"}.`,
       messages: [

package/src/daemon/whisper-server.py

@@ -64,12 +64,18 @@ class _Handler(BaseHTTPRequestHandler):
         pass  # suppress access log; APX daemon handles its own logging
 
     def _send_json(self, code, body):
-        data = json.dumps(body).encode()
-        self.send_response(code)
-        self.send_header("Content-Type", "application/json")
-        self.send_header("Content-Length", str(len(data)))
-        self.end_headers()
-        self.wfile.write(data)
+        # Swallow BrokenPipe / ConnectionReset — these happen when the daemon
+        # times out and aborts the request before we finish responding, and
+        # they used to fill the daemon log with multi-page Python tracebacks.
+        try:
+            data = json.dumps(body).encode()
+            self.send_response(code)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+        except (BrokenPipeError, ConnectionResetError):
+            pass
 
     def _read_body(self):
         n = int(self.headers.get("Content-Length", 0))
@@ -92,6 +98,51 @@ class _Handler(BaseHTTPRequestHandler):
             self._send_json(404, {"ok": False, "error": "not found"})
 
     def do_POST(self):
+        # /transcribe_chunk reads raw bytes — must be handled BEFORE _read_body()
+        # which would consume rfile for JSON endpoints.
+        if self.path == "/transcribe_chunk":
+            _touch()
+            content_length = int(self.headers.get("Content-Length", 0))
+            if content_length <= 0:
+                self._send_json(400, {"ok": False, "error": "empty body"})
+                return
+            audio_bytes = self.rfile.read(content_length)
+            audio_format = (self.headers.get("X-Audio-Format") or "webm").strip().lstrip(".")
+            language_hdr = self.headers.get("X-Language") or None
+            language = language_hdr if language_hdr and language_hdr != "auto" else None
+            beam_size = int(self.headers.get("X-Beam-Size") or 3)
+
+            with _model_lock:
+                try:
+                    m = _load_model_if_needed(_Handler.model_name, _Handler.device, _Handler.compute_type)
+                except ImportError:
+                    self._send_json(500, {"ok": False, "error": "faster-whisper not installed"})
+                    return
+                except Exception as e:
+                    self._send_json(500, {"ok": False, "error": f"model load failed: {e}"})
+                    return
+
+                import tempfile
+                tmp = tempfile.NamedTemporaryFile(suffix=f".{audio_format}", delete=False)
+                try:
+                    tmp.write(audio_bytes)
+                    tmp.close()
+                    segments, info = m.transcribe(tmp.name, beam_size=beam_size, language=language)
+                    text = " ".join(seg.text.strip() for seg in segments).strip()
+                    self._send_json(200, {
+                        "ok": True, "text": text,
+                        "language": info.language,
+                        "language_probability": round(info.language_probability, 4),
+                        "duration": round(info.duration, 2) if hasattr(info, "duration") else None,
+                        "model": _model_name,
+                    })
+                except Exception as e:
+                    self._send_json(500, {"ok": False, "error": f"chunk transcription failed: {e}"})
+                finally:
+                    try: os.unlink(tmp.name)
+                    except Exception: pass
+            return
+
         req = self._read_body()
 
         if self.path == "/transcribe":

package/src/overlay/index.html

@@ -0,0 +1,44 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta http-equiv="Content-Security-Policy"
+        content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; media-src *;">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>APX</title>
+  <link rel="stylesheet" href="style.css">
+</head>
+<body>
+  <div id="app">
+    <!-- Header -->
+    <div id="header">
+      <span class="logo">APX</span>
+      <span id="status-text" class="status">Ready</span>
+      <div id="conn-badge" title="Daemon disconnected"></div>
+      <button class="btn-close" id="btn-close" title="Close (Esc)">✕</button>
+    </div>
+
+    <!-- Messages -->
+    <div id="messages">
+      <div id="empty-state">
+        <div class="icon">🎙</div>
+        <div>Press <kbd id="empty-shortcut-hint">⌘G</kbd> to start speaking</div>
+      </div>
+    </div>
+
+    <!-- Live transcription bar -->
+    <div id="live-bar">
+      <div class="rec-dot"></div>
+      <div id="live-text">Listening…</div>
+    </div>
+
+    <!-- Hint bar -->
+    <div id="hint-bar">
+      <span class="hint"><kbd id="shortcut-hint">⌘⇧\</kbd> record</span>
+      <span class="hint"><kbd>Esc</kbd> cancel / close</span>
+    </div>
+  </div>
+
+  <script src="renderer.js"></script>
+</body>
+</html>