@tiens.nguyen/gonext-local-worker 1.0.153 → 1.0.155

package/gonext-local-worker.mjs

@@ -143,6 +143,8 @@ const OCR_CORRECT_TIMEOUT_MS = 120_000;
 // cold load of a big/multimodal model into VRAM can take ~20s+; the request isn't
 // awaited, so a long ceiling just avoids aborting an in-progress load.
 const AGENT_WARMUP_TIMEOUT_MS = 180_000;
+// How often the background keep-warm re-asserts the coding model's keep_alive:-1.
+const AGENT_WARMUP_INTERVAL_MS = 240_000; // 4 min — under Ollama's 5-min default unload
 // OCR translation/correction is hardcoded to a managed Ollama service + model.
 // (The user-configurable "OCR translation model" chooser was removed.)
 const OCR_TRANSLATE_OLLAMA_URL = "https://ollama1.gomarsic.cc";
@@ -1436,33 +1438,37 @@ async function runTranscribeJob(job) {
   }
 }
 
-/**
- * Fire-and-forget pre-load of the agent's Ollama coding model. Ollama evicts idle
- * models (default OLLAMA_KEEP_ALIVE=5m), so the first agent step otherwise blocks
- * ~20s+ while llama-server spins up and pages the weights into VRAM. The agent
- * talks to Ollama over the OpenAI-compat /v1 endpoint, which IGNORES keep_alive —
- * so we hit the NATIVE /api/generate here (no prompt = load-only, returns as soon
- * as the model is resident) with keep_alive:-1 to trigger the load AND pin it in
- * memory. Kicked off the moment the job is claimed so the load overlaps python
- * startup + prompt assembly. Best-effort: a non-Ollama coding server just 404s and
- * the failure is swallowed — the job proceeds regardless.
- */
-function warmUpOllamaCodingModel(codingBaseURL, codingModelId) {
-  if (!codingBaseURL || !codingModelId) return;
-  // codingBaseURL is normalized to …/v1 for OpenAI-compat; strip it to reach the
-  // Ollama root where /api/generate lives.
-  const root = String(codingBaseURL).replace(/\/+$/, "").replace(/\/v1$/i, "");
+// —— Ollama coding-model keep-warm ——————————————————————————————————————————————
+// Ollama evicts idle models (default OLLAMA_KEEP_ALIVE=5m), so a cold agent step
+// blocks ~20s+ while llama-server pages the weights into VRAM. Instead of warming on
+// EVERY question (noisy, redundant), we remember the coding model from agent jobs and
+// re-assert a keep_alive:-1 load on a BACKGROUND INTERVAL so it stays resident between
+// questions. The agent talks to Ollama over /v1 (which ignores keep_alive), so we hit
+// the NATIVE /api/generate (no prompt = load-only, returns once the model is resident).
+// Best-effort throughout: a non-Ollama coding server just 404s and it's swallowed.
+let _warmCoding = null; // { root, model }
+let _warmTimer = null;
+let _warmLoggedOk = false;
+
+function _warmupPing(quiet) {
+  if (!_warmCoding) return;
+  const { root, model } = _warmCoding;
   const url = `${root}/api/generate`;
   fetch(url, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({ model: codingModelId, keep_alive: -1 }),
+    body: JSON.stringify({ model, keep_alive: -1 }),
     signal: AbortSignal.timeout(AGENT_WARMUP_TIMEOUT_MS),
   })
     .then((res) => {
-      console.log(
-        `[gonext-worker] agent warm-up ${res.ok ? "ok (model loading/resident)" : `HTTP ${res.status}`} → ${url} model=${codingModelId}`
-      );
+      if (!res.ok) {
+        console.log(`[gonext-worker] agent warm-up HTTP ${res.status} → ${url} model=${model}`);
+      } else if (!quiet || !_warmLoggedOk) {
+        // Quiet interval pings only log the first success per model; a fresh/changed
+        // model always logs so you can see it warm.
+        console.log(`[gonext-worker] agent warm-up ok (model loading/resident) → ${url} model=${model}`);
+        _warmLoggedOk = true;
+      }
     })
     .catch((err) => {
       const reason =
@@ -1471,12 +1477,37 @@ function warmUpOllamaCodingModel(codingBaseURL, codingModelId) {
     });
 }
 
+/**
+ * Remember the agent's Ollama coding model (from a claimed agent_chat job) and keep it
+ * resident via a background interval. Warms IMMEDIATELY only when the model is first
+ * seen or CHANGES — not on every question — so repeat questions don't re-warm/re-log.
+ */
+function trackCodingModelForWarmup(codingBaseURL, codingModelId) {
+  if (!codingBaseURL || !codingModelId) return;
+  // codingBaseURL is normalized to …/v1 for OpenAI-compat; strip it to reach the
+  // Ollama root where /api/generate lives.
+  const root = String(codingBaseURL).replace(/\/+$/, "").replace(/\/v1$/i, "");
+  const changed = !_warmCoding || _warmCoding.root !== root || _warmCoding.model !== codingModelId;
+  _warmCoding = { root, model: codingModelId };
+  if (changed) {
+    _warmLoggedOk = false;
+    _warmupPing(false); // warm the newly-seen/changed model now so the next question isn't cold
+  }
+  if (!_warmTimer) {
+    // 4 min — under Ollama's 5-min default unload, so the model stays resident even
+    // if a proxy strips keep_alive. unref() so this timer never blocks worker exit.
+    _warmTimer = setInterval(() => _warmupPing(true), AGENT_WARMUP_INTERVAL_MS);
+    if (typeof _warmTimer.unref === "function") _warmTimer.unref();
+  }
+}
+
 async function runAgentChatJob(job) {
   const { jobId, payload } = job;
   const start = Date.now();
-  // Kick the Ollama coding model's cold load off immediately (non-blocking) so it
-  // overlaps prompt assembly instead of stalling the first step. See helper above.
-  warmUpOllamaCodingModel(payload?.codingBaseURL ?? "", payload?.codingModelId ?? "");
+  // Remember this job's Ollama coding model and keep it resident via a background
+  // interval. Only warms now if the model is new/changed — repeat questions rely on
+  // the interval, so we don't re-warm (or re-log) on every question.
+  trackCodingModelForWarmup(payload?.codingBaseURL ?? "", payload?.codingModelId ?? "");
   const runRes = await workerFetch(`/api/worker/jobs/${jobId}`, {
     method: "PATCH",
     body: JSON.stringify({ jobStatus: "running" }),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiens.nguyen/gonext-local-worker",
-  "version": "1.0.153",
+  "version": "1.0.155",
   "description": "Polls GoNext cloud API for async local LLM jobs and runs them against Ollama/OpenAI-compatible servers on this Mac",
   "type": "module",
   "license": "MIT",