@iletai/nzb 1.7.0 → 1.7.4

package/dist/copilot/orchestrator.js

@@ -1,7 +1,11 @@
 import { approveAll } from "@github/copilot-sdk";
 import { config, DEFAULT_MODEL } from "../config.js";
 import { SESSIONS_DIR } from "../paths.js";
-import { completeTeam, deleteState, getMemorySummary, getRecentConversation, getState, logConversation, setState, updateTeamMemberResult, } from "../store/db.js";
+import { deleteState, getState, setState } from "../store/db.js";
+import { getRecentConversation, logConversation } from "../store/conversation.js";
+import { getMemorySummary } from "../store/memory.js";
+import { completeTeam, updateTeamMemberResult } from "../store/team-store.js";
+import { formatAge, withTimeout } from "../utils.js";
 import { resetClient } from "./client.js";
 import { loadMcpConfig } from "./mcp-config.js";
 import { getSkillDirectories } from "./skills.js";
@@ -27,10 +31,13 @@ let copilotClient;
 const workers = new Map();
 const teams = new Map();
 let healthCheckTimer;
+let workerReaperTimer;
 // Persistent orchestrator session
 let orchestratorSession;
 // Coalesces concurrent ensureOrchestratorSession calls
 let sessionCreatePromise;
+// Tracks when the orchestrator session was created for TTL enforcement
+let sessionCreatedAt;
 // Tracks in-flight context recovery injection so we don't race with real messages
 let recoveryInjectionPromise;
 const messageQueue = [];
@@ -60,6 +67,7 @@ function getSessionConfig() {
                     workerNotifyFn(event, channel);
                 }
             },
+            getCurrentSourceChannel,
         });
         cachedToolsClientRef = copilotClient;
     }
@@ -138,27 +146,37 @@ async function ensureClient() {
     return resetPromise;
 }
 /** Start periodic health check that proactively reconnects the client. */
+let healthCheckRunning = false;
 function startHealthCheck() {
     if (healthCheckTimer)
         return;
     healthCheckTimer = setInterval(async () => {
         if (!copilotClient)
             return;
+        if (healthCheckRunning)
+            return;
+        if (processing)
+            return; // Don't interfere while processing messages
+        healthCheckRunning = true;
         try {
             const state = copilotClient.getState();
             if (state !== "connected") {
                 console.log(`[nzb] Health check: client state is '${state}', resetting…`);
                 const previousClient = copilotClient;
-                await ensureClient();
+                await withTimeout(ensureClient(), 15_000, "health check");
                 // Only invalidate session if the underlying client actually changed
                 if (copilotClient !== previousClient) {
                     orchestratorSession = undefined;
+                    sessionCreatedAt = undefined;
                 }
             }
         }
         catch (err) {
             console.error(`[nzb] Health check error:`, err instanceof Error ? err.message : err);
         }
+        finally {
+            healthCheckRunning = false;
+        }
     }, HEALTH_CHECK_INTERVAL_MS);
 }
 /** Stop the periodic health check timer. Call during shutdown. */
@@ -168,17 +186,64 @@ export function stopHealthCheck() {
         healthCheckTimer = undefined;
     }
 }
+/** Periodically kills workers that have exceeded 2× their configured timeout. */
+function startWorkerReaper() {
+    if (workerReaperTimer)
+        return;
+    workerReaperTimer = setInterval(async () => {
+        const maxAge = config.workerTimeoutMs * 2;
+        const now = Date.now();
+        for (const [name, worker] of workers) {
+            if (worker.startedAt && now - worker.startedAt > maxAge) {
+                console.log(`[nzb] Reaping stuck worker '${name}' (age: ${formatAge(worker.startedAt)})`);
+                try {
+                    await withTimeout(worker.session.disconnect(), 5_000, `reaper: worker '${name}'`);
+                }
+                catch (err) {
+                    console.error(`[nzb] Reaper: worker '${name}' disconnect failed:`, err instanceof Error ? err.message : err);
+                }
+                workers.delete(name);
+                feedBackgroundResult(name, `⚠ Worker '${name}' was automatically killed after exceeding timeout.`);
+            }
+        }
+    }, 5 * 60 * 1000);
+    workerReaperTimer.unref();
+}
+const SESSION_MAX_AGE_MS = 4 * 60 * 60 * 1000; // 4 hours
 /** Create or resume the persistent orchestrator session. */
 async function ensureOrchestratorSession() {
-    if (orchestratorSession)
-        return orchestratorSession;
+    if (orchestratorSession) {
+        // Validate session is still usable — check client connectivity
+        try {
+            const clientState = copilotClient?.getState?.();
+            if (clientState && clientState !== "connected") {
+                console.log(`[nzb] Session stale (client state: ${clientState}), recreating…`);
+                orchestratorSession = undefined;
+                sessionCreatedAt = undefined;
+            }
+        }
+        catch {
+            console.log("[nzb] Session validation failed, recreating…");
+            orchestratorSession = undefined;
+            sessionCreatedAt = undefined;
+        }
+        // Enforce session TTL
+        if (sessionCreatedAt && Date.now() - sessionCreatedAt > SESSION_MAX_AGE_MS) {
+            console.log("[nzb] Session TTL expired, recreating…");
+            orchestratorSession = undefined;
+            sessionCreatedAt = undefined;
+        }
+        if (orchestratorSession)
+            return orchestratorSession;
+    }
     // Coalesce concurrent callers — wait for an in-flight creation
     if (sessionCreatePromise)
         return sessionCreatePromise;
-    sessionCreatePromise = createOrResumeSession();
+    sessionCreatePromise = withTimeout(createOrResumeSession(), 30_000, "session create/resume");
     try {
         const session = await sessionCreatePromise;
         orchestratorSession = session;
+        sessionCreatedAt = Date.now();
         return session;
     }
     finally {
@@ -250,7 +315,7 @@ async function createOrResumeSession() {
     // Recover conversation context if available (session was lost, not first run)
     // Runs concurrently but is awaited before any real message is sent on the session
     const recentHistory = getRecentConversation(10);
-    if (recentHistory) {
+    if (!recoveryInjectionPromise && recentHistory) {
         console.log(`[nzb] Injecting recent conversation context into new session`);
         recoveryInjectionPromise = session
             .sendAndWait({
@@ -290,6 +355,7 @@ export async function initOrchestrator(client) {
     console.log(`[nzb] Skill directories: ${skillDirectories.join(", ") || "(none)"}`);
     console.log(`[nzb] Persistent session mode — conversation history maintained by SDK`);
     startHealthCheck();
+    startWorkerReaper();
     // Eagerly create/resume the orchestrator session
     try {
         await ensureOrchestratorSession();
@@ -303,53 +369,85 @@ async function executeOnSession(prompt, callback, onToolEvent, onUsage, attachme
     const session = await ensureOrchestratorSession();
     // Wait for any in-flight context recovery injection to finish before sending
     if (recoveryInjectionPromise) {
-        console.log(`[nzb] Waiting for context recovery injection to complete before sending…`);
-        await recoveryInjectionPromise;
+        console.log("[nzb] Waiting for context recovery…");
+        await withTimeout(recoveryInjectionPromise, 25_000, "recovery injection wait").catch(() => {
+            console.log("[nzb] Recovery injection wait timed out, proceeding anyway");
+        });
     }
     currentCallback = callback;
     let accumulated = "";
     let toolCallExecuted = false;
     const unsubToolStart = session.on("tool.execution_start", (event) => {
-        const toolName = event?.data?.toolName || event?.data?.name || "tool";
-        const args = event?.data?.arguments;
-        const detail = args?.description ||
-            args?.command?.slice(0, 80) ||
-            args?.intent ||
-            args?.pattern ||
-            args?.prompt?.slice(0, 80) ||
-            undefined;
-        onToolEvent?.({ type: "tool_start", toolName, detail });
+        try {
+            const toolName = event?.data?.toolName || event?.data?.name || "tool";
+            const args = event?.data?.arguments;
+            const detail = args?.description ||
+                args?.command?.slice(0, 80) ||
+                args?.intent ||
+                args?.pattern ||
+                args?.prompt?.slice(0, 80) ||
+                undefined;
+            onToolEvent?.({ type: "tool_start", toolName, detail });
+        }
+        catch (err) {
+            console.error("[nzb] Error in tool.execution_start listener:", err instanceof Error ? err.message : err);
+        }
     });
     const unsubToolDone = session.on("tool.execution_complete", (event) => {
-        toolCallExecuted = true;
-        const toolName = event?.data?.toolName || event?.data?.name || "tool";
-        onToolEvent?.({ type: "tool_complete", toolName });
+        try {
+            toolCallExecuted = true;
+            const toolName = event?.data?.toolName || event?.data?.name || "tool";
+            onToolEvent?.({ type: "tool_complete", toolName });
+        }
+        catch (err) {
+            console.error("[nzb] Error in tool.execution_complete listener:", err instanceof Error ? err.message : err);
+        }
     });
     const unsubDelta = session.on("assistant.message_delta", (event) => {
-        // After a tool call completes, ensure a line break separates the text blocks
-        // so they don't visually run together in the TUI.
-        if (toolCallExecuted && accumulated.length > 0 && !accumulated.endsWith("\n")) {
-            accumulated += "\n";
-        }
-        toolCallExecuted = false;
-        accumulated += event.data.deltaContent;
-        callback(accumulated, false);
+        try {
+            // After a tool call completes, ensure a line break separates the text blocks
+            // so they don't visually run together in the TUI.
+            if (toolCallExecuted && accumulated.length > 0 && !accumulated.endsWith("\n")) {
+                accumulated += "\n";
+            }
+            toolCallExecuted = false;
+            accumulated += event.data.deltaContent;
+            callback(accumulated, false);
+        }
+        catch (err) {
+            console.error("[nzb] Error in message_delta listener:", err instanceof Error ? err.message : err);
+        }
     });
     const unsubError = session.on("session.error", (event) => {
-        const errMsg = event?.data?.message || event?.data?.error || "Unknown session error";
-        console.error(`[nzb] Session error event: ${errMsg}`);
+        try {
+            const errMsg = event?.data?.message || event?.data?.error || "Unknown session error";
+            console.error(`[nzb] Session error event: ${errMsg}`);
+        }
+        catch (err) {
+            console.error("[nzb] Error in session.error listener:", err instanceof Error ? err.message : err);
+        }
     });
     const unsubPartialResult = session.on("tool.execution_partial_result", (event) => {
-        const toolName = event?.data?.toolName || event?.data?.name || "tool";
-        const partialOutput = event?.data?.partialOutput || "";
-        onToolEvent?.({ type: "tool_partial_result", toolName, detail: partialOutput });
+        try {
+            const toolName = event?.data?.toolName || event?.data?.name || "tool";
+            const partialOutput = event?.data?.partialOutput || "";
+            onToolEvent?.({ type: "tool_partial_result", toolName, detail: partialOutput });
+        }
+        catch (err) {
+            console.error("[nzb] Error in tool.execution_partial_result listener:", err instanceof Error ? err.message : err);
+        }
     });
     const unsubUsage = session.on("assistant.usage", (event) => {
-        const inputTokens = event?.data?.inputTokens || 0;
-        const outputTokens = event?.data?.outputTokens || 0;
-        const model = event?.data?.model || undefined;
-        const duration = event?.data?.duration || undefined;
-        onUsage?.({ inputTokens, outputTokens, model, duration });
+        try {
+            const inputTokens = event?.data?.inputTokens || 0;
+            const outputTokens = event?.data?.outputTokens || 0;
+            const model = event?.data?.model || undefined;
+            const duration = event?.data?.duration || undefined;
+            onUsage?.({ inputTokens, outputTokens, model, duration });
+        }
+        catch (err) {
+            console.error("[nzb] Error in assistant.usage listener:", err instanceof Error ? err.message : err);
+        }
     });
     try {
         const sendPayload = { prompt };
@@ -364,6 +462,16 @@ async function executeOnSession(prompt, callback, onToolEvent, onUsage, attachme
     }
     catch (err) {
         const msg = err instanceof Error ? err.message : String(err);
+        // Vision not supported — the session is now tainted with image data in its history,
+        // so ALL subsequent messages would fail. Force-recreate the session to recover.
+        // The retry (with stripped attachments) is handled by sendToOrchestrator's retry loop.
+        if (/not supported for vision/i.test(msg)) {
+            console.log(`[nzb] Model '${config.copilotModel}' does not support vision — destroying tainted session`);
+            orchestratorSession = undefined;
+            sessionCreatedAt = undefined;
+            deleteState(ORCHESTRATOR_SESSION_KEY);
+            throw err;
+        }
         // On timeout, deliver whatever was accumulated instead of retrying from scratch
         if (/timeout/i.test(msg) && accumulated.length > 0) {
             console.log(`[nzb] Timeout — delivering ${accumulated.length} chars of partial content`);
@@ -373,6 +481,7 @@ async function executeOnSession(prompt, callback, onToolEvent, onUsage, attachme
         if (/closed|destroy|disposed|invalid|expired|not found/i.test(msg)) {
             console.log(`[nzb] Session appears dead, will recreate: ${msg}`);
             orchestratorSession = undefined;
+            sessionCreatedAt = undefined;
             deleteState(ORCHESTRATOR_SESSION_KEY);
         }
         throw err;
@@ -396,19 +505,30 @@ async function processQueue() {
         return;
     }
     processing = true;
-    while (messageQueue.length > 0) {
-        const item = messageQueue.shift();
-        currentSourceChannel = item.sourceChannel;
-        try {
-            const result = await executeOnSession(item.prompt, item.callback, item.onToolEvent, item.onUsage, item.attachments);
-            item.resolve(result);
-        }
-        catch (err) {
-            item.reject(err);
+    try {
+        while (messageQueue.length > 0) {
+            const item = messageQueue.shift();
+            currentSourceChannel = item.sourceChannel;
+            try {
+                const result = await executeOnSession(item.prompt, item.callback, item.onToolEvent, item.onUsage, item.attachments);
+                item.resolve(result);
+            }
+            catch (err) {
+                item.reject(err);
+            }
+            currentSourceChannel = undefined;
         }
-        currentSourceChannel = undefined;
     }
-    processing = false;
+    finally {
+        processing = false;
+    }
+    // Re-check for messages that arrived during the last executeOnSession call
+    if (messageQueue.length > 0) {
+        processQueue().catch((err) => {
+            console.error("[nzb] processQueue re-check failed:", err instanceof Error ? err.message : err);
+            processing = false;
+        });
+    }
 }
 function isRecoverableError(err) {
     const msg = err instanceof Error ? err.message : String(err);
@@ -445,94 +565,137 @@ export async function sendToOrchestrator(prompt, source, callback, onToolEvent,
     const sourceChannel = source.type === "telegram" ? "telegram" : source.type === "tui" ? "tui" : undefined;
     // Enqueue with priority — user messages go before background messages
     void (async () => {
-        for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
-            try {
-                const finalContent = await new Promise((resolve, reject) => {
-                    const item = {
-                        prompt: taggedPrompt,
-                        attachments,
-                        callback,
-                        onToolEvent,
-                        onUsage,
-                        sourceChannel,
-                        resolve,
-                        reject,
-                    };
-                    if (source.type === "background") {
-                        // Background results go to the back of the queue
-                        messageQueue.push(item);
-                    }
-                    else {
-                        // User messages inserted before any background messages (priority)
-                        const bgIndex = messageQueue.findIndex(isBackgroundMessage);
-                        if (bgIndex >= 0) {
-                            messageQueue.splice(bgIndex, 0, item);
+        // Safety timeout for entire message processing chain.
+        // Uses a flag to prevent double-callback if timeout fires while processing completes.
+        const GLOBAL_MSG_TIMEOUT_MS = 300_000; // 5 minutes
+        let globalTimedOut = false;
+        const globalTimer = setTimeout(() => {
+            globalTimedOut = true;
+            console.error("[nzb] Global message processing timeout (5 min). Force-failing.");
+            Promise.resolve(callback("Error: Message processing timed out after 5 minutes. Please try again.", true)).catch(() => { });
+        }, GLOBAL_MSG_TIMEOUT_MS);
+        try {
+            for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+                try {
+                    const finalContent = await new Promise((resolve, reject) => {
+                        const item = {
+                            prompt: taggedPrompt,
+                            attachments,
+                            callback,
+                            onToolEvent,
+                            onUsage,
+                            sourceChannel,
+                            resolve,
+                            reject,
+                        };
+                        if (source.type === "background") {
+                            // Background results go to the back of the queue
+                            messageQueue.push(item);
                         }
                         else {
-                            messageQueue.push(item);
+                            // User messages inserted before any background messages (priority)
+                            const bgIndex = messageQueue.findIndex(isBackgroundMessage);
+                            if (bgIndex >= 0) {
+                                messageQueue.splice(bgIndex, 0, item);
+                            }
+                            else {
+                                messageQueue.push(item);
+                            }
                         }
+                        processQueue();
+                    });
+                    // Deliver response to user FIRST, then log best-effort
+                    try {
+                        logMessage("out", sourceLabel, finalContent);
                     }
-                    processQueue();
-                });
-                // Deliver response to user FIRST, then log best-effort
-                try {
-                    logMessage("out", sourceLabel, finalContent);
-                }
-                catch {
-                    /* best-effort */
-                }
-                // Log both sides of the conversation before delivery so we have the row ID
-                let assistantLogId;
-                try {
-                    const telegramMsgId = source.type === "telegram" ? source.messageId : undefined;
-                    logConversation(logRole, prompt, sourceLabel, telegramMsgId);
-                }
-                catch {
-                    /* best-effort */
-                }
-                try {
-                    assistantLogId = logConversation("assistant", finalContent, sourceLabel);
-                }
-                catch {
-                    /* best-effort */
-                }
-                await callback(finalContent, true, { assistantLogId });
-                // Auto-continue: if the response was cut short by timeout, automatically
-                // send a follow-up "Continue" message so the user doesn't have to
-                if (finalContent.includes("⏱ Response was cut short (timeout)") && _autoContinueCount < MAX_AUTO_CONTINUE) {
-                    console.log(`[nzb] Auto-continuing after timeout (${_autoContinueCount + 1}/${MAX_AUTO_CONTINUE})…`);
-                    await sleep(1000);
-                    void sendToOrchestrator("Continue from where you left off. Do not repeat what was already said.", source, callback, onToolEvent, onUsage, _autoContinueCount + 1);
-                }
-                return;
-            }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                // Don't retry cancelled messages
-                if (/cancelled|abort/i.test(msg)) {
-                    return;
-                }
-                if (isRecoverableError(err) && attempt < MAX_RETRIES) {
-                    const delay = RECONNECT_DELAYS_MS[Math.min(attempt, RECONNECT_DELAYS_MS.length - 1)];
-                    console.error(`[nzb] Recoverable error: ${msg}. Retry ${attempt + 1}/${MAX_RETRIES} after ${delay}ms…`);
-                    await sleep(delay);
-                    // Reset client before retry in case the connection is stale
+                    catch {
+                        /* best-effort */
+                    }
+                    // Log both sides of the conversation before delivery so we have the row ID
+                    let assistantLogId;
                     try {
-                        await ensureClient();
+                        const telegramMsgId = source.type === "telegram" ? source.messageId : undefined;
+                        logConversation(logRole, prompt, sourceLabel, telegramMsgId);
                     }
                     catch {
-                        /* will fail again on next attempt */
+                        /* best-effort */
+                    }
+                    try {
+                        assistantLogId = logConversation("assistant", finalContent, sourceLabel);
+                    }
+                    catch {
+                        /* best-effort */
+                    }
+                    try {
+                        if (!globalTimedOut) {
+                            await callback(finalContent, true, { assistantLogId });
+                        }
+                    }
+                    catch (callbackErr) {
+                        console.error("[nzb] Callback error after successful response:", callbackErr instanceof Error ? callbackErr.message : callbackErr);
                     }
-                    continue;
+                    // Auto-continue: if the response was cut short by timeout, automatically
+                    // send a follow-up "Continue" message so the user doesn't have to
+                    if (finalContent.includes("⏱ Response was cut short (timeout)") && _autoContinueCount < MAX_AUTO_CONTINUE) {
+                        console.log(`[nzb] Auto-continuing after timeout (${_autoContinueCount + 1}/${MAX_AUTO_CONTINUE})…`);
+                        await sleep(1000);
+                        void sendToOrchestrator("Continue from where you left off. Do not repeat what was already said.", source, callback, onToolEvent, onUsage, _autoContinueCount + 1);
+                    }
+                    return;
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    // Don't retry cancelled messages
+                    if (/cancelled|abort/i.test(msg)) {
+                        if (!globalTimedOut) {
+                            try {
+                                await callback("Request was cancelled.", true);
+                            }
+                            catch { /* best-effort */ }
+                        }
+                        return;
+                    }
+                    // Vision not supported — strip attachments and retry with text-only prompt.
+                    // executeOnSession already destroyed the tainted session.
+                    if (/not supported for vision/i.test(msg)) {
+                        console.log(`[nzb] Vision not supported — retrying without attachments`);
+                        attachments = undefined;
+                        taggedPrompt =
+                            `[System: The current model '${config.copilotModel}' does not support image/vision analysis. ` +
+                                `The image path is already included in the user's message below. ` +
+                                `Please inform the user that the current model doesn't support direct image analysis, ` +
+                                `and suggest switching to a vision-capable model (e.g. gpt-4o, claude-sonnet-4, gemini-2.0-flash) ` +
+                                `using the /model command.]\n\n${taggedPrompt}`;
+                        continue;
+                    }
+                    if (isRecoverableError(err) && attempt < MAX_RETRIES) {
+                        const delay = RECONNECT_DELAYS_MS[Math.min(attempt, RECONNECT_DELAYS_MS.length - 1)];
+                        console.error(`[nzb] Recoverable error: ${msg}. Retry ${attempt + 1}/${MAX_RETRIES} after ${delay}ms…`);
+                        await sleep(delay);
+                        // Reset client before retry in case the connection is stale
+                        try {
+                            await ensureClient();
+                        }
+                        catch {
+                            /* will fail again on next attempt */
+                        }
+                        continue;
+                    }
+                    console.error(`[nzb] Error processing message: ${msg}`);
+                    if (!globalTimedOut) {
+                        await callback(`Error: ${msg}`, true);
+                    }
+                    return;
                 }
-                console.error(`[nzb] Error processing message: ${msg}`);
-                await callback(`Error: ${msg}`, true);
-                return;
             }
         }
-    })();
+        finally {
+            clearTimeout(globalTimer);
+        }
+    })().catch((err) => {
+        console.error(`[nzb] Unhandled error in sendToOrchestrator: ${err instanceof Error ? err.message : String(err)}`);
+    });
 }
-/** Cancel the in-flight message and drain the queue. */
 export async function cancelCurrentMessage() {
     // Drain any queued messages
     const drained = messageQueue.length;
@@ -583,6 +746,7 @@ export async function resetSession() {
         }
         catch { }
         orchestratorSession = undefined;
+        sessionCreatedAt = undefined;
     }
     // Clear persisted session ID so a fresh session is created
     deleteState(ORCHESTRATOR_SESSION_KEY);

package/dist/copilot/skills.js

@@ -41,7 +41,8 @@ export function listSkills() {
         try {
             entries = readdirSync(dir);
         }
-        catch {
+        catch (err) {
+            console.error("[nzb] Failed to read skill directory", dir + ":", err instanceof Error ? err.message : err);
             continue;
         }
         for (const entry of entries) {
@@ -60,7 +61,8 @@ export function listSkills() {
                     source,
                 });
             }
-            catch {
+            catch (err) {
+                console.error("[nzb] Failed to parse SKILL.md for", entry + ":", err instanceof Error ? err.message : err);
                 skills.push({
                     slug: entry,
                     name: entry,