@link-assistant/hive-mind 2.0.3 → 2.0.4

package/src/telegram-bot.mjs

@@ -328,7 +328,10 @@ const { isOldMessage: _isOldMessage, isGroupChat: _isGroupChat, isChatAuthorized
 const { installTelegramFormattingFallback, isTelegramFormattingError, isTelegramMessageTooLongError, safeEditMessageText, safeReply, TELEGRAM_TEXT_LIMIT } = await import('./telegram-safe-reply.lib.mjs');
 const { registerTerminalWatchCommand, startAutoTerminalWatchForSession } = await import('./telegram-terminal-watch-command.lib.mjs');
 const { launchBotWithRetry } = await import('./telegram-bot-launcher.lib.mjs');
-const { trackSession, startSessionMonitoring, hasActiveSessionForUrlAsync, findStoppableSessionByUrl } = await import('./session-monitor.lib.mjs');
+const { trackSession, startSessionMonitoring, hasActiveSessionForUrlAsync, findStoppableSessionByUrl, setSessionStore, setSessionLogger, resumeTrackedSessions, getActiveSessionCount } = await import('./session-monitor.lib.mjs');
+const { createBotLogger } = await import('./bot-logger.lib.mjs');
+const { createSessionStore } = await import('./session-store.lib.mjs');
+const { createHeartbeat, resumeSessionsOnLaunch, createShutdownHandler } = await import('./bot-lifecycle.lib.mjs');
 const { formatExecutingWorkSessionMessage, formatStartingWorkSessionMessage } = await import('./work-session-formatting.lib.mjs');
 const { buildTelegramHelpMessage, buildTelegramInfoBlock, buildSolveQueuedMessage } = await import('./telegram-ui-messages.lib.mjs');
 
@@ -354,6 +357,18 @@ installTelegramFormattingFallback(bot.telegram, { verbose: VERBOSE });
 
 // Track bot startup time (Unix seconds to match Telegram's message.date format)
 const BOT_START_TIME = Math.floor(Date.now() / 1000);
+
+// Issue #1927: durable, timestamped bot log + durable session store. The logger
+// preserves the previous run's log under a timestamped backup (never overwriting
+// it) so the moment of a total failure stays discoverable, and every line is
+// timestamped. The session store mirrors the in-memory session registry to disk
+// so a restart can resume monitoring detached sessions that were still running.
+const botLogger = createBotLogger({ verbose: VERBOSE });
+const sessionStore = createSessionStore({ verbose: VERBOSE, logger: botLogger });
+setSessionLogger(botLogger);
+setSessionStore(sessionStore);
+botLogger.event('bot_starting', { pid: process.pid, ppid: process.ppid, botStartTime: BOT_START_TIME, startTimeIso: new Date(BOT_START_TIME * 1000).toISOString(), logFile: botLogger.filePath, sessionSnapshot: sessionStore.snapshotPath });
+
 // Wrapper functions binding filter logic to bot state (actual logic in telegram-message-filters.lib.mjs, issue #1207)
 function isChatAuthorized(chatId) {
   return _isChatAuthorized(chatId, allowedChats);
@@ -1341,13 +1356,28 @@ function startSessionMonitoringOnce() {
   sessionMonitoringTimer = startSessionMonitoring(bot, VERBOSE);
 }
 
+// Issue #1927 (requirements #3/#4): a periodic timestamped heartbeat so the "last
+// time the bot was alive" is always discoverable from the log. The heartbeat
+// logic lives in bot-lifecycle.lib.mjs so it can be unit tested.
+const heartbeat = createHeartbeat({ logger: botLogger, getActiveSessionCount });
+
 async function onBotLaunched() {
   if (isShuttingDown || launchAnnouncementShown) return;
   launchAnnouncementShown = true;
 
   console.log('✅ SwarmMindBot is now running!');
   console.log('Press Ctrl+C to stop');
+  botLogger.event('bot_launched', { pid: process.pid, botStartTime: BOT_START_TIME });
+
+  // Issue #1927 (requirements #2/#4): after a restart, reload sessions that were
+  // still being tracked when the previous process died and re-register them so
+  // the monitor resumes watching — and finally reports any that were killed while
+  // the bot was down. Done before starting the monitor so the first tick already
+  // sees the resumed sessions.
+  await resumeSessionsOnLaunch({ resumeTrackedSessions, botStartTime: BOT_START_TIME, verbose: VERBOSE, logger: botLogger });
+
   startSessionMonitoringOnce();
+  heartbeat.start();
 
   if (VERBOSE) {
     console.log('[VERBOSE] Bot launched successfully');
@@ -1429,22 +1459,33 @@ const stopSolveQueue = () => {
   }
 };
 
+// Issue #1927: record the shutdown (with a timestamp) so the log shows the bot
+// stopped cleanly — the ABSENCE of this line before the next startup is how a
+// later analysis tells an orderly stop apart from a hard kill. The handler lives
+// in bot-lifecycle.lib.mjs; the timer/flag mutations stay here via the closures
+// (issue #1240: still abort the retry loop on the way out).
+const handleShutdownSignal = createShutdownHandler({
+  logger: botLogger,
+  getActiveSessionCount,
+  verbose: VERBOSE,
+  bot,
+  onShutdown: () => {
+    isShuttingDown = true;
+  },
+  cleanup: () => {
+    launchAbortController.abort();
+    if (sessionMonitoringTimer) clearInterval(sessionMonitoringTimer);
+    heartbeat.stop();
+    stopSolveQueue();
+  },
+});
+
 process.once('SIGINT', () => {
-  isShuttingDown = true;
   console.log('\n🛑 Received SIGINT (Ctrl+C), stopping bot...');
-  if (VERBOSE) console.log(`[VERBOSE] Signal: SIGINT, PID: ${process.pid}, PPID: ${process.ppid}`);
-  launchAbortController.abort(); // Cancel retry loop if still retrying (issue #1240)
-  if (sessionMonitoringTimer) clearInterval(sessionMonitoringTimer);
-  stopSolveQueue();
-  bot.stop('SIGINT');
+  handleShutdownSignal('SIGINT');
 });
 
 process.once('SIGTERM', () => {
-  isShuttingDown = true;
   console.log('\n🛑 Received SIGTERM, stopping bot... (Check system logs: journalctl -u <service> or dmesg)');
-  if (VERBOSE) console.log(`[VERBOSE] Signal: SIGTERM, PID: ${process.pid}, PPID: ${process.ppid}`);
-  launchAbortController.abort(); // Cancel retry loop if still retrying (issue #1240)
-  if (sessionMonitoringTimer) clearInterval(sessionMonitoringTimer);
-  stopSolveQueue();
-  bot.stop('SIGTERM');
+  handleShutdownSignal('SIGTERM');
 });

package/src/telegram-command-execution.lib.mjs

@@ -109,7 +109,9 @@ export function buildExecuteAndUpdateMessage(deps) {
       }
     };
     const requesterUserId = ctx.from?.id ?? null; // Issue #1688: suppress duplicate /subscribe DM
-    const baseSessionInfo = { chatId: ctx.chat.id, messageId: msgId, startTime: new Date(), url: args[0], command: commandName, tool, infoBlock, urlContext, requesterUserId, showLimits, limitsAtStart, locale }; // #594: showLimits/limitsAtStart
+    // #1927 review follow-up: persist the full args so a killed /solve can be
+    //   resumed with its exact original invocation + `--resume <lastSessionId>`.
+    const baseSessionInfo = { chatId: ctx.chat.id, messageId: msgId, startTime: new Date(), url: args[0], command: commandName, tool, infoBlock, urlContext, requesterUserId, showLimits, limitsAtStart, locale, args: Array.isArray(args) ? [...args] : undefined }; // #594: showLimits/limitsAtStart
     const iso = await resolveIsolation(perCommandIsolation, ISOLATION_BACKEND, isolationRunner, VERBOSE);
     let result, session, sessionInfo;
     if (iso) {

package/src/telegram-terminal-watch-command.lib.mjs

@@ -7,6 +7,8 @@
 
 import fs from 'fs/promises';
 import { extractSessionIdFromText, decideLogDestination, resolveLogPath } from './telegram-log-command.lib.mjs';
+import { parseSessionExitFooter } from './isolation-runner.lib.mjs';
+import { classifyExitStatus, isFailureSessionStatus } from './session-status.lib.mjs';
 
 const DEFAULT_WIDTH = 120;
 const DEFAULT_HEIGHT = 25;
@@ -124,7 +126,11 @@ export function formatTerminalWatchMessage({ sessionId, statusResult = null, log
   const width = options.width || DEFAULT_WIDTH;
   const height = options.height || DEFAULT_HEIGHT;
   const snapshot = sanitizeCodeBlock(tailTextForTerminal(logText, options));
-  const title = completed ? '✅ Terminal watch complete' : '🔄 Live terminal watch';
+  // Issue #1927: a completed-but-failed/killed session must not wear a success
+  // ✅ — surface the failure so an OOM/SIGKILL is reported, not mistaken for a
+  // clean finish. Both titles keep the "Terminal watch complete" phrase.
+  const failed = completed && isFailureSessionStatus(status);
+  const title = !completed ? '🔄 Live terminal watch' : failed ? '❌ Terminal watch complete — session failed' : '✅ Terminal watch complete';
   const lines = [title, `Session: \`${sessionId}\``, `Status: \`${status}\``, `Terminal: \`${width}x${height}\``];
   if (repoDescription) lines.push(`Repo: \`${repoDescription}\``);
   if (!completed) lines.push(`Updates: ${updateCount}`);
@@ -183,6 +189,37 @@ function getDisplayedTerminalSnapshot(logText, options) {
   return sanitizeCodeBlock(tailTextForTerminal(logText, options));
 }
 
+/**
+ * Issue #1927: decide whether a watched session has actually finished.
+ *
+ * A non-terminal `$ --status` (e.g. `executing`) is NOT trusted on its own —
+ * start-command can keep reporting `executing` after the wrapped command was
+ * SIGKILLed/OOM-killed (a lingering shell outlives it). Trusting that status
+ * would make this watch poll forever and render a misleading "still running"
+ * snapshot — the same silent-hang that left issue #1927's killed `/solve`
+ * unreported, here in the watch loop. The execution-log FOOTER ("Exit Code: N")
+ * that `start` writes is authoritative: once present the command has terminated,
+ * full stop. In that case the displayed status is corrected to the real terminal
+ * status (e.g. `killed`) so the kill is surfaced instead of a perpetual
+ * `executing`.
+ *
+ * @returns {{completed: boolean, statusResult: object|null}}
+ */
+export function reconcileWatchCompletion(statusResult, logText, isTerminalSessionStatus) {
+  if (statusResult?.status && isTerminalSessionStatus(statusResult.status)) {
+    return { completed: true, statusResult };
+  }
+  const footer = parseSessionExitFooter(logText);
+  if (footer.finished) {
+    const corrected = classifyExitStatus(footer.exitCode) || (footer.exitCode === 0 ? 'executed' : 'failed');
+    return {
+      completed: true,
+      statusResult: { ...(statusResult || {}), status: corrected, exitCode: footer.exitCode, endTime: statusResult?.endTime || footer.endTime || null },
+    };
+  }
+  return { completed: false, statusResult };
+}
+
 export function watchTerminalLogSession({ bot, chatId, messageId, sessionId, logPath, querySessionStatus, isTerminalSessionStatus, options = {}, repoDescription = null, verbose = false, initialStatusResult = null, initialLogText = null, initialMessage = '' }) {
   const key = `${chatId}:${messageId}:${sessionId}`;
   activeWatches.get(key)?.stop();
@@ -190,7 +227,8 @@ export function watchTerminalLogSession({ bot, chatId, messageId, sessionId, log
   let stopped = false;
   const hasInitialLogText = initialLogText !== null && initialLogText !== undefined;
   let lastSnapshot = hasInitialLogText ? getDisplayedTerminalSnapshot(initialLogText, options) : null;
-  let lastMessage = initialMessage || (hasInitialLogText ? formatTerminalWatchMessage({ sessionId, statusResult: initialStatusResult, logText: initialLogText, options, updateCount: 0, completed: !!initialStatusResult?.status && isTerminalSessionStatus(initialStatusResult.status), repoDescription }) : '');
+  const initialReconciled = hasInitialLogText ? reconcileWatchCompletion(initialStatusResult, initialLogText, isTerminalSessionStatus) : { completed: false, statusResult: initialStatusResult };
+  let lastMessage = initialMessage || (hasInitialLogText ? formatTerminalWatchMessage({ sessionId, statusResult: initialReconciled.statusResult, logText: initialLogText, options, updateCount: 0, completed: initialReconciled.completed, repoDescription }) : '');
   let updateCount = 0;
   let timer = null;
   const intervalMs = options.intervalMs || DEFAULT_INTERVAL_MS;
@@ -198,9 +236,12 @@ export function watchTerminalLogSession({ bot, chatId, messageId, sessionId, log
   const tick = async () => {
     if (stopped) return;
     try {
-      const statusResult = await querySessionStatus(sessionId, verbose);
-      const completed = !!statusResult?.status && isTerminalSessionStatus(statusResult.status);
+      const rawStatus = await querySessionStatus(sessionId, verbose);
       const logText = await readLogFile(logPath);
+      // Issue #1927: cross-check the authoritative log footer so a session killed
+      // while `$ --status` still reports `executing` is detected as finished
+      // instead of being polled forever with a misleading "running" snapshot.
+      const { completed, statusResult } = reconcileWatchCompletion(rawStatus, logText, isTerminalSessionStatus);
       const snapshot = getDisplayedTerminalSnapshot(logText, options);
       const snapshotChanged = snapshot !== lastSnapshot;
       if (snapshotChanged) updateCount++;
@@ -272,8 +313,8 @@ async function startWatchFromResolvedSession({ bot, ctx, sessionId, statusResult
   if (!targetChatId) return { started: false, reason: 'Missing target chat id' };
 
   const initialLogText = await readLogFile(logPath);
-  const initialCompleted = !!statusResult?.status && isTerminalSessionStatus(statusResult.status);
-  const initialText = formatTerminalWatchMessage({ sessionId, statusResult, logText: initialLogText, options: watchOptions, completed: initialCompleted, repoDescription });
+  const { completed: initialCompleted, statusResult: reconciledInitialStatus } = reconcileWatchCompletion(statusResult, initialLogText, isTerminalSessionStatus);
+  const initialText = formatTerminalWatchMessage({ sessionId, statusResult: reconciledInitialStatus, logText: initialLogText, options: watchOptions, completed: initialCompleted, repoDescription });
   let replyToMessageId = ctx.message?.message_id || undefined;
   if (decision.destination === 'dm' && ctx.chat.type !== 'private') {
     replyToMessageId = await forwardOrCopyToDm(ctx, ctx.message?.reply_to_message || ctx.message);

package/src/work-session-formatting.lib.mjs

@@ -1,7 +1,6 @@
 import { t } from './i18n.lib.mjs';
 import { escapeMarkdown } from './telegram-markdown.lib.mjs';
-
-const FAILURE_STATUSES = new Set(['failed', 'cancelled', 'canceled', 'error']);
+import { FAILURE_SESSION_STATUSES, KILLED_SESSION_STATUSES, isKilledSessionStatus, describeExitSignal, normalizeExitCode } from './session-status.lib.mjs';
 
 function text(locale, key, fallback, params = {}) {
   if (!locale) return fallback;
@@ -14,12 +13,6 @@ function parseDateValue(value) {
   return Number.isNaN(date.getTime()) ? null : date;
 }
 
-function normalizeExitCode(value) {
-  if (value === null || value === undefined) return null;
-  const numeric = Number(value);
-  return Number.isFinite(numeric) ? numeric : null;
-}
-
 export function getSessionCompletionExitCode({ exitCode = null, statusResult = null } = {}) {
   const explicitExitCode = normalizeExitCode(exitCode);
   if (explicitExitCode !== null) return explicitExitCode;
@@ -28,11 +21,34 @@ export function getSessionCompletionExitCode({ exitCode = null, statusResult = n
   if (statusExitCode !== null) return statusExitCode;
 
   const status = String(statusResult?.status || '').toLowerCase();
-  if (FAILURE_STATUSES.has(status)) return 1;
+  if (FAILURE_SESSION_STATUSES.has(status)) return 1;
 
   return null;
 }
 
+/**
+ * Decide how a completed session should be presented: success, generic failure,
+ * or an explicit kill (OOM/SIGKILL/SIGTERM/…). A session counts as "killed"
+ * when its exit code is a signal exit (>128) or its status is one of the kill
+ * statuses. This is what stops a SIGKILLed /solve from ever being labelled
+ * "finished successfully" (issue #1927, requirement #1).
+ *
+ * @param {Object} params
+ * @param {number|null} params.exitCode - Resolved final exit code
+ * @param {string|null} [params.status] - Session status string, if known
+ * @returns {{ failed: boolean, killed: boolean, signal: object|null }}
+ */
+export function classifySessionOutcome({ exitCode = null, status = null } = {}) {
+  const code = normalizeExitCode(exitCode);
+  const signal = describeExitSignal(code);
+  const killedByStatus = isKilledSessionStatus(status);
+  const killed = Boolean(signal) || killedByStatus;
+  const failed = killed || (code !== null && code !== 0) || FAILURE_SESSION_STATUSES.has(String(status || '').toLowerCase());
+  return { failed, killed, signal };
+}
+
+export { KILLED_SESSION_STATUSES };
+
 export function formatSessionDurationSeconds(seconds) {
   const totalSeconds = Math.max(0, Math.round(Number(seconds) || 0));
   const days = Math.floor(totalSeconds / 86400);
@@ -104,10 +120,27 @@ export function appendPullRequestLine(infoBlock, pullRequestUrl, { locale = null
 
 export function formatSessionCompletionMessage({ sessionName, sessionInfo, statusResult = null, observedEndTime = new Date(), exitCode = null, infoBlock = '', pullRequestUrl = null, extraSections = [], locale = null } = {}) {
   const finalExitCode = getSessionCompletionExitCode({ exitCode, statusResult });
-  const failed = finalExitCode !== null && finalExitCode !== 0;
+  const outcome = classifySessionOutcome({ exitCode: finalExitCode, status: statusResult?.status || null });
+  const { failed, killed, signal } = outcome;
   const statusEmoji = failed ? '❌' : '✅';
   const messageLocale = locale || sessionInfo?.locale || null;
-  const statusText = failed ? text(messageLocale, 'telegram.work_session_failed', `Work session failed (exit code: ${finalExitCode})`, { exitCode: finalExitCode }) : text(messageLocale, 'telegram.work_session_finished', 'Work session finished successfully');
+  // Issue #1927: a killed session (OOM/SIGKILL/SIGTERM) must never read as a
+  // success, and the signal/reason is surfaced explicitly so an operator can
+  // tell an out-of-memory kill apart from an ordinary non-zero exit.
+  let statusText;
+  if (killed) {
+    // A real signal exit is always >128; an exit code of exactly 1 on a
+    // status-only kill (process vanished, code unknown) is a synthesized failure
+    // sentinel, so suppress the misleading "(exit code: 1)" in that case.
+    const showCode = finalExitCode !== null && !(!signal && finalExitCode === 1);
+    const exitSuffix = showCode ? ` (exit code: ${finalExitCode})` : '';
+    const reason = signal ? signal.reason : 'killed';
+    statusText = text(messageLocale, 'telegram.work_session_killed', `Work session ${reason}${exitSuffix}`, { reason, exitCode: finalExitCode ?? '', signal: signal?.signal ?? '' });
+  } else if (failed) {
+    statusText = text(messageLocale, 'telegram.work_session_failed', `Work session failed (exit code: ${finalExitCode})`, { exitCode: finalExitCode });
+  } else {
+    statusText = text(messageLocale, 'telegram.work_session_finished', 'Work session finished successfully');
+  }
   const durationLabel = text(messageLocale, 'telegram.duration_label', 'Duration');
   const sessionLabel = text(messageLocale, 'telegram.session_label', 'Session');
   const isolationLabel = text(messageLocale, 'telegram.isolation_label', 'Isolation');