appback-remoteagent 0.14.4 → 0.14.6

package/.env.example

@@ -14,6 +14,7 @@ TELEGRAM_ACTIVE_POLL_INTERVAL_MS=3000
 TELEGRAM_RUNNING_POLL_INTERVAL_MS=60000
 TELEGRAM_SECONDARY_POLL_INTERVAL_MS=60000
 TELEGRAM_TERTIARY_POLL_INTERVAL_MS=180000
+TELEGRAM_RECOVERY_CHECK_INTERVAL_MS=60000
 ARTIFACT_CLEANUP_ENABLED=true
 ARTIFACT_RETENTION_DAYS=30
 ARTIFACT_CLEANUP_INTERVAL_MS=86400000

package/dist/bot.js

@@ -89,6 +89,7 @@ const RECOGNIZED_COMMANDS = new Set([
 ]);
 const TELEGRAM_STALE_UPDATE_GRACE_SECONDS = 10;
 const TELEGRAM_PROCESS_STARTED_AT_SECONDS = Math.floor(Date.now() / 1000);
+const activeWorkLoopKeys = new Set();
 const REPORT_CONTINUE_PROMPT = [
     "Continue the same task now.",
     "Do more concrete work before replying again.",
@@ -1132,6 +1133,16 @@ async function runWithPendingAnimation(botToken, chatId, task) {
 async function routeTelegramWorkLoop(bridge, botId, chatId, message, label, botManagement, helpers, autoContinue, memoryService, transform = (blocks) => blocks) {
     const currentSession = await bridge.status(botId, chatId);
     const sessionId = currentSession?.session.sessionId;
+    const activeKey = workLoopKey(botId, chatId, sessionId);
+    if (activeWorkLoopKeys.has(activeKey)) {
+        const message = [
+            `Session ${currentSession?.session.publicId ?? sessionId ?? `${botId}:${chatId}`} is already running.`,
+            "Send /stop to interrupt the active work, then send the instruction again.",
+        ].join("\n");
+        await bridge.logSystem(botId, chatId, `Rejected overlapping Telegram work loop for ${activeKey}.`);
+        return [message];
+    }
+    activeWorkLoopKeys.add(activeKey);
     if (currentSession) {
         await memoryService.recordInstruction(currentSession.session, message);
     }
@@ -1319,6 +1330,7 @@ async function routeTelegramWorkLoop(bridge, botId, chatId, message, label, botM
         }
     }
     finally {
+        activeWorkLoopKeys.delete(activeKey);
         if (currentSession) {
             if (providerCompleted) {
                 await botManagement.markProviderCompleted(botId, sessionId);
@@ -1329,6 +1341,9 @@ async function routeTelegramWorkLoop(bridge, botId, chatId, message, label, botM
         }
     }
 }
+function workLoopKey(botId, chatId, sessionId) {
+    return sessionId ? `session:${sessionId}` : `chat:${botId}:${chatId}`;
+}
 class SilentTelegramAbort extends Error {
     constructor(message) {
         super(message);

package/dist/config.js

@@ -140,6 +140,7 @@ export const config = {
     telegramRunningPollIntervalMs: readTimeout("TELEGRAM_RUNNING_POLL_INTERVAL_MS", 60_000),
     telegramSecondaryPollIntervalMs: readTimeout("TELEGRAM_SECONDARY_POLL_INTERVAL_MS", 60_000),
     telegramTertiaryPollIntervalMs: readTimeout("TELEGRAM_TERTIARY_POLL_INTERVAL_MS", 180_000),
+    telegramRecoveryCheckIntervalMs: readTimeout("TELEGRAM_RECOVERY_CHECK_INTERVAL_MS", 60_000),
     telegramOwnerId: readOptional("TELEGRAM_OWNER_ID"),
     telegramMessageBatchMs: readNonNegativeTimeout("TELEGRAM_MESSAGE_BATCH_MS", 1500),
     telegramTypingIntervalMs: readTimeout("TELEGRAM_TYPING_INTERVAL_MS", 10_000),

package/dist/index.js

@@ -15,6 +15,7 @@ import { BotManagementService } from "./services/bot-management-service.js";
 import { LocalUiService } from "./services/local-ui-service.js";
 import { AgentMemoryService } from "./services/agent-memory-service.js";
 import { BotPollingStateService } from "./services/bot-polling-state-service.js";
+import { ProviderRecoveryService } from "./services/provider-recovery-service.js";
 import { computePolicyPollIntervalMs, computeRecentMessageRanks } from "./services/polling-policy.js";
 import { terminateAllSpawnedExecutions } from "./adapters/windows-shell.js";
 import { setTelegramCommandMenu } from "./telegram-command-menu.js";
@@ -139,6 +140,9 @@ main().catch((error) => {
 async function startManualPollingScheduler(bots) {
     const pollingBots = bots;
     const runtimeStates = new Map();
+    const providerRecovery = new ProviderRecoveryService(config.dataDir, botPollingState);
+    let nextRecoveryCheckAt = 0;
+    let recoveryCheckInFlight = false;
     const botIds = pollingBots.map((bot) => String(bot.botInfo.id));
     await botPollingState.prune(botIds);
     for (const [index, bot] of pollingBots.entries()) {
@@ -158,6 +162,23 @@ async function startManualPollingScheduler(bots) {
     }
     while (true) {
         const now = Date.now();
+        if (!recoveryCheckInFlight && now >= nextRecoveryCheckAt) {
+            recoveryCheckInFlight = true;
+            nextRecoveryCheckAt = now + config.telegramRecoveryCheckIntervalMs;
+            void providerRecovery.reconcileStaleRunningStates(pollingBots.map((bot) => ({
+                botId: String(bot.botInfo.id),
+                username: bot.botInfo.username,
+                token: bot.token,
+            }))).then((result) => {
+                if (result.recovered > 0) {
+                    console.warn(`[provider-recovery] cleared ${result.recovered} stale running session(s) after checking ${result.checked}.`);
+                }
+            }).catch((error) => {
+                console.error("[provider-recovery] reconciliation failed:", error);
+            }).finally(() => {
+                recoveryCheckInFlight = false;
+            });
+        }
         let activePolls = [...runtimeStates.values()].filter((state) => state.inFlight).length;
         const pollingStates = await botPollingState.list();
         const rankByBotId = computeRecentMessageRanks(botIds, pollingStates);

package/dist/services/bot-polling-state-service.js

@@ -98,6 +98,25 @@ export class BotPollingStateService {
         };
         this.scheduleWrite();
     }
+    async clearRunningSessions(botId, sessionIds, patch = {}) {
+        const remove = new Set(sessionIds);
+        if (remove.size === 0) {
+            return undefined;
+        }
+        const state = await this.read();
+        const current = state.bots[botId];
+        if (!current?.runningSessionIds?.length) {
+            return current;
+        }
+        current.runningSessionIds = current.runningSessionIds.filter((value) => !remove.has(value));
+        if (current.runningSessionIds.length === 0) {
+            delete current.runningSessionIds;
+        }
+        Object.assign(current, patch);
+        state.bots[botId] = current;
+        await this.writeNow();
+        return current;
+    }
     async prune(validBotIds) {
         const valid = new Set(validBotIds);
         const state = await this.read();

package/dist/services/provider-recovery-service.js

@@ -0,0 +1,261 @@
+import { execFile } from "node:child_process";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { promisify } from "node:util";
+const execFileAsync = promisify(execFile);
+const EMPTY_STATE = { chats: {}, sessions: {}, telegramContacts: {}, settings: {} };
+export class ProviderRecoveryService {
+    dataDir;
+    pollingState;
+    noticePath;
+    constructor(dataDir, pollingState) {
+        this.dataDir = dataDir;
+        this.pollingState = pollingState;
+        this.noticePath = path.join(dataDir, "provider-recovery-notices.json");
+    }
+    async reconcileStaleRunningStates(bots) {
+        const botById = new Map(bots.map((bot) => [bot.botId, bot]));
+        const pollingStates = await this.pollingState.list();
+        const runningEntries = Object.entries(pollingStates)
+            .filter(([, state]) => state.runningSessionIds && state.runningSessionIds.length > 0);
+        if (runningEntries.length === 0) {
+            return { checked: 0, recovered: 0 };
+        }
+        const bridgeState = await this.readBridgeState();
+        const processNeedles = await readProviderProcessNeedles();
+        let checked = 0;
+        let recovered = 0;
+        for (const [botId, state] of runningEntries) {
+            const bot = botById.get(botId);
+            if (!bot) {
+                continue;
+            }
+            const staleSessionIds = [];
+            for (const sessionId of state.runningSessionIds ?? []) {
+                checked += 1;
+                const session = bridgeState.sessions[sessionId];
+                if (!session) {
+                    staleSessionIds.push(sessionId);
+                    continue;
+                }
+                const needles = providerNeedlesForSession(session);
+                if (needles.length === 0 || processNeedles.hasAny(needles)) {
+                    continue;
+                }
+                staleSessionIds.push(sessionId);
+            }
+            if (staleSessionIds.length === 0) {
+                continue;
+            }
+            const now = new Date().toISOString();
+            await this.pollingState.clearRunningSessions(botId, staleSessionIds, {
+                username: bot.username,
+                lastProviderFinishedAt: now,
+                lastRecoveryAt: now,
+                lastRecoveryReason: "stale running marker cleared; no matching provider process was found",
+            });
+            recovered += staleSessionIds.length;
+            for (const sessionId of staleSessionIds) {
+                const session = bridgeState.sessions[sessionId];
+                const binding = findBinding(bridgeState, botId, sessionId);
+                if (!session || !binding) {
+                    continue;
+                }
+                await this.notifyRecovered(bot, binding, session).catch((error) => {
+                    console.error(`[provider-recovery] failed to notify @${bot.username ?? bot.botId} for ${session.publicId}:`, summarizeError(error));
+                });
+            }
+        }
+        return { checked, recovered };
+    }
+    async notifyRecovered(bot, binding, session) {
+        const noticeKey = [
+            "stale-running-cleared",
+            bot.botId,
+            binding.chatId,
+            session.sessionId,
+            session.updatedAt,
+        ].join(":");
+        if (await this.wasNoticeDelivered(noticeKey)) {
+            return;
+        }
+        const text = [
+            `[RemoteAgent Recovery | ${session.publicId}]`,
+            "Stale running state was cleared.",
+            "",
+            "Reason:",
+            "RemoteAgent had a running marker for this session, but no matching Codex/Claude process was found.",
+            "",
+            "Action:",
+            "- cleared the running marker",
+            "- kept session, workspace, and history unchanged",
+            "- the next message can start a fresh provider execution",
+        ].join("\n");
+        await sendTelegramPlainText(bot.token, binding.chatId, text);
+        await this.markNoticeDelivered(noticeKey);
+    }
+    async wasNoticeDelivered(key) {
+        const notices = await this.readNoticeFile();
+        return Boolean(notices.delivered[key]);
+    }
+    async markNoticeDelivered(key) {
+        const notices = await this.readNoticeFile();
+        notices.updatedAt = new Date().toISOString();
+        notices.delivered[key] = notices.updatedAt;
+        await writeJsonAtomic(this.noticePath, notices);
+    }
+    async readNoticeFile() {
+        const raw = await fs.readFile(this.noticePath, "utf8").catch(() => "");
+        if (!raw.trim()) {
+            return { version: 1, updatedAt: "", delivered: {} };
+        }
+        try {
+            const parsed = JSON.parse(raw);
+            return {
+                version: 1,
+                updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "",
+                delivered: parsed.delivered && typeof parsed.delivered === "object" ? parsed.delivered : {},
+            };
+        }
+        catch {
+            return { version: 1, updatedAt: "", delivered: {} };
+        }
+    }
+    async readBridgeState() {
+        const state = await this.readLegacyState();
+        const sessionsDir = path.join(this.dataDir, "sessions");
+        const channelsDir = path.join(this.dataDir, "channels", "telegram");
+        const sessionDirs = await fs.readdir(sessionsDir, { withFileTypes: true }).catch(() => []);
+        for (const entry of sessionDirs) {
+            if (!entry.isDirectory()) {
+                continue;
+            }
+            const session = await readJson(path.join(sessionsDir, entry.name, "session.json"));
+            if (session?.sessionId) {
+                state.sessions[session.sessionId] = session;
+            }
+        }
+        const botDirs = await fs.readdir(channelsDir, { withFileTypes: true }).catch(() => []);
+        for (const botEntry of botDirs) {
+            if (!botEntry.isDirectory()) {
+                continue;
+            }
+            const botId = decodeURIComponent(botEntry.name);
+            const bindingFiles = await fs.readdir(path.join(channelsDir, botEntry.name), { withFileTypes: true }).catch(() => []);
+            for (const fileEntry of bindingFiles) {
+                if (!fileEntry.isFile() || !fileEntry.name.endsWith(".json")) {
+                    continue;
+                }
+                const binding = await readJson(path.join(channelsDir, botEntry.name, fileEntry.name));
+                if (binding?.chatId && binding.sessionId) {
+                    binding.botId = binding.botId || botId;
+                    state.chats[`${binding.botId}:${binding.chatId}`] = binding;
+                }
+            }
+        }
+        return state;
+    }
+    async readLegacyState() {
+        const raw = await fs.readFile(path.join(this.dataDir, "state.json"), "utf8").catch(() => "");
+        if (!raw.trim()) {
+            return { ...EMPTY_STATE, chats: {}, sessions: {}, telegramContacts: {}, settings: {} };
+        }
+        try {
+            const parsed = JSON.parse(raw);
+            return {
+                chats: parsed.chats && typeof parsed.chats === "object" ? parsed.chats : {},
+                sessions: parsed.sessions && typeof parsed.sessions === "object" ? parsed.sessions : {},
+                telegramContacts: parsed.telegramContacts && typeof parsed.telegramContacts === "object" ? parsed.telegramContacts : {},
+                settings: parsed.settings && typeof parsed.settings === "object" ? parsed.settings : {},
+            };
+        }
+        catch {
+            return { ...EMPTY_STATE, chats: {}, sessions: {}, telegramContacts: {}, settings: {} };
+        }
+    }
+}
+function providerNeedlesForSession(session) {
+    return [
+        `REMOTEAGENT_SESSION_ID=${session.sessionId}`,
+        session.codex?.sessionId,
+        session.claude?.sessionId,
+    ].filter((value) => Boolean(value && value.length >= 8));
+}
+function findBinding(state, botId, sessionId) {
+    return Object.values(state.chats).find((binding) => binding.botId === botId && binding.sessionId === sessionId);
+}
+async function readProviderProcessNeedles() {
+    const text = process.platform === "linux"
+        ? await readLinuxProcessText()
+        : await readPsProcessText();
+    return {
+        hasAny(needles) {
+            return needles.some((needle) => text.includes(needle));
+        },
+    };
+}
+async function readLinuxProcessText() {
+    const entries = await fs.readdir("/proc", { withFileTypes: true }).catch(() => []);
+    const chunks = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) {
+            continue;
+        }
+        const procDir = path.join("/proc", entry.name);
+        const [cmdline, environ] = await Promise.all([
+            fs.readFile(path.join(procDir, "cmdline"), "utf8").catch(() => ""),
+            fs.readFile(path.join(procDir, "environ"), "utf8").catch(() => ""),
+        ]);
+        if (cmdline || environ) {
+            chunks.push(cmdline.replace(/\0/g, " "), environ.replace(/\0/g, " "));
+        }
+    }
+    return chunks.join("\n");
+}
+async function readPsProcessText() {
+    const { stdout } = await execFileAsync("ps", ["-eo", "pid=,args="]).catch(() => ({ stdout: "" }));
+    return stdout;
+}
+async function sendTelegramPlainText(token, chatId, text) {
+    const payload = JSON.stringify({
+        chat_id: chatId,
+        text,
+    });
+    const { stdout } = await execFileAsync("curl", [
+        "-sS",
+        "-4",
+        "--max-time",
+        "15",
+        "-H",
+        "Content-Type: application/json",
+        "-d",
+        payload,
+        `https://api.telegram.org/bot${token}/sendMessage`,
+    ]);
+    const parsed = JSON.parse(stdout);
+    if (!parsed.ok) {
+        throw new Error(parsed.description || "sendMessage failed");
+    }
+}
+async function readJson(filePath) {
+    const raw = await fs.readFile(filePath, "utf8").catch(() => "");
+    if (!raw.trim()) {
+        return undefined;
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+}
+async function writeJsonAtomic(filePath, value) {
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    const temporaryPath = `${filePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
+    await fs.writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+    await fs.rename(temporaryPath, filePath);
+}
+function summarizeError(error) {
+    return error instanceof Error ? error.message.replace(/bot\d+:[A-Za-z0-9_-]+/g, "bot[redacted]") : String(error);
+}

package/dist/telegram-command-menu.js

@@ -4,13 +4,19 @@ const execFileAsync = promisify(execFile);
 export const TELEGRAM_COMMAND_MENU = [
     { command: "start", description: "Start a new Codex or Claude session" },
     { command: "list", description: "List sessions" },
+    { command: "new", description: "Start a fresh session" },
     { command: "switch", description: "Switch to a session" },
     { command: "status", description: "Show current session status" },
+    { command: "attach", description: "Attach an existing provider session" },
     { command: "state", description: "Show or edit session state notes" },
     { command: "option", description: "Show or change runtime options" },
+    { command: "secret", description: "Store or manage hidden secret values" },
+    { command: "docs", description: "Pin or find session documents" },
     { command: "model", description: "Show or change provider model" },
     { command: "stop", description: "Stop active work and clear queued messages" },
+    { command: "sandbox", description: "Set Codex sandbox mode" },
     { command: "batch", description: "Collect and send a multi-message batch" },
+    { command: "artifacts", description: "List or clean uploaded artifacts" },
     { command: "bots", description: "List configured Telegram bots" },
     { command: "bot", description: "Manage Telegram bots" },
     { command: "install", description: "Install or update Codex or Claude" },

package/docs/OPERATIONS.md

@@ -100,8 +100,11 @@ journalctl -u remoteagent -f
 Restart after build:
 
 ```bash
-sudo npm install -g appback-remoteagent@<version>
+export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/versions/node/v22.22.0/bin:$PATH"
+npm install -g appback-remoteagent@<version>
+remoteagent-install
 sudo systemctl restart remoteagent
+node -p 'require("/home/au2223/.nvm/versions/node/v22.22.0/lib/node_modules/appback-remoteagent/package.json").version'
 ```
 
 Check the lock owner:

package/docs/RELEASING.md

@@ -100,14 +100,20 @@ Server 30 uses a systemd service:
 VERSION=0.14.0
 ssh au2223@192.168.0.30 "VERSION=$VERSION bash -s" <<'REMOTE'
 set -euo pipefail
-sudo -n env "PATH=$PATH" npm install -g appback-remoteagent@$VERSION
+export PATH="/home/au2223/.local/bin:/home/au2223/.nvm/versions/node/v22.22.0/bin:$PATH"
+npm install -g appback-remoteagent@$VERSION
 /home/au2223/.nvm/versions/node/v22.22.0/bin/remoteagent-install
 sudo -n systemctl restart remoteagent
 systemctl is-active remoteagent
+node -p 'require("/home/au2223/.nvm/versions/node/v22.22.0/lib/node_modules/appback-remoteagent/package.json").version'
 journalctl -u remoteagent --since '2 minutes ago' --no-pager
 REMOTE
 ```
 
+Do not run `sudo npm install -g` on server 30. The global RemoteAgent package is installed under
+`/home/au2223/.nvm/versions/node/v22.22.0`; only the `systemctl restart remoteagent` step needs sudo.
+Runtime state remains in `/home/au2223/.remoteagent` and must not be deleted during npm upgrades.
+
 Server 26 currently uses user-level start/stop scripts, not a systemd `remoteagent.service`:
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appback-remoteagent",
-  "version": "0.14.4",
+  "version": "0.14.6",
   "description": "Personal installable session server for continuing local AI work across PC and Telegram",
   "license": "MIT",
   "type": "module",