appback-remoteagent 0.14.3 → 0.14.5

package/.env.example

@@ -14,6 +14,7 @@ TELEGRAM_ACTIVE_POLL_INTERVAL_MS=3000
 TELEGRAM_RUNNING_POLL_INTERVAL_MS=60000
 TELEGRAM_SECONDARY_POLL_INTERVAL_MS=60000
 TELEGRAM_TERTIARY_POLL_INTERVAL_MS=180000
+TELEGRAM_RECOVERY_CHECK_INTERVAL_MS=60000
 ARTIFACT_CLEANUP_ENABLED=true
 ARTIFACT_RETENTION_DAYS=30
 ARTIFACT_CLEANUP_INTERVAL_MS=86400000

package/dist/bot.js

@@ -1087,10 +1087,14 @@ async function runWithPendingAnimation(botToken, chatId, task) {
                 const rendered = formatProviderTelegramChunks(progressChunks, parseMode);
                 const extra = rendered.parseMode ? { parse_mode: rendered.parseMode } : undefined;
                 for (const chunk of rendered.chunks) {
-                    await sendTelegramMessage(botToken, chatId, chunk, extra);
+                    await sendTelegramMessage(botToken, chatId, chunk, extra).catch((error) => {
+                        console.warn(`[telegram-progress-delivery] chat=${chatId} dropped progress message: ${formatTelegramDeliveryError(error)}`);
+                    });
                 }
                 if (normalized.documents.length > 0) {
-                    await sendTelegramDocuments(botToken, chatId, normalized.documents);
+                    await sendTelegramDocuments(botToken, chatId, normalized.documents).catch((error) => {
+                        console.warn(`[telegram-progress-delivery] chat=${chatId} dropped progress document(s): ${formatTelegramDeliveryError(error)}`);
+                    });
                 }
             },
         };
@@ -2233,23 +2237,38 @@ async function sendTelegramDocument(botToken, chatId, document) {
 async function sendTelegramMessage(botToken, chatId, text, extra) {
     const startedAt = Date.now();
     try {
-        try {
-            return await callTelegramApi(botToken, "sendMessage", {
-                chat_id: String(chatId),
-                text,
-                parse_mode: extra?.parse_mode,
-            });
-        }
-        catch (error) {
-            if (!extra?.parse_mode || !isTelegramEntityParseError(error)) {
-                throw error;
+        let lastError;
+        for (let attempt = 1; attempt <= 3; attempt += 1) {
+            try {
+                try {
+                    return await callTelegramApi(botToken, "sendMessage", {
+                        chat_id: String(chatId),
+                        text,
+                        parse_mode: extra?.parse_mode,
+                    });
+                }
+                catch (error) {
+                    if (!extra?.parse_mode || !isTelegramEntityParseError(error)) {
+                        throw error;
+                    }
+                    console.warn(`[telegram-sendMessage-fallback] chat=${chatId} parseMode=${extra.parse_mode}: ${formatTelegramDeliveryError(error)}`);
+                    return await callTelegramApi(botToken, "sendMessage", {
+                        chat_id: String(chatId),
+                        text: stripTelegramHtml(text),
+                    });
+                }
+            }
+            catch (error) {
+                lastError = error;
+                if (attempt >= 3 || !isRetryableTelegramDeliveryError(error)) {
+                    throw error;
+                }
+                const delayMs = attempt * 1500;
+                console.warn(`[telegram-sendMessage-retry] chat=${chatId} attempt=${attempt}/3 delayMs=${delayMs}: ${formatTelegramDeliveryError(error)}`);
+                await sleep(delayMs);
             }
-            console.warn(`[telegram-sendMessage-fallback] chat=${chatId} parseMode=${extra.parse_mode}: ${error instanceof Error ? error.message : String(error)}`);
-            return await callTelegramApi(botToken, "sendMessage", {
-                chat_id: String(chatId),
-                text: stripTelegramHtml(text),
-            });
         }
+        throw lastError instanceof Error ? lastError : new Error(String(lastError));
     }
     finally {
         const elapsedMs = Date.now() - startedAt;
@@ -2289,7 +2308,16 @@ async function callTelegramApi(botToken, method, params) {
             args.push("--data-urlencode", `${key}=${value}`);
         }
     }
-    const { stdout, stderr } = await execFileAsync("curl", args);
+    let stdout;
+    let stderr;
+    try {
+        const result = await execFileAsync("curl", args);
+        stdout = result.stdout;
+        stderr = result.stderr;
+    }
+    catch (error) {
+        throw new TelegramApiError(method, summarizeCurlTelegramError(error, method));
+    }
     if (stderr?.trim()) {
         console.error(`curl stderr for ${method}: ${stderr.trim()}`);
     }
@@ -2309,6 +2337,26 @@ class TelegramApiError extends Error {
         this.name = "TelegramApiError";
     }
 }
+function summarizeCurlTelegramError(error, method) {
+    if (!(error instanceof Error)) {
+        return `Telegram API ${method} failed: ${String(error)}`;
+    }
+    const stderr = typeof error === "object" && error !== null && "stderr" in error
+        ? String(error.stderr ?? "").trim()
+        : "";
+    const code = typeof error === "object" && error !== null && "code" in error
+        ? String(error.code ?? "")
+        : "";
+    const message = stderr || error.message;
+    return [`Telegram API ${method} curl failed`, code ? `code=${code}` : undefined, message].filter(Boolean).join(": ");
+}
+function formatTelegramDeliveryError(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function isRetryableTelegramDeliveryError(error) {
+    const message = formatTelegramDeliveryError(error);
+    return /timed out|timeout|Bad Gateway|502|503|504|ECONNRESET|connection reset|EAI_AGAIN|ENOTFOUND/i.test(message);
+}
 function isTelegramForbiddenError(error) {
     if (error instanceof GrammyError) {
         return error.error_code === 403 || /^Forbidden:/i.test(error.description);

package/dist/config.js

@@ -140,6 +140,7 @@ export const config = {
     telegramRunningPollIntervalMs: readTimeout("TELEGRAM_RUNNING_POLL_INTERVAL_MS", 60_000),
     telegramSecondaryPollIntervalMs: readTimeout("TELEGRAM_SECONDARY_POLL_INTERVAL_MS", 60_000),
     telegramTertiaryPollIntervalMs: readTimeout("TELEGRAM_TERTIARY_POLL_INTERVAL_MS", 180_000),
+    telegramRecoveryCheckIntervalMs: readTimeout("TELEGRAM_RECOVERY_CHECK_INTERVAL_MS", 60_000),
     telegramOwnerId: readOptional("TELEGRAM_OWNER_ID"),
     telegramMessageBatchMs: readNonNegativeTimeout("TELEGRAM_MESSAGE_BATCH_MS", 1500),
     telegramTypingIntervalMs: readTimeout("TELEGRAM_TYPING_INTERVAL_MS", 10_000),

package/dist/index.js

@@ -15,6 +15,7 @@ import { BotManagementService } from "./services/bot-management-service.js";
 import { LocalUiService } from "./services/local-ui-service.js";
 import { AgentMemoryService } from "./services/agent-memory-service.js";
 import { BotPollingStateService } from "./services/bot-polling-state-service.js";
+import { ProviderRecoveryService } from "./services/provider-recovery-service.js";
 import { computePolicyPollIntervalMs, computeRecentMessageRanks } from "./services/polling-policy.js";
 import { terminateAllSpawnedExecutions } from "./adapters/windows-shell.js";
 import { setTelegramCommandMenu } from "./telegram-command-menu.js";
@@ -139,6 +140,9 @@ main().catch((error) => {
 async function startManualPollingScheduler(bots) {
     const pollingBots = bots;
     const runtimeStates = new Map();
+    const providerRecovery = new ProviderRecoveryService(config.dataDir, botPollingState);
+    let nextRecoveryCheckAt = 0;
+    let recoveryCheckInFlight = false;
     const botIds = pollingBots.map((bot) => String(bot.botInfo.id));
     await botPollingState.prune(botIds);
     for (const [index, bot] of pollingBots.entries()) {
@@ -158,6 +162,23 @@ async function startManualPollingScheduler(bots) {
     }
     while (true) {
         const now = Date.now();
+        if (!recoveryCheckInFlight && now >= nextRecoveryCheckAt) {
+            recoveryCheckInFlight = true;
+            nextRecoveryCheckAt = now + config.telegramRecoveryCheckIntervalMs;
+            void providerRecovery.reconcileStaleRunningStates(pollingBots.map((bot) => ({
+                botId: String(bot.botInfo.id),
+                username: bot.botInfo.username,
+                token: bot.token,
+            }))).then((result) => {
+                if (result.recovered > 0) {
+                    console.warn(`[provider-recovery] cleared ${result.recovered} stale running session(s) after checking ${result.checked}.`);
+                }
+            }).catch((error) => {
+                console.error("[provider-recovery] reconciliation failed:", error);
+            }).finally(() => {
+                recoveryCheckInFlight = false;
+            });
+        }
         let activePolls = [...runtimeStates.values()].filter((state) => state.inFlight).length;
         const pollingStates = await botPollingState.list();
         const rankByBotId = computeRecentMessageRanks(botIds, pollingStates);

package/dist/services/bot-polling-state-service.js

@@ -98,6 +98,25 @@ export class BotPollingStateService {
         };
         this.scheduleWrite();
     }
+    async clearRunningSessions(botId, sessionIds, patch = {}) {
+        const remove = new Set(sessionIds);
+        if (remove.size === 0) {
+            return undefined;
+        }
+        const state = await this.read();
+        const current = state.bots[botId];
+        if (!current?.runningSessionIds?.length) {
+            return current;
+        }
+        current.runningSessionIds = current.runningSessionIds.filter((value) => !remove.has(value));
+        if (current.runningSessionIds.length === 0) {
+            delete current.runningSessionIds;
+        }
+        Object.assign(current, patch);
+        state.bots[botId] = current;
+        await this.writeNow();
+        return current;
+    }
     async prune(validBotIds) {
         const valid = new Set(validBotIds);
         const state = await this.read();

package/dist/services/provider-recovery-service.js

@@ -0,0 +1,261 @@
+import { execFile } from "node:child_process";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { promisify } from "node:util";
+const execFileAsync = promisify(execFile);
+const EMPTY_STATE = { chats: {}, sessions: {}, telegramContacts: {}, settings: {} };
+export class ProviderRecoveryService {
+    dataDir;
+    pollingState;
+    noticePath;
+    constructor(dataDir, pollingState) {
+        this.dataDir = dataDir;
+        this.pollingState = pollingState;
+        this.noticePath = path.join(dataDir, "provider-recovery-notices.json");
+    }
+    async reconcileStaleRunningStates(bots) {
+        const botById = new Map(bots.map((bot) => [bot.botId, bot]));
+        const pollingStates = await this.pollingState.list();
+        const runningEntries = Object.entries(pollingStates)
+            .filter(([, state]) => state.runningSessionIds && state.runningSessionIds.length > 0);
+        if (runningEntries.length === 0) {
+            return { checked: 0, recovered: 0 };
+        }
+        const bridgeState = await this.readBridgeState();
+        const processNeedles = await readProviderProcessNeedles();
+        let checked = 0;
+        let recovered = 0;
+        for (const [botId, state] of runningEntries) {
+            const bot = botById.get(botId);
+            if (!bot) {
+                continue;
+            }
+            const staleSessionIds = [];
+            for (const sessionId of state.runningSessionIds ?? []) {
+                checked += 1;
+                const session = bridgeState.sessions[sessionId];
+                if (!session) {
+                    staleSessionIds.push(sessionId);
+                    continue;
+                }
+                const needles = providerNeedlesForSession(session);
+                if (needles.length === 0 || processNeedles.hasAny(needles)) {
+                    continue;
+                }
+                staleSessionIds.push(sessionId);
+            }
+            if (staleSessionIds.length === 0) {
+                continue;
+            }
+            const now = new Date().toISOString();
+            await this.pollingState.clearRunningSessions(botId, staleSessionIds, {
+                username: bot.username,
+                lastProviderFinishedAt: now,
+                lastRecoveryAt: now,
+                lastRecoveryReason: "stale running marker cleared; no matching provider process was found",
+            });
+            recovered += staleSessionIds.length;
+            for (const sessionId of staleSessionIds) {
+                const session = bridgeState.sessions[sessionId];
+                const binding = findBinding(bridgeState, botId, sessionId);
+                if (!session || !binding) {
+                    continue;
+                }
+                await this.notifyRecovered(bot, binding, session).catch((error) => {
+                    console.error(`[provider-recovery] failed to notify @${bot.username ?? bot.botId} for ${session.publicId}:`, summarizeError(error));
+                });
+            }
+        }
+        return { checked, recovered };
+    }
+    async notifyRecovered(bot, binding, session) {
+        const noticeKey = [
+            "stale-running-cleared",
+            bot.botId,
+            binding.chatId,
+            session.sessionId,
+            session.updatedAt,
+        ].join(":");
+        if (await this.wasNoticeDelivered(noticeKey)) {
+            return;
+        }
+        const text = [
+            `[RemoteAgent Recovery | ${session.publicId}]`,
+            "Stale running state was cleared.",
+            "",
+            "Reason:",
+            "RemoteAgent had a running marker for this session, but no matching Codex/Claude process was found.",
+            "",
+            "Action:",
+            "- cleared the running marker",
+            "- kept session, workspace, and history unchanged",
+            "- the next message can start a fresh provider execution",
+        ].join("\n");
+        await sendTelegramPlainText(bot.token, binding.chatId, text);
+        await this.markNoticeDelivered(noticeKey);
+    }
+    async wasNoticeDelivered(key) {
+        const notices = await this.readNoticeFile();
+        return Boolean(notices.delivered[key]);
+    }
+    async markNoticeDelivered(key) {
+        const notices = await this.readNoticeFile();
+        notices.updatedAt = new Date().toISOString();
+        notices.delivered[key] = notices.updatedAt;
+        await writeJsonAtomic(this.noticePath, notices);
+    }
+    async readNoticeFile() {
+        const raw = await fs.readFile(this.noticePath, "utf8").catch(() => "");
+        if (!raw.trim()) {
+            return { version: 1, updatedAt: "", delivered: {} };
+        }
+        try {
+            const parsed = JSON.parse(raw);
+            return {
+                version: 1,
+                updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "",
+                delivered: parsed.delivered && typeof parsed.delivered === "object" ? parsed.delivered : {},
+            };
+        }
+        catch {
+            return { version: 1, updatedAt: "", delivered: {} };
+        }
+    }
+    async readBridgeState() {
+        const state = await this.readLegacyState();
+        const sessionsDir = path.join(this.dataDir, "sessions");
+        const channelsDir = path.join(this.dataDir, "channels", "telegram");
+        const sessionDirs = await fs.readdir(sessionsDir, { withFileTypes: true }).catch(() => []);
+        for (const entry of sessionDirs) {
+            if (!entry.isDirectory()) {
+                continue;
+            }
+            const session = await readJson(path.join(sessionsDir, entry.name, "session.json"));
+            if (session?.sessionId) {
+                state.sessions[session.sessionId] = session;
+            }
+        }
+        const botDirs = await fs.readdir(channelsDir, { withFileTypes: true }).catch(() => []);
+        for (const botEntry of botDirs) {
+            if (!botEntry.isDirectory()) {
+                continue;
+            }
+            const botId = decodeURIComponent(botEntry.name);
+            const bindingFiles = await fs.readdir(path.join(channelsDir, botEntry.name), { withFileTypes: true }).catch(() => []);
+            for (const fileEntry of bindingFiles) {
+                if (!fileEntry.isFile() || !fileEntry.name.endsWith(".json")) {
+                    continue;
+                }
+                const binding = await readJson(path.join(channelsDir, botEntry.name, fileEntry.name));
+                if (binding?.chatId && binding.sessionId) {
+                    binding.botId = binding.botId || botId;
+                    state.chats[`${binding.botId}:${binding.chatId}`] = binding;
+                }
+            }
+        }
+        return state;
+    }
+    async readLegacyState() {
+        const raw = await fs.readFile(path.join(this.dataDir, "state.json"), "utf8").catch(() => "");
+        if (!raw.trim()) {
+            return { ...EMPTY_STATE, chats: {}, sessions: {}, telegramContacts: {}, settings: {} };
+        }
+        try {
+            const parsed = JSON.parse(raw);
+            return {
+                chats: parsed.chats && typeof parsed.chats === "object" ? parsed.chats : {},
+                sessions: parsed.sessions && typeof parsed.sessions === "object" ? parsed.sessions : {},
+                telegramContacts: parsed.telegramContacts && typeof parsed.telegramContacts === "object" ? parsed.telegramContacts : {},
+                settings: parsed.settings && typeof parsed.settings === "object" ? parsed.settings : {},
+            };
+        }
+        catch {
+            return { ...EMPTY_STATE, chats: {}, sessions: {}, telegramContacts: {}, settings: {} };
+        }
+    }
+}
+function providerNeedlesForSession(session) {
+    return [
+        `REMOTEAGENT_SESSION_ID=${session.sessionId}`,
+        session.codex?.sessionId,
+        session.claude?.sessionId,
+    ].filter((value) => Boolean(value && value.length >= 8));
+}
+function findBinding(state, botId, sessionId) {
+    return Object.values(state.chats).find((binding) => binding.botId === botId && binding.sessionId === sessionId);
+}
+async function readProviderProcessNeedles() {
+    const text = process.platform === "linux"
+        ? await readLinuxProcessText()
+        : await readPsProcessText();
+    return {
+        hasAny(needles) {
+            return needles.some((needle) => text.includes(needle));
+        },
+    };
+}
+async function readLinuxProcessText() {
+    const entries = await fs.readdir("/proc", { withFileTypes: true }).catch(() => []);
+    const chunks = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) {
+            continue;
+        }
+        const procDir = path.join("/proc", entry.name);
+        const [cmdline, environ] = await Promise.all([
+            fs.readFile(path.join(procDir, "cmdline"), "utf8").catch(() => ""),
+            fs.readFile(path.join(procDir, "environ"), "utf8").catch(() => ""),
+        ]);
+        if (cmdline || environ) {
+            chunks.push(cmdline.replace(/\0/g, " "), environ.replace(/\0/g, " "));
+        }
+    }
+    return chunks.join("\n");
+}
+async function readPsProcessText() {
+    const { stdout } = await execFileAsync("ps", ["-eo", "pid=,args="]).catch(() => ({ stdout: "" }));
+    return stdout;
+}
+async function sendTelegramPlainText(token, chatId, text) {
+    const payload = JSON.stringify({
+        chat_id: chatId,
+        text,
+    });
+    const { stdout } = await execFileAsync("curl", [
+        "-sS",
+        "-4",
+        "--max-time",
+        "15",
+        "-H",
+        "Content-Type: application/json",
+        "-d",
+        payload,
+        `https://api.telegram.org/bot${token}/sendMessage`,
+    ]);
+    const parsed = JSON.parse(stdout);
+    if (!parsed.ok) {
+        throw new Error(parsed.description || "sendMessage failed");
+    }
+}
+async function readJson(filePath) {
+    const raw = await fs.readFile(filePath, "utf8").catch(() => "");
+    if (!raw.trim()) {
+        return undefined;
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+}
+async function writeJsonAtomic(filePath, value) {
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    const temporaryPath = `${filePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
+    await fs.writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+    await fs.rename(temporaryPath, filePath);
+}
+function summarizeError(error) {
+    return error instanceof Error ? error.message.replace(/bot\d+:[A-Za-z0-9_-]+/g, "bot[redacted]") : String(error);
+}

package/dist/telegram-command-menu.js

@@ -4,13 +4,19 @@ const execFileAsync = promisify(execFile);
 export const TELEGRAM_COMMAND_MENU = [
     { command: "start", description: "Start a new Codex or Claude session" },
     { command: "list", description: "List sessions" },
+    { command: "new", description: "Start a fresh session" },
     { command: "switch", description: "Switch to a session" },
     { command: "status", description: "Show current session status" },
+    { command: "attach", description: "Attach an existing provider session" },
     { command: "state", description: "Show or edit session state notes" },
     { command: "option", description: "Show or change runtime options" },
+    { command: "secret", description: "Store or manage hidden secret values" },
+    { command: "docs", description: "Pin or find session documents" },
     { command: "model", description: "Show or change provider model" },
     { command: "stop", description: "Stop active work and clear queued messages" },
+    { command: "sandbox", description: "Set Codex sandbox mode" },
     { command: "batch", description: "Collect and send a multi-message batch" },
+    { command: "artifacts", description: "List or clean uploaded artifacts" },
     { command: "bots", description: "List configured Telegram bots" },
     { command: "bot", description: "Manage Telegram bots" },
     { command: "install", description: "Install or update Codex or Claude" },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appback-remoteagent",
-  "version": "0.14.3",
+  "version": "0.14.5",
   "description": "Personal installable session server for continuing local AI work across PC and Telegram",
   "license": "MIT",
   "type": "module",