@vellumai/assistant 0.11.1 → 0.11.2-staging.2

package/src/messaging/providers/telegram-bot/withdraw.ts

@@ -0,0 +1,166 @@
+/**
+ * Withdraw a Telegram approval card when its underlying guardian request
+ * resolves.
+ *
+ * Telegram bots cannot re-read a delivered message, so the Slack shape
+ * (fetch the original content, strip the buttons, append a status line in
+ * one edit) is not available here. The closest faithful projection is:
+ *
+ *   1. Remove the inline keyboard in place (`editMessageReplyMarkup`), which
+ *      preserves the card's text for the audit trail while dropping every
+ *      live affordance.
+ *   2. Post the terminal outcome as a silent reply quoting the card, so the
+ *      message thread durably shows what was decided. The reply is skipped
+ *      when the decision originated on Telegram: the guardian reply router
+ *      already delivers its own outcome message in that chat, and a second
+ *      notice would read as a duplicate.
+ *
+ * Bot API wrappers differ on the "remove markup" form, so the edit tries the
+ * explicit `null` first and falls back to an empty inline keyboard, mirroring
+ * the gateway's callback-time clearing. "message is not modified" means the
+ * keyboard is already gone (e.g. the gateway cleared it when the deciding tap
+ * arrived) and is treated as success. Unlike the gateway's tap path, a failed
+ * edit never deletes the message: withdrawal must preserve the audit trail,
+ * and a tap on a leftover button already gets the stale "already resolved"
+ * handling.
+ */
+
+import {
+  isParkAction,
+  resolveDecisionStatusWord,
+} from "../../../runtime/channel-approval-types.js";
+import { getLogger } from "../../../util/logger.js";
+import { callTelegramBotApi } from "./api.js";
+
+const log = getLogger("telegram-withdraw");
+
+const STATUS_GLYPH: Record<string, string> = {
+  approved: "✅",
+  denied: "❌",
+  expired: "⌛",
+  cancelled: "\u{1f6ab}",
+};
+
+/**
+ * Glyph for a parked (leave-unverified) decision: a neutral "on hold" mark
+ * rather than the `denied` cross, since the sender was neither trusted nor
+ * kept out.
+ */
+const PARK_STATUS_GLYPH = "⏸️";
+
+export interface WithdrawTelegramApprovalCardParams {
+  /** Telegram chat the approval message lives in. */
+  chatId: string;
+  /** Channel-native id of the approval message to withdraw. */
+  messageId: string;
+  /** Terminal status of the request (e.g. "approved", "denied", "expired"). */
+  status: string;
+  /**
+   * The action the guardian took, when known. A `denied` status reached by a
+   * park action (`leave_unverified`) reads as the neutral park label (see
+   * {@link resolveDecisionStatusWord}) rather than "Denied"; `block`/`reject`
+   * stay a denial. Omitted for status-only transitions (e.g. the expiry
+   * sweep).
+   */
+  decidedAction?: string;
+  /**
+   * Whether to post the quoted status reply. False when the decision
+   * originated on Telegram, where the guardian reply router already delivers
+   * the outcome message in the same chat.
+   */
+  postStatusReply: boolean;
+}
+
+/** Build the plain-text outcome line for the quoted status reply. */
+function buildStatusText(status: string, decidedAction?: string): string {
+  const park = status === "denied" && isParkAction(decidedAction);
+  const glyph = park ? PARK_STATUS_GLYPH : (STATUS_GLYPH[status] ?? "");
+  const word = resolveDecisionStatusWord(status, decidedAction);
+  return glyph ? `${glyph} ${word}` : word;
+}
+
+/** True for the Bot API's "message is not modified" no-op edit rejection. */
+function isNoOpMarkupError(err: unknown): boolean {
+  const msg = err instanceof Error ? err.message : String(err);
+  return msg.includes("message is not modified");
+}
+
+/**
+ * Remove the card's inline keyboard in place, keeping its text. Tries the
+ * explicit `reply_markup: null` form first, then an empty inline keyboard.
+ * Returns true when the keyboard is gone (including the already-cleared
+ * no-op case); false when both edit forms failed.
+ */
+async function clearInlineKeyboard(
+  chatId: string,
+  messageId: number,
+): Promise<boolean> {
+  const basePayload = { chat_id: chatId, message_id: messageId };
+  try {
+    await callTelegramBotApi("editMessageReplyMarkup", {
+      ...basePayload,
+      reply_markup: null,
+    });
+    return true;
+  } catch (primaryErr) {
+    if (isNoOpMarkupError(primaryErr)) {
+      return true;
+    }
+    try {
+      await callTelegramBotApi("editMessageReplyMarkup", {
+        ...basePayload,
+        reply_markup: { inline_keyboard: [] },
+      });
+      return true;
+    } catch (fallbackErr) {
+      if (isNoOpMarkupError(fallbackErr)) {
+        return true;
+      }
+      log.warn(
+        { primaryErr, fallbackErr, chatId, messageId },
+        "Failed to clear inline keyboard on resolved Telegram approval card",
+      );
+      return false;
+    }
+  }
+}
+
+/**
+ * Project a resolved guardian request onto its Telegram approval card:
+ * drop the inline keyboard in place and (unless suppressed) post a silent
+ * reply quoting the card with the terminal outcome.
+ *
+ * Both steps are attempted independently, so a failed keyboard edit does not
+ * lose the durable outcome notice. Throws only if the status reply send
+ * fails, which the caller treats as non-fatal.
+ */
+export async function withdrawTelegramApprovalCard(
+  params: WithdrawTelegramApprovalCardParams,
+): Promise<void> {
+  const parsedMessageId = Number(params.messageId);
+  if (!Number.isFinite(parsedMessageId)) {
+    log.warn(
+      { chatId: params.chatId, messageId: params.messageId },
+      "Skipping Telegram card withdrawal due to invalid message id",
+    );
+    return;
+  }
+
+  await clearInlineKeyboard(params.chatId, parsedMessageId);
+
+  if (!params.postStatusReply) {
+    return;
+  }
+
+  // Silent (no push) reply quoting the card. `allow_sending_without_reply`
+  // keeps the outcome notice deliverable even if the card was deleted.
+  await callTelegramBotApi("sendMessage", {
+    chat_id: params.chatId,
+    text: buildStatusText(params.status, params.decidedAction),
+    reply_parameters: {
+      message_id: parsedMessageId,
+      allow_sending_without_reply: true,
+    },
+    disable_notification: true,
+  });
+}

package/src/monitoring/__tests__/db-integrity-sample.test.ts

@@ -8,7 +8,13 @@ import { closeSync, mkdtempSync, openSync, rmSync, writeSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { Database } from "bun:sqlite";
-import { afterEach, beforeEach, expect, test } from "bun:test";
+import {
+  afterEach,
+  beforeEach,
+  expect,
+  setDefaultTimeout,
+  test,
+} from "bun:test";
 
 import { assertNotLiveDb } from "../../__tests__/assert-not-live-db.js";
 import {
@@ -17,6 +23,10 @@ import {
   runIntegrityCheck,
 } from "../db-integrity-check.js";
 
+// quick_check and the cold `bun run` subprocess are disk-bound; loaded CI
+// runners blow past the 5s default.
+setDefaultTimeout(30_000);
+
 let dir: string;
 let dbPath: string;
 
@@ -36,9 +46,13 @@ function seedDb(walMode: boolean): void {
     db.exec(`PRAGMA journal_mode=${walMode ? "WAL" : "DELETE"}`);
     db.exec("CREATE TABLE t (id TEXT PRIMARY KEY, v TEXT)");
     const ins = db.prepare("INSERT INTO t (id, v) VALUES (?, ?)");
-    for (let i = 0; i < 200; i++) {
-      ins.run(`k-${i}`, `v-${i}`);
-    }
+    // One transaction: 200 per-insert commits are fsync-bound and take
+    // seconds on loaded CI runners.
+    db.transaction(() => {
+      for (let i = 0; i < 200; i++) {
+        ins.run(`k-${i}`, `v-${i}`);
+      }
+    })();
   } finally {
     db.close();
   }

package/src/monitoring/__tests__/file-descriptors.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Tests for the descriptor poller's pure layers: limit parsing and the pressure
+ * decision. The collection functions read fixed `/proc` paths, so only the
+ * parsing and threshold logic are exercised here.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import {
+  createFdPressureState,
+  evaluateFdPressure,
+  parseOpenFileLimits,
+  type ProcessFdPressure,
+  type ProcessFdUsage,
+} from "../file-descriptors.js";
+
+const LIMITS = `Limit                     Soft Limit           Hard Limit           Units
+Max cpu time              unlimited            unlimited            seconds
+Max file size             unlimited            unlimited            bytes
+Max open files            1024                 1048576              files
+Max locked memory         8388608              8388608              bytes
+`;
+
+/** A process whose soft limit is known, so it carries a comparable ratio. */
+function usage(fields: {
+  pid?: number;
+  command?: string;
+  openCount: number;
+  softLimit?: number;
+}): ProcessFdPressure {
+  const {
+    pid = 1,
+    command = "daemon-main",
+    openCount,
+    softLimit = 1024,
+  } = fields;
+  return {
+    pid,
+    command,
+    openCount,
+    softLimit,
+    hardLimit: 1_048_576,
+    ratio: openCount / softLimit,
+  };
+}
+
+describe("parseOpenFileLimits", () => {
+  test("reads the soft and hard open-file limits", () => {
+    expect(parseOpenFileLimits(LIMITS)).toEqual({
+      soft: 1024,
+      hard: 1_048_576,
+    });
+  });
+
+  test("reports null for an unlimited descriptor limit", () => {
+    expect(
+      parseOpenFileLimits(
+        "Max open files            unlimited  unlimited  files\n",
+      ),
+    ).toEqual({ soft: null, hard: null });
+  });
+
+  test("reports null when the row is absent or the file is empty", () => {
+    expect(
+      parseOpenFileLimits("Max cpu time  unlimited  unlimited  seconds\n"),
+    ).toEqual({ soft: null, hard: null });
+    expect(parseOpenFileLimits("")).toEqual({ soft: null, hard: null });
+  });
+});
+
+describe("evaluateFdPressure", () => {
+  const options = { thresholdRatio: 0.8, warnCooldownMs: 60_000 };
+
+  test("stays quiet while every process is under the threshold", () => {
+    const result = evaluateFdPressure(
+      [usage({ openCount: 700 }), usage({ pid: 2, openCount: 100 })],
+      options,
+      createFdPressureState(),
+      1_000,
+    );
+    expect(result.action).toEqual({ kind: "none" });
+    expect(result.state).toEqual({ lastWarnAt: 0, overThreshold: false });
+  });
+
+  test("warns on the crossing, naming the over-threshold processes", () => {
+    const hot = usage({ pid: 7, openCount: 900 });
+    const result = evaluateFdPressure(
+      [hot, usage({ pid: 2, openCount: 10 })],
+      options,
+      createFdPressureState(),
+      5_000,
+    );
+    expect(result.action).toEqual({ kind: "warn", processes: [hot] });
+    expect(result.state).toEqual({ lastWarnAt: 5_000, overThreshold: true });
+  });
+
+  test("throttles repeats to one warn per cooldown while usage stays over", () => {
+    const hot = usage({ openCount: 1000 });
+    const warned = { lastWarnAt: 5_000, overThreshold: true };
+
+    const cooling = evaluateFdPressure([hot], options, warned, 30_000);
+    expect(cooling.action).toEqual({ kind: "none" });
+    // The warn timestamp is preserved so the cooldown measures from the warn.
+    expect(cooling.state).toEqual(warned);
+
+    const reWarn = evaluateFdPressure([hot], options, warned, 65_000);
+    expect(reWarn.action).toEqual({ kind: "warn", processes: [hot] });
+    expect(reWarn.state).toEqual({ lastWarnAt: 65_000, overThreshold: true });
+  });
+
+  test("reports recovery once, then goes quiet", () => {
+    const cool = usage({ openCount: 10 });
+    const recovered = evaluateFdPressure(
+      [cool],
+      options,
+      { lastWarnAt: 5_000, overThreshold: true },
+      70_000,
+    );
+    expect(recovered.action).toEqual({ kind: "recovered" });
+    expect(recovered.state).toEqual({ lastWarnAt: 0, overThreshold: false });
+
+    expect(
+      evaluateFdPressure([cool], options, recovered.state, 80_000).action,
+    ).toEqual({ kind: "none" });
+  });
+
+  test("ignores processes whose soft limit is unknown", () => {
+    const unlimited: ProcessFdUsage = {
+      pid: 3,
+      command: "vellum-qdrant",
+      openCount: 100_000,
+      softLimit: null,
+      hardLimit: null,
+      ratio: null,
+    };
+    const result = evaluateFdPressure(
+      [unlimited],
+      options,
+      createFdPressureState(),
+      1_000,
+    );
+    expect(result.action).toEqual({ kind: "none" });
+  });
+});

package/src/monitoring/file-descriptors.ts

@@ -0,0 +1,262 @@
+/**
+ * Open file-descriptor accounting for the container's processes, polled by the
+ * resource monitor.
+ *
+ * Descriptor exhaustion is the memory limit's quieter twin: a process that
+ * reaches its `RLIMIT_NOFILE` soft limit starts failing every `open()`,
+ * `accept()`, and `socket()` with EMFILE, so a leak surfaces as unrelated
+ * connection and file errors scattered across the daemon rather than as a
+ * single recognizable failure. The limit is per process (unlike the cgroup
+ * memory limit, which is shared), so usage is measured per PID: the open count
+ * from `/proc/<pid>/fd` against the soft limit from `/proc/<pid>/limits`.
+ *
+ * Enumerating descriptors costs a readdir plus a small read per process, which
+ * is why this polls on its own slow timer instead of riding the 250ms resource
+ * sampler. Descriptor counts climb over minutes, not milliseconds.
+ *
+ * Reads are best-effort: a process that exits mid-walk, or one owned by another
+ * user, is skipped rather than failing the pass.
+ */
+
+import { readdirSync, readFileSync } from "node:fs";
+
+import type { MonitoringConfig } from "../config/schemas/monitoring.js";
+import { getLogger } from "../util/logger.js";
+import { readProcessCommand } from "../util/process-tree.js";
+
+const log = getLogger("file-descriptors");
+
+/** How many over-threshold processes a single warn line reports. */
+const WARN_PROCESS_LIMIT = 5;
+
+export interface OpenFileLimits {
+  /** The limit actually enforced on `open()`; null when unlimited or absent. */
+  soft: number | null;
+  /** Ceiling the process may raise its soft limit to; null when unlimited. */
+  hard: number | null;
+}
+
+export interface ProcessFdUsage {
+  pid: number;
+  /** Redacted process descriptor (see {@link readProcessCommand}). */
+  command: string;
+  /** Entries in `/proc/<pid>/fd`. */
+  openCount: number;
+  softLimit: number | null;
+  hardLimit: number | null;
+  /** openCount / softLimit, or null when the soft limit is unknown. */
+  ratio: number | null;
+}
+
+/** A process whose soft limit is known, so its usage ratio is comparable. */
+export type ProcessFdPressure = ProcessFdUsage & { ratio: number };
+
+/** `Max open files  <soft>  <hard>  files` in `/proc/<pid>/limits`. */
+const MAX_OPEN_FILES_RE = /^Max open files\s+(\S+)\s+(\S+)/m;
+
+/**
+ * Parse the descriptor limits out of `/proc/<pid>/limits`. Both fields are null
+ * when the row is missing, and an individual field is null when the kernel
+ * reports `unlimited`.
+ */
+export function parseOpenFileLimits(raw: string): OpenFileLimits {
+  const match = MAX_OPEN_FILES_RE.exec(raw);
+  if (!match) {
+    return { soft: null, hard: null };
+  }
+  const count = (field: string) => {
+    const parsed = parseInt(field, 10);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+  };
+  return { soft: count(match[1]), hard: count(match[2]) };
+}
+
+/**
+ * Descriptor usage for one PID, or null when the process is gone or its `/proc`
+ * entry is unreadable. The count includes the descriptor `readdirSync` itself
+ * holds when reading this process's own `/proc/self/fd`, so a process's
+ * self-measurement runs one high.
+ */
+function readProcessFdUsage(pid: number): ProcessFdUsage | null {
+  let openCount: number;
+  try {
+    openCount = readdirSync(`/proc/${pid}/fd`).length;
+  } catch {
+    // Exited between readdir and read, or owned by another user.
+    return null;
+  }
+  const command = readProcessCommand(pid);
+  if (command == null) {
+    return null;
+  }
+  let limits: OpenFileLimits = { soft: null, hard: null };
+  try {
+    limits = parseOpenFileLimits(readFileSync(`/proc/${pid}/limits`, "utf-8"));
+  } catch {
+    // Limits unreadable; the raw count is still worth reporting.
+  }
+  return {
+    pid,
+    command,
+    openCount,
+    softLimit: limits.soft,
+    hardLimit: limits.hard,
+    ratio: limits.soft != null ? openCount / limits.soft : null,
+  };
+}
+
+/**
+ * Descriptor usage for every readable process, closest to its soft limit first
+ * (processes with an unknown limit sort last, by raw count). Empty when
+ * `/proc` is unavailable, e.g. on macOS.
+ */
+export function collectFdUsage(): ProcessFdUsage[] {
+  let entries: string[];
+  try {
+    entries = readdirSync("/proc").filter((entry) => /^\d+$/.test(entry));
+  } catch {
+    return [];
+  }
+
+  const rows: ProcessFdUsage[] = [];
+  for (const entry of entries) {
+    const usage = readProcessFdUsage(Number(entry));
+    if (usage != null) {
+      rows.push(usage);
+    }
+  }
+
+  rows.sort(
+    (a, b) => (b.ratio ?? -1) - (a.ratio ?? -1) || b.openCount - a.openCount,
+  );
+  return rows;
+}
+
+/** The top `limit` processes by descriptor pressure, most pressured first. */
+export function topProcessesByFd(limit: number): ProcessFdUsage[] {
+  return collectFdUsage().slice(0, limit);
+}
+
+/** What a pass concluded about the container's descriptor pressure. */
+export type FdPressureAction =
+  | { kind: "none" }
+  | { kind: "warn"; processes: ProcessFdPressure[] }
+  | { kind: "recovered" };
+
+/** Pass-to-pass state, so a sustained condition warns on a cooldown. */
+export interface FdPressureState {
+  /** When the last warn was emitted; 0 when none has been. */
+  lastWarnAt: number;
+  /** Whether the previous pass found any process over the threshold. */
+  overThreshold: boolean;
+}
+
+export function createFdPressureState(): FdPressureState {
+  return { lastWarnAt: 0, overThreshold: false };
+}
+
+/**
+ * Decide what a pass should report, given the pressure it measured and the
+ * previous pass's state. Crossing the threshold warns immediately; staying over
+ * it re-warns at most once per `warnCooldownMs`, so a leak logs a rising count
+ * instead of a line per poll. Dropping back under reports a single recovery.
+ */
+export function evaluateFdPressure(
+  usage: readonly ProcessFdUsage[],
+  options: { thresholdRatio: number; warnCooldownMs: number },
+  state: FdPressureState,
+  now: number,
+): { action: FdPressureAction; state: FdPressureState } {
+  const over = usage.filter(
+    (p): p is ProcessFdPressure =>
+      p.ratio != null && p.ratio >= options.thresholdRatio,
+  );
+
+  if (over.length === 0) {
+    return {
+      action: state.overThreshold ? { kind: "recovered" } : { kind: "none" },
+      state: createFdPressureState(),
+    };
+  }
+
+  if (state.overThreshold && now - state.lastWarnAt < options.warnCooldownMs) {
+    return { action: { kind: "none" }, state };
+  }
+
+  return {
+    action: { kind: "warn", processes: over },
+    state: { lastWarnAt: now, overThreshold: true },
+  };
+}
+
+function logFdPressure(action: FdPressureAction, thresholdRatio: number): void {
+  if (action.kind === "warn") {
+    log.warn(
+      {
+        thresholdRatio,
+        processCount: action.processes.length,
+        // PID only, no command: a command line can carry secrets passed as
+        // arguments, and this line lands in the rotating workspace log.
+        processes: action.processes.slice(0, WARN_PROCESS_LIMIT).map((p) => ({
+          pid: p.pid,
+          openCount: p.openCount,
+          softLimit: p.softLimit,
+          hardLimit: p.hardLimit,
+          ratio: Math.round(p.ratio * 1000) / 1000,
+        })),
+      },
+      "Processes near their open file-descriptor limit",
+    );
+    return;
+  }
+  if (action.kind === "recovered") {
+    log.info(
+      { thresholdRatio },
+      "Open file-descriptor usage back under threshold",
+    );
+  }
+}
+
+export interface FileDescriptorMonitorHandle {
+  stop: () => void;
+}
+
+/**
+ * Start the descriptor poller. The first pass runs one interval in, not at
+ * boot: descriptor counts are still ramping while the daemon opens its sockets
+ * and database handles, and the walk is I/O the boot path does not need.
+ *
+ * The timer is unref'd: the resource sampler is the monitor process's
+ * keep-alive, and this loop must not extend its life on its own.
+ */
+export function startFileDescriptorMonitor(
+  config: MonitoringConfig,
+  clock: () => number = Date.now,
+): FileDescriptorMonitorHandle {
+  let state = createFdPressureState();
+
+  const poll = () => {
+    try {
+      const result = evaluateFdPressure(
+        collectFdUsage(),
+        {
+          thresholdRatio: config.highFdThresholdRatio,
+          warnCooldownMs: config.fdWarnCooldownMs,
+        },
+        state,
+        clock(),
+      );
+      state = result.state;
+      logFdPressure(result.action, config.highFdThresholdRatio);
+    } catch (err) {
+      log.warn({ err }, "File-descriptor poll failed");
+    }
+  };
+
+  const timer = setInterval(poll, config.fdPollIntervalMs);
+  timer.unref?.();
+
+  return {
+    stop: () => clearInterval(timer),
+  };
+}

package/src/monitoring/process-memory.ts

@@ -11,7 +11,7 @@
 
 import { readdirSync, readFileSync } from "node:fs";
 
-import { deriveName } from "../util/process-tree.js";
+import { readProcessCommand } from "../util/process-tree.js";
 
 /** Linux page size assumed when converting `/proc/<pid>/statm` pages to bytes. */
 const PAGE_SIZE_BYTES = 4096;
@@ -101,16 +101,9 @@ export function topProcessesByMemory(limit: number): ProcessMemory[] {
     if (mem == null) {
       continue;
     }
-    // Redact the raw command line via deriveName to strip secrets (tokens,
-    // API keys, database URLs) commonly passed as process arguments. The
-    // raw command line is read here but never stored.
-    let command: string;
-    try {
-      const raw = readFileSync(`/proc/${pid}/cmdline`, "utf-8");
-      command =
-        deriveName(raw.split("\0").filter(Boolean).join(" ")) || `pid ${pid}`;
-    } catch {
-      // Process exited between readdir and read — skip.
+    const command = readProcessCommand(pid);
+    if (command == null) {
+      // Process exited between readdir and read; skip it.
       continue;
     }
     rows.push({ pid, command, ...mem });

package/src/monitoring/resource-sampler.ts

@@ -36,6 +36,7 @@ import { getLogger } from "../util/logger.js";
 import { getMonitoringDataDir } from "../util/platform.js";
 import { buildProcessTree, listProcesses } from "../util/process-tree.js";
 import { readActiveConversations } from "./active-conversations.js";
+import { topProcessesByFd } from "./file-descriptors.js";
 import { getTrackedDataFiles, readFileResidency } from "./page-cache.js";
 import { topProcessesByMemory } from "./process-memory.js";
 import { prunePrefixedJsonFiles } from "./prune-snapshots.js";
@@ -176,6 +177,10 @@ export async function writeSnapshot(
     // Slab memory belongs to no process; without this, cgroup usage that
     // exceeds the per-process sum has no visible owner.
     topSlabCaches: topSlabCaches(10),
+    // Descriptor pressure, ranked against each process's own soft limit. An
+    // EMFILE storm and a memory spike look alike from the outside (unrelated
+    // failures across the daemon), so the capture records both.
+    topProcessesByFd: topProcessesByFd(15),
     processTree: tree,
   };