@vellumai/assistant 0.6.3 → 0.6.5

package/src/credential-execution/process-manager.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, mock, test } from "bun:test";
+
+import { logCesLine } from "./process-manager.js";
+
+function makeLogger() {
+  return {
+    debug: mock((_obj: object, _msg: string) => {}),
+    info: mock((_obj: object, _msg: string) => {}),
+    warn: mock((_obj: object, _msg: string) => {}),
+    error: mock((_obj: object, _msg: string) => {}),
+  };
+}
+
+describe("logCesLine", () => {
+  test("pino JSON INFO line routes to log.info (not log.error)", () => {
+    const logger = makeLogger();
+    const line = JSON.stringify({
+      level: 30,
+      msg: "CES ready",
+      time: Date.now(),
+    });
+
+    logCesLine(line, 42, logger);
+
+    expect(logger.info).toHaveBeenCalledTimes(1);
+    expect(logger.error).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+
+    const [meta, msg] = logger.info.mock.calls[0] as [object, string];
+    expect(meta).toEqual({ pid: 42 });
+    expect(msg).toBe(`[ces-stderr] ${line}`);
+  });
+
+  test("pino JSON ERROR line routes to log.error", () => {
+    const logger = makeLogger();
+    const line = JSON.stringify({
+      level: 50,
+      msg: "credential store failed",
+      time: Date.now(),
+    });
+
+    logCesLine(line, 42, logger);
+
+    expect(logger.error).toHaveBeenCalledTimes(1);
+    expect(logger.info).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+
+  test("non-JSON fragment like 'args: []' routes to log.info", () => {
+    const logger = makeLogger();
+
+    logCesLine("args: []", 42, logger);
+
+    expect(logger.info).toHaveBeenCalledTimes(1);
+    expect(logger.error).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+
+    const [meta, msg] = logger.info.mock.calls[0] as [object, string];
+    expect(meta).toEqual({ pid: 42 });
+    expect(msg).toBe("[ces-stderr] args: []");
+  });
+
+  test("non-JSON line starting with 'ERROR:' routes to log.error", () => {
+    const logger = makeLogger();
+
+    logCesLine("ERROR: bad thing happened", 42, logger);
+
+    expect(logger.error).toHaveBeenCalledTimes(1);
+    expect(logger.info).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+
+  test("pino-pretty timestamped ERROR line routes to log.error", () => {
+    const logger = makeLogger();
+
+    logCesLine("[12:07:37.467] ERROR oh no", 42, logger);
+
+    expect(logger.error).toHaveBeenCalledTimes(1);
+    expect(logger.info).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+
+  test("pino-pretty timestamped WARN line routes to log.warn", () => {
+    const logger = makeLogger();
+
+    logCesLine("[12:07:37.467] WARN wat", 42, logger);
+
+    expect(logger.warn).toHaveBeenCalledTimes(1);
+    expect(logger.error).not.toHaveBeenCalled();
+    expect(logger.info).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+
+  test("pino-pretty timestamped INFO line routes to log.info", () => {
+    const logger = makeLogger();
+
+    logCesLine("[12:07:37.467] INFO starting", 42, logger);
+
+    expect(logger.info).toHaveBeenCalledTimes(1);
+    expect(logger.error).not.toHaveBeenCalled();
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).not.toHaveBeenCalled();
+  });
+});

package/src/credential-execution/process-manager.ts

@@ -240,7 +240,7 @@ export function createCesProcessManager(
       cmd: [discovery.executablePath],
       stdin: "pipe",
       stdout: "pipe",
-      stderr: "ignore",
+      stderr: "pipe",
       env: {
         ...process.env,
         // Signal to CES that it was launched by the assistant
@@ -251,6 +251,7 @@ export function createCesProcessManager(
     childProcess = proc;
 
     log.info({ pid: proc.pid }, "CES child process started");
+    forwardStderrToLogger(proc);
 
     return createStdioTransport(proc);
   }
@@ -272,7 +273,7 @@ export function createCesProcessManager(
       cmd: [bunPath, "run", discovery.sourcePath],
       stdin: "pipe",
       stdout: "pipe",
-      stderr: "ignore",
+      stderr: "pipe",
       env: {
         ...process.env,
         // Signal to CES that it was launched by the assistant
@@ -283,6 +284,7 @@ export function createCesProcessManager(
     childProcess = proc;
 
     log.info({ pid: proc.pid }, "CES child process started (from source)");
+    forwardStderrToLogger(proc);
 
     return createStdioTransport(proc);
   }
@@ -442,6 +444,98 @@ function createSocketTransport(socket: Socket): CesTransport {
 // Helpers
 // ---------------------------------------------------------------------------
 
+/**
+ * Route a single CES stderr line to the appropriate log level so that
+ * only genuine errors reach the Sentry stream.
+ *
+ * CES is a pino-backed child process that writes INFO/DEBUG/WARN/ERROR
+ * all to stderr (stdout is reserved for the stdio-RPC transport). We
+ * parse the embedded pino level (JSON path) or match a severity prefix
+ * (pretty-printed path) and route each line to the matching log method.
+ *
+ * Exported for testing.
+ */
+export function logCesLine(
+  line: string,
+  pid: number | undefined,
+  logger: {
+    debug: (obj: object, msg: string) => void;
+    info: (obj: object, msg: string) => void;
+    warn: (obj: object, msg: string) => void;
+    error: (obj: object, msg: string) => void;
+  } = log,
+): void {
+  const meta = { pid };
+  const msg = `[ces-stderr] ${line}`;
+
+  // Pino JSON path: parse the line and bucket by numeric `level`.
+  try {
+    const parsed = JSON.parse(line);
+    if (
+      parsed &&
+      typeof parsed === "object" &&
+      typeof (parsed as { level?: unknown }).level === "number"
+    ) {
+      const level = (parsed as { level: number }).level;
+      if (level >= 50) {
+        logger.error(meta, msg);
+      } else if (level >= 40) {
+        logger.warn(meta, msg);
+      } else if (level >= 30) {
+        logger.info(meta, msg);
+      } else {
+        logger.debug(meta, msg);
+      }
+      return;
+    }
+  } catch {
+    // Not JSON — fall through to prefix-based routing.
+  }
+
+  // Pretty-printed / fragment path: look for a level prefix on the line.
+  // Strip an optional pino-pretty-style leading timestamp: "[HH:MM:SS.mmm] ".
+  const prefixStripped = line.replace(
+    /^\[\d{1,2}:\d{2}:\d{2}(?:\.\d{1,3})?\]\s+/,
+    "",
+  );
+  if (/^(FATAL|ERROR)\b/i.test(prefixStripped)) {
+    logger.error(meta, msg);
+  } else if (/^(WARN|WARNING)\b/i.test(prefixStripped)) {
+    logger.warn(meta, msg);
+  } else {
+    logger.info(meta, msg);
+  }
+}
+
+function forwardStderrToLogger(proc: Subprocess): void {
+  if (!proc.stderr || typeof proc.stderr === "number") return;
+
+  const reader = proc.stderr.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+
+  void (async () => {
+    try {
+      while (true) {
+        const { value, done } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        let newlineIdx: number;
+        while ((newlineIdx = buffer.indexOf("\n")) !== -1) {
+          const line = buffer.slice(0, newlineIdx).trimEnd();
+          buffer = buffer.slice(newlineIdx + 1);
+          if (line) logCesLine(line, proc.pid);
+        }
+      }
+      const trailing = buffer.trimEnd();
+      if (trailing) logCesLine(trailing, proc.pid);
+    } catch {
+      // Process ended or stream closed; nothing to forward.
+    }
+  })();
+}
+
 async function stopLocalProcess(proc: Subprocess): Promise<void> {
   log.info({ pid: proc.pid }, "Stopping CES child process");
   proc.kill("SIGTERM");

package/src/credential-health/credential-health-service.ts

@@ -0,0 +1,366 @@
+/**
+ * Proactive credential health monitoring.
+ *
+ * Enumerates all active OAuth connections and validates each one for:
+ * - Token presence in secure storage
+ * - Token expiry (expired or expiring within the warning window)
+ * - Scope coverage (grantedScopes vs provider defaultScopes)
+ * - Liveness ping (for providers with a pingUrl)
+ *
+ * Designed to run during the heartbeat cycle. All checks are diagnostic —
+ * no token refresh or recovery is attempted.
+ */
+
+import {
+  isTokenExpired,
+  oauthConnectionAccessTokenPath,
+} from "@vellumai/credential-storage";
+
+import {
+  getProvider,
+  listActiveConnectionsByProvider,
+  listProviders,
+} from "../oauth/oauth-store.js";
+import { getSecureKeyAsync } from "../security/secure-keys.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("credential-health");
+
+/** 7 days in milliseconds — warn if token expires within this window. */
+const EXPIRY_WARNING_MS = 7 * 24 * 60 * 60 * 1000;
+
+/** Timeout for liveness pings. */
+const PING_TIMEOUT_MS = 5_000;
+
+// ── Types ─────────────────────────────────────────────────────────────
+
+export type CredentialHealthStatus =
+  | "healthy"
+  | "expiring"
+  | "expired"
+  | "missing_token"
+  | "missing_scopes"
+  | "revoked"
+  | "ping_failed";
+
+export interface CredentialHealthResult {
+  connectionId: string;
+  provider: string;
+  accountInfo: string | null;
+  status: CredentialHealthStatus;
+  details: string;
+  missingScopes: string[];
+  canAutoRecover: boolean;
+}
+
+export interface CredentialHealthReport {
+  checkedAt: number;
+  results: CredentialHealthResult[];
+  unhealthy: CredentialHealthResult[];
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────
+
+function safeJsonParse<T>(raw: string | null | undefined, fallback: T): T {
+  if (!raw) return fallback;
+  try {
+    return JSON.parse(raw) as T;
+  } catch {
+    return fallback;
+  }
+}
+
+function scopeDifference(required: string[], granted: string[]): string[] {
+  const grantedSet = new Set(granted);
+  return required.filter((s) => !grantedSet.has(s));
+}
+
+// ── Liveness ping ─────────────────────────────────────────────────────
+
+/** @internal Exposed for test injection. */
+export let _fetchFn: typeof fetch = fetch;
+
+/** @internal Test-only: override the fetch function used for pings. */
+export function _setFetchFn(fn: typeof fetch): void {
+  _fetchFn = fn;
+}
+
+async function pingProvider(
+  token: string,
+  pingUrl: string,
+  pingMethod: string | null,
+  pingHeaders: string | null,
+  pingBody: string | null,
+): Promise<{ ok: boolean; authError: boolean }> {
+  const method = pingMethod ?? "GET";
+  const headers: Record<string, string> = {
+    Authorization: `Bearer ${token}`,
+    ...safeJsonParse<Record<string, string>>(pingHeaders, {}),
+  };
+
+  const body =
+    method !== "GET" && pingBody
+      ? (typeof pingBody === "string" ? pingBody : JSON.stringify(pingBody))
+      : undefined;
+
+  try {
+    const response = await _fetchFn(pingUrl, {
+      method,
+      headers,
+      body,
+      signal: AbortSignal.timeout(PING_TIMEOUT_MS),
+    });
+
+    if (response.ok) return { ok: true, authError: false };
+    if (response.status === 401 || response.status === 403) {
+      return { ok: false, authError: true };
+    }
+    return { ok: false, authError: false };
+  } catch {
+    // Network error or timeout — treat as non-auth failure
+    return { ok: false, authError: false };
+  }
+}
+
+// ── Core check ────────────────────────────────────────────────────────
+
+interface CheckConnectionOpts {
+  connectionId: string;
+  provider: string;
+  accountInfo: string | null;
+  expiresAt: number | null;
+  hasRefreshToken: boolean;
+  grantedScopesRaw: string;
+  defaultScopesRaw: string;
+  pingUrl: string | null;
+  pingMethod: string | null;
+  pingHeaders: string | null;
+  pingBody: string | null;
+}
+
+async function checkConnection(
+  opts: CheckConnectionOpts,
+): Promise<CredentialHealthResult> {
+  const {
+    connectionId,
+    provider,
+    accountInfo,
+    expiresAt,
+    hasRefreshToken,
+    grantedScopesRaw,
+    defaultScopesRaw,
+    pingUrl,
+    pingMethod,
+    pingHeaders,
+    pingBody,
+  } = opts;
+
+  const base = { connectionId, provider, accountInfo, missingScopes: [] as string[] };
+
+  // 1. Check token presence
+  const token = await getSecureKeyAsync(
+    oauthConnectionAccessTokenPath(connectionId),
+  );
+  if (!token) {
+    return {
+      ...base,
+      status: "missing_token",
+      details: `No access token found for ${provider}. Re-authorization required.`,
+      canAutoRecover: false,
+    };
+  }
+
+  // 2. Check token expiry
+  if (isTokenExpired(expiresAt)) {
+    return {
+      ...base,
+      status: hasRefreshToken ? "expiring" : "expired",
+      details: hasRefreshToken
+        ? `Token for ${provider} is expired but has a refresh token — auto-recovery may work.`
+        : `Token for ${provider} is expired with no refresh token. Re-authorization required.`,
+      canAutoRecover: hasRefreshToken,
+    };
+  }
+
+  // Check if expiring within warning window (but not yet expired by the 5-min buffer)
+  if (expiresAt && Date.now() >= expiresAt - EXPIRY_WARNING_MS) {
+    // Token works now but will expire soon
+    if (!hasRefreshToken) {
+      return {
+        ...base,
+        status: "expiring",
+        details: `Token for ${provider} expires within 7 days and has no refresh token. Re-authorization will be needed soon.`,
+        canAutoRecover: false,
+      };
+    }
+    // Has refresh token — not an issue, auto-refresh will handle it
+  }
+
+  // 3. Check scope coverage
+  const grantedScopes = safeJsonParse<string[]>(grantedScopesRaw, []);
+  const defaultScopes = safeJsonParse<string[]>(defaultScopesRaw, []);
+  if (defaultScopes.length > 0 && grantedScopes.length > 0) {
+    const missing = scopeDifference(defaultScopes, grantedScopes);
+    if (missing.length > 0) {
+      return {
+        ...base,
+        status: "missing_scopes",
+        details: `${provider} is missing required scopes: ${missing.join(", ")}. Features may not work correctly.`,
+        missingScopes: missing,
+        canAutoRecover: false,
+      };
+    }
+  }
+
+  // 4. Liveness ping
+  if (pingUrl) {
+    const pingResult = await pingProvider(
+      token,
+      pingUrl,
+      pingMethod,
+      pingHeaders,
+      pingBody,
+    );
+    if (!pingResult.ok) {
+      if (pingResult.authError) {
+        return {
+          ...base,
+          status: "revoked",
+          details: `${provider} token was rejected (401/403). The token may have been revoked. Re-authorization required.`,
+          canAutoRecover: false,
+        };
+      }
+      // Non-auth ping failure — log but don't mark as critical.
+      // Could be a transient API issue.
+      log.debug(
+        { provider, connectionId },
+        "Credential ping failed with non-auth error",
+      );
+      return {
+        ...base,
+        status: "ping_failed",
+        details: `${provider} liveness check failed (non-auth error). This may be transient.`,
+        canAutoRecover: false,
+      };
+    }
+  }
+
+  return {
+    ...base,
+    status: "healthy",
+    details: `${provider} credential is healthy.`,
+    canAutoRecover: hasRefreshToken,
+  };
+}
+
+// ── Public API ────────────────────────────────────────────────────────
+
+/**
+ * Check the health of all active OAuth connections.
+ *
+ * Iterates every registered provider, looks up active connections, and
+ * validates each one. Returns a structured report with overall results
+ * and a filtered list of unhealthy credentials.
+ */
+export async function checkAllCredentials(): Promise<CredentialHealthReport> {
+  const checkedAt = Date.now();
+  const results: CredentialHealthResult[] = [];
+
+  let providers;
+  try {
+    providers = listProviders();
+  } catch (err) {
+    log.warn({ err }, "Failed to list OAuth providers");
+    return { checkedAt, results, unhealthy: [] };
+  }
+
+  for (const providerRow of providers) {
+    let connections;
+    try {
+      connections = listActiveConnectionsByProvider(providerRow.provider);
+    } catch (err) {
+      log.warn(
+        { err, provider: providerRow.provider },
+        "Failed to list connections for provider",
+      );
+      continue;
+    }
+
+    for (const conn of connections) {
+      try {
+        const result = await checkConnection({
+          connectionId: conn.id,
+          provider: conn.provider,
+          accountInfo: conn.accountInfo,
+          expiresAt: conn.expiresAt,
+          hasRefreshToken: !!conn.hasRefreshToken,
+          grantedScopesRaw: conn.grantedScopes,
+          defaultScopesRaw: providerRow.defaultScopes,
+          pingUrl: providerRow.pingUrl,
+          pingMethod: providerRow.pingMethod,
+          pingHeaders: providerRow.pingHeaders,
+          pingBody: providerRow.pingBody,
+        });
+        results.push(result);
+      } catch (err) {
+        log.warn(
+          { err, provider: conn.provider, connectionId: conn.id },
+          "Failed to check credential health",
+        );
+      }
+    }
+  }
+
+  const unhealthy = results.filter((r) => r.status !== "healthy");
+  if (unhealthy.length > 0) {
+    log.info(
+      {
+        total: results.length,
+        unhealthy: unhealthy.length,
+        providers: [...new Set(unhealthy.map((r) => r.provider))],
+      },
+      "Credential health check found issues",
+    );
+  } else {
+    log.debug({ total: results.length }, "All credentials healthy");
+  }
+
+  return { checkedAt, results, unhealthy };
+}
+
+/**
+ * Check credential health for a single provider. Returns the health
+ * result for the most recent active connection, or null if no connection
+ * exists.
+ *
+ * Used by the watcher engine for pre-poll gating.
+ */
+export async function checkCredentialForProvider(
+  provider: string,
+): Promise<CredentialHealthResult | null> {
+  let connections;
+  try {
+    connections = listActiveConnectionsByProvider(provider);
+  } catch {
+    return null;
+  }
+  if (connections.length === 0) return null;
+
+  const conn = connections[0]!;
+  const providerRow = getProvider(conn.provider);
+  if (!providerRow) return null;
+
+  return checkConnection({
+    connectionId: conn.id,
+    provider: conn.provider,
+    accountInfo: conn.accountInfo,
+    expiresAt: conn.expiresAt,
+    hasRefreshToken: !!conn.hasRefreshToken,
+    grantedScopesRaw: conn.grantedScopes,
+    defaultScopesRaw: providerRow.defaultScopes,
+    pingUrl: providerRow.pingUrl,
+    pingMethod: providerRow.pingMethod,
+    pingHeaders: providerRow.pingHeaders,
+    pingBody: providerRow.pingBody,
+  });
+}