@vellumai/assistant 0.5.16 → 0.6.1

package/src/mcp/mcp-oauth-provider.ts

@@ -2,9 +2,21 @@
  * OAuthClientProvider implementation for MCP servers.
  *
  * Uses secure-keys (credential store) for persistent credential storage
- * and a loopback HTTP server for the browser callback.
+ * and either a loopback HTTP server or the gateway callback registry
+ * for the browser callback.
+ *
+ * Two callback transports:
+ *
+ * 1. **Loopback** (default) — starts a temporary HTTP server on localhost.
+ *    Works when the daemon runs on the user's machine (desktop app).
+ *
+ * 2. **Gateway** — routes callbacks through the platform's public ingress
+ *    URL and the in-memory oauth-callback-registry. Used when the daemon
+ *    runs inside Docker/platform where localhost is unreachable from the
+ *    user's browser.
  */
 
+import { randomBytes } from "node:crypto";
 import { createServer, type Server } from "node:http";
 
 import type {
@@ -22,8 +34,8 @@ import {
   getSecureKeyAsync,
   setSecureKeyAsync,
 } from "../security/secure-keys.js";
+import { openInHostBrowser } from "../util/browser.js";
 import { getLogger } from "../util/logger.js";
-import { isLinux, isMacOS } from "../util/platform.js";
 
 const log = getLogger("mcp-oauth");
 
@@ -46,24 +58,41 @@ export interface McpOAuthCallbackResult {
   codePromise: Promise<string>;
 }
 
+/** Which callback transport to use for receiving the OAuth redirect. */
+export type McpOAuthCallbackTransport = "loopback" | "gateway";
+
 export class McpOAuthProvider implements OAuthClientProvider {
   private readonly serverId: string;
   private readonly serverUrl: string;
   private readonly interactive: boolean;
+  private readonly callbackTransport: McpOAuthCallbackTransport;
   private _codeVerifier: string | undefined;
+  private _state: string | undefined;
   private _redirectUrl: string | undefined;
   private _codePromise: Promise<string> | null = null;
   private callbackServer: Server | null = null;
   private callbackTimeout: ReturnType<typeof setTimeout> | null = null;
+  /** Deferred resolver/rejector for the gateway code promise. */
+  private _gatewayCodeResolve: ((code: string) => void) | undefined;
+  private _gatewayCodeReject: ((err: Error) => void) | undefined;
 
   /**
    * @param interactive When true (e.g. `mcp auth` CLI), opens browser for OAuth.
    *                    When false (daemon), logs a message instead.
+   * @param callbackTransport Which transport to use for the OAuth redirect.
+   *   - `"loopback"` (default): localhost HTTP server — for desktop clients.
+   *   - `"gateway"`: platform ingress + callback registry — for Docker/platform.
    */
-  constructor(serverId: string, serverUrl: string, interactive = false) {
+  constructor(
+    serverId: string,
+    serverUrl: string,
+    interactive = false,
+    callbackTransport: McpOAuthCallbackTransport = "loopback",
+  ) {
     this.serverId = serverId;
     this.serverUrl = serverUrl;
     this.interactive = interactive;
+    this.callbackTransport = callbackTransport;
   }
 
   // --- redirectUrl ---
@@ -161,6 +190,26 @@ export class McpOAuthProvider implements OAuthClientProvider {
     return this._codeVerifier;
   }
 
+  // --- State (CSRF token for OAuth) ---
+
+  /**
+   * Return a `state` value for the authorization URL.
+   *
+   * The MCP SDK calls `provider.state?.()` and, when the return value is
+   * truthy, appends it as the `state` query parameter.  For the **gateway**
+   * transport the state is mandatory because it is the key used by the
+   * `oauth-callback-registry` to route the redirect back to this flow.
+   * For the **loopback** transport the state is optional (the loopback
+   * server matches on the callback URL itself), but we generate one anyway
+   * for defense-in-depth.
+   */
+  async state(): Promise<string> {
+    if (!this._state) {
+      this._state = randomBytes(16).toString("hex");
+    }
+    return this._state;
+  }
+
   // --- Discovery State ---
 
   async discoveryState(): Promise<OAuthDiscoveryState | undefined> {
@@ -191,6 +240,35 @@ export class McpOAuthProvider implements OAuthClientProvider {
   async redirectToAuthorization(authorizationUrl: URL): Promise<void> {
     const url = authorizationUrl.toString();
 
+    // For gateway transport, extract the SDK-generated `state` from the
+    // authorization URL and register it with the callback registry now.
+    if (
+      this.callbackTransport === "gateway" &&
+      this._gatewayCodeResolve &&
+      this._gatewayCodeReject
+    ) {
+      const sdkState = authorizationUrl.searchParams.get("state");
+      if (sdkState) {
+        // Dynamic import to avoid circular deps
+        const { registerPendingCallback } =
+          await import("../security/oauth-callback-registry.js");
+        registerPendingCallback(
+          sdkState,
+          this._gatewayCodeResolve,
+          this._gatewayCodeReject,
+        );
+        log.info(
+          { serverId: this.serverId, state: sdkState },
+          "MCP OAuth gateway callback registered with SDK state",
+        );
+      } else {
+        log.warn(
+          { serverId: this.serverId },
+          "Authorization URL missing state parameter — gateway callback may not resolve",
+        );
+      }
+    }
+
     if (!this.interactive) {
       // Daemon mode — don't open browser, just log guidance
       log.info(
@@ -208,28 +286,8 @@ export class McpOAuthProvider implements OAuthClientProvider {
       `[MCP] Opening browser for OAuth authorization of "${this.serverId}"...`,
     );
 
-    try {
-      const { execFile } = await import("node:child_process");
-      const onError = (err: Error | null) => {
-        if (err) {
-          log.warn({ err }, "Failed to open browser");
-          console.log(`[MCP] Please open this URL in your browser:\n${url}`);
-        }
-      };
-      if (isMacOS()) {
-        execFile("open", [url], onError);
-      } else if (isLinux()) {
-        execFile("xdg-open", [url], onError);
-      } else {
-        log.warn(
-          "Unsupported platform for browser open — please visit the URL manually",
-        );
-        console.log(`[MCP] Please open this URL in your browser:\n${url}`);
-      }
-    } catch (err) {
-      log.warn({ err }, "Failed to open browser");
-      console.log(`[MCP] Please open this URL in your browser:\n${url}`);
-    }
+    await openInHostBrowser(url);
+    console.log(`[MCP] If the browser did not open, visit this URL:\n${url}`);
   }
 
   // --- Invalidate Credentials ---
@@ -272,6 +330,7 @@ export class McpOAuthProvider implements OAuthClientProvider {
     }
     if (scope === "all" || scope === "verifier") {
       this._codeVerifier = undefined;
+      this._state = undefined;
     }
     if (scope === "all" || scope === "discovery") {
       const result = await deleteSecureKeyAsync(discoveryKey(this.serverId));
@@ -292,10 +351,64 @@ export class McpOAuthProvider implements OAuthClientProvider {
   // --- Callback Server ---
 
   /**
-   * Start a loopback HTTP server to receive the OAuth callback.
-   * Returns a promise that resolves with the authorization code.
+   * Start listening for the OAuth callback.
+   *
+   * - **Loopback transport**: starts a temporary HTTP server on localhost.
+   * - **Gateway transport**: registers a pending callback with the
+   *   oauth-callback-registry and resolves a public redirect URL via
+   *   the platform callback registration system.
+   *
+   * Returns a promise that resolves with the authorization code promise.
    */
   startCallbackServer(): Promise<McpOAuthCallbackResult> {
+    if (this.callbackTransport === "gateway") {
+      return this.startGatewayCallback();
+    }
+    return this.startLoopbackServer();
+  }
+
+  /**
+   * Gateway transport: resolve the public redirect URL and create a
+   * deferred code promise.  The actual `registerPendingCallback` call
+   * is deferred until `redirectToAuthorization` where we can extract
+   * the SDK-generated `state` parameter from the authorization URL.
+   */
+  private async startGatewayCallback(): Promise<McpOAuthCallbackResult> {
+    const { resolveCallbackUrl } =
+      await import("../inbound/platform-callback-registration.js");
+    const { getOAuthCallbackUrl } =
+      await import("../inbound/public-ingress-urls.js");
+    const { loadConfig } = await import("../config/loader.js");
+
+    const appConfig = loadConfig();
+    const redirectUrl = await resolveCallbackUrl(
+      () => getOAuthCallbackUrl(appConfig),
+      "webhooks/oauth/callback",
+      "mcp_oauth",
+    );
+
+    this._redirectUrl = redirectUrl;
+
+    // Create a deferred promise — it will be wired to the callback
+    // registry in redirectToAuthorization() once we know the SDK's state.
+    const codePromise = new Promise<string>((resolve, reject) => {
+      this._gatewayCodeResolve = resolve;
+      this._gatewayCodeReject = reject;
+    });
+    this._codePromise = codePromise;
+
+    log.info(
+      { serverId: this.serverId, redirectUrl },
+      "MCP OAuth gateway callback prepared (awaiting state from auth URL)",
+    );
+
+    return { codePromise };
+  }
+
+  /**
+   * Loopback transport: start a temporary HTTP server on localhost.
+   */
+  private startLoopbackServer(): Promise<McpOAuthCallbackResult> {
     return new Promise((resolveSetup, rejectSetup) => {
       let settled = false;
       let listening = false;
@@ -423,6 +536,15 @@ export class McpOAuthProvider implements OAuthClientProvider {
       this.callbackServer.close();
       this.callbackServer = null;
     }
+    // Gateway transport cleanup — reject the deferred promise so callers
+    // awaiting codePromise don't hang indefinitely.
+    if (this._gatewayCodeReject) {
+      this._gatewayCodeReject(
+        new Error("MCP OAuth gateway callback cancelled"),
+      );
+      this._gatewayCodeResolve = undefined;
+      this._gatewayCodeReject = undefined;
+    }
   }
 }
 

package/src/memory/admin.ts

@@ -6,16 +6,21 @@ import { deleteMemoryCheckpoint } from "./checkpoints.js";
 import { getConversationMemoryScopeId } from "./conversation-crud.js";
 import { getDb, rawGet } from "./db.js";
 import { getMemoryBackendStatus } from "./embedding-backend.js";
-import { enqueueBackfillJob, enqueueRebuildIndexJob, MIN_SEGMENT_CHARS } from "./indexer.js";
+import { handleRecall, type RecallResult } from "./graph/tool-handlers.js";
 import {
-  enqueueCleanupStaleSupersededItemsJob,
-  enqueueMemoryJob,
-  getMemoryJobCounts,
-} from "./jobs-store.js";
+  enqueueBackfillJob,
+  enqueueRebuildIndexJob,
+  MIN_SEGMENT_CHARS,
+} from "./indexer.js";
+import { enqueueMemoryJob, getMemoryJobCounts } from "./jobs-store.js";
 import { withQdrantBreaker } from "./qdrant-circuit-breaker.js";
 import { getQdrantClient } from "./qdrant-client.js";
-import { queryMemoryForCli } from "./retriever.js";
-import { conversations, memorySegments, memorySummaries, messages } from "./schema.js";
+import {
+  conversations,
+  memorySegments,
+  memorySummaries,
+  messages,
+} from "./schema.js";
 
 const log = getLogger("memory-admin");
 
@@ -27,47 +32,22 @@ export interface MemorySystemStatus {
   model: string | null;
   counts: {
     segments: number;
-    items: number;
+    graphNodes: number;
     summaries: number;
     embeddings: number;
   };
-  cleanup: {
-    supersededBacklog: number;
-    supersededCompleted24h: number;
-  };
   jobs: Record<string, number>;
 }
 
-interface CleanupStatsRow {
-  superseded_backlog: number | null;
-  superseded_completed_24h: number | null;
-}
-
 export async function getMemorySystemStatus(): Promise<MemorySystemStatus> {
   const config = getConfig();
   const backend = await getMemoryBackendStatus(config);
   const counts = {
     segments: countTable("memory_segments"),
-    items: countTable("memory_items"),
+    graphNodes: countTable("memory_graph_nodes"),
     summaries: countTable("memory_summaries"),
     embeddings: countTable("memory_embeddings"),
   };
-  const throughputWindowStartMs = Date.now() - 24 * 60 * 60 * 1000;
-  const cleanupStats = rawGet<CleanupStatsRow>(
-    `
-    SELECT
-      SUM(CASE
-        WHEN type = 'cleanup_stale_superseded_items' AND status IN ('pending', 'running')
-        THEN 1 ELSE 0 END
-      ) AS superseded_backlog,
-      SUM(CASE
-        WHEN type = 'cleanup_stale_superseded_items' AND status = 'completed' AND updated_at >= ?
-        THEN 1 ELSE 0 END
-      ) AS superseded_completed_24h
-    FROM memory_jobs
-  `,
-    throughputWindowStartMs,
-  );
   return {
     enabled: backend.enabled,
     degraded: backend.degraded,
@@ -75,10 +55,6 @@ export async function getMemorySystemStatus(): Promise<MemorySystemStatus> {
     provider: backend.provider,
     model: backend.model,
     counts,
-    cleanup: {
-      supersededBacklog: cleanupStats?.superseded_backlog ?? 0,
-      supersededCompleted24h: cleanupStats?.superseded_completed_24h ?? 0,
-    },
     jobs: getMemoryJobCounts(),
   };
 
@@ -99,20 +75,12 @@ export function requestMemoryRebuildIndex(): string {
   return id;
 }
 
-export function requestMemoryCleanup(retentionMs?: number): {
-  staleSupersededItemsJobId: string;
-} {
-  const staleSupersededItemsJobId =
-    enqueueCleanupStaleSupersededItemsJob(retentionMs);
-  log.info(
-    { staleSupersededItemsJobId, retentionMs },
-    "Queued memory cleanup jobs",
-  );
-  return { staleSupersededItemsJobId };
-}
-
-export async function queryMemory(query: string, conversationId: string) {
-  return queryMemoryForCli(query, conversationId, getConfig());
+export async function queryMemory(
+  query: string,
+  _conversationId: string,
+): Promise<RecallResult> {
+  const config = getConfig();
+  return handleRecall({ query }, config, "default");
 }
 
 // ── Short segment cleanup ─────────────────────────────────────────────
@@ -128,9 +96,9 @@ export interface CleanupShortSegmentsResult {
  * These short fragments waste embedding budget, retrieval slots, and
  * injection tokens.
  */
-export async function cleanupShortSegments(
-  opts?: { dryRun?: boolean },
-): Promise<CleanupShortSegmentsResult> {
+export async function cleanupShortSegments(opts?: {
+  dryRun?: boolean;
+}): Promise<CleanupShortSegmentsResult> {
   const db = getDb();
 
   const shortSegments = db
@@ -151,18 +119,22 @@ export async function cleanupShortSegments(
       await withQdrantBreaker(() => qdrant.deleteByTarget("segment", row.id));
     } catch (err) {
       // Keep the SQLite row so the target ID is preserved for retry
-      log.warn({ segmentId: row.id, err }, "Qdrant deletion failed — skipping SQLite deletion to preserve target ID");
+      log.warn(
+        { segmentId: row.id, err },
+        "Qdrant deletion failed — skipping SQLite deletion to preserve target ID",
+      );
       failed++;
       continue;
     }
 
-    db.delete(memorySegments)
-      .where(eq(memorySegments.id, row.id))
-      .run();
+    db.delete(memorySegments).where(eq(memorySegments.id, row.id)).run();
     removed++;
   }
 
-  log.info({ removed, failed, threshold: MIN_SEGMENT_CHARS }, "Cleaned up short segments");
+  log.info(
+    { removed, failed, threshold: MIN_SEGMENT_CHARS },
+    "Cleaned up short segments",
+  );
   return { removed, failed };
 }
 
@@ -194,7 +166,7 @@ export function findReextractTargets(limit: number): ReextractTarget[] {
     .from(conversations)
     .leftJoin(messages, eq(messages.conversationId, conversations.id))
     .where(
-      sql`${conversations.conversationType} NOT IN ('background', 'private')`,
+      sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`,
     )
     .groupBy(conversations.id)
     .orderBy(desc(sql`count(${messages.id})`))
@@ -238,25 +210,21 @@ export function findReextractTarget(
 /**
  * Queue re-extraction for a set of conversations.
  * Resets extraction checkpoints and clears extraction summaries so the
- * batch extraction handler processes all messages from scratch with
+ * graph extraction handler processes all messages from scratch with
  * expanded supersession context.
  */
-export function requestReextract(
-  targets: ReextractTarget[],
-): { jobIds: string[] } {
+export function requestReextract(targets: ReextractTarget[]): {
+  jobIds: string[];
+} {
   const db = getDb();
   const jobIds: string[] = [];
 
   for (const target of targets) {
     const { conversationId } = target;
 
-    // Reset batch extraction checkpoints
-    deleteMemoryCheckpoint(
-      `batch_extract:${conversationId}:last_message_id`,
-    );
-    deleteMemoryCheckpoint(
-      `batch_extract:${conversationId}:pending_count`,
-    );
+    // Reset graph extraction checkpoints
+    deleteMemoryCheckpoint(`graph_extract:${conversationId}:last_ts`);
+    deleteMemoryCheckpoint(`graph_extract:${conversationId}:pending_count`);
 
     // Clear the extraction summary so it starts fresh
     db.delete(memorySummaries)
@@ -268,12 +236,11 @@ export function requestReextract(
       )
       .run();
 
-    // Resolve scope and enqueue with fullReextract flag
+    // Resolve scope and enqueue re-extraction
     const scopeId = getConversationMemoryScopeId(conversationId);
-    const jobId = enqueueMemoryJob("batch_extract", {
+    const jobId = enqueueMemoryJob("graph_extract", {
       conversationId,
       scopeId,
-      fullReextract: true,
     });
     jobIds.push(jobId);
 

package/src/memory/app-store.ts

@@ -216,6 +216,8 @@ export function generateAppDirName(
 
 /** Cache of id -> dirName mappings to avoid repeated filesystem scans. */
 const idToDirNameCache = new Map<string, string>();
+/** Reverse cache: dirName -> id. */
+const dirNameToIdCache = new Map<string, string>();
 
 /**
  * Resolve an app's directory name and path from its ID.
@@ -265,12 +267,79 @@ export function getAppDirPath(appId: string): string {
   return resolveAppDir(appId).appDir;
 }
 
+/**
+ * Resolve an app ID from its directory name (slug).
+ * Checks caches first, then reads the JSON definition file directly.
+ */
+export function resolveAppIdByDirName(dirName: string): string | null {
+  const cached = dirNameToIdCache.get(dirName);
+  if (cached) return cached;
+
+  // Check forward cache (reverse iteration)
+  for (const [id, dn] of idToDirNameCache) {
+    if (dn === dirName) {
+      dirNameToIdCache.set(dirName, id);
+      return id;
+    }
+  }
+
+  // Read the JSON definition file directly
+  const dir = getAppsDir();
+  const jsonPath = join(dir, `${dirName}.json`);
+  if (existsSync(jsonPath)) {
+    try {
+      const raw = readFileSync(jsonPath, "utf-8");
+      const parsed = JSON.parse(raw) as { id?: string; dirName?: string };
+      if (parsed.id) {
+        dirNameToIdCache.set(dirName, parsed.id);
+        idToDirNameCache.set(parsed.id, dirName);
+        return parsed.id;
+      }
+    } catch {
+      // skip malformed files
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Extract app ID from an absolute file path if it falls within the apps
+ * directory and targets a source file (not records/ or dist/).
+ */
+export function resolveAppIdFromPath(filePath: string): string | null {
+  let appsDir: string;
+  try {
+    appsDir = getAppsDir();
+  } catch {
+    return null;
+  }
+  if (!filePath.startsWith(appsDir + "/")) return null;
+
+  const relPath = filePath.slice(appsDir.length + 1);
+  const slashIdx = relPath.indexOf("/");
+  if (slashIdx === -1) return null; // file directly in apps/ (e.g. the .json definition)
+
+  const dirName = relPath.slice(0, slashIdx);
+  const innerPath = relPath.slice(slashIdx + 1);
+
+  // Skip non-source directories
+  if (innerPath.startsWith("records/") || innerPath.startsWith("dist/")) {
+    return null;
+  }
+
+  return resolveAppIdByDirName(dirName);
+}
+
 /** Invalidate the id->dirName cache for a specific app or all apps. */
 function invalidateDirNameCache(appId?: string): void {
   if (appId) {
+    const dirName = idToDirNameCache.get(appId);
     idToDirNameCache.delete(appId);
+    if (dirName) dirNameToIdCache.delete(dirName);
   } else {
     idToDirNameCache.clear();
+    dirNameToIdCache.clear();
   }
 }
 

package/src/memory/conversation-bootstrap.ts

@@ -6,11 +6,12 @@ import {
 } from "./conversation-title-service.js";
 
 export interface BootstrapConversationOptions {
-  conversationType?: "standard" | "private" | "background";
+  conversationType?: "standard" | "private" | "background" | "scheduled";
   source?: string;
   origin: TitleOrigin;
   systemHint: string;
   scheduleJobId?: string;
+  groupId?: string;
 }
 
 export function bootstrapConversation(opts: BootstrapConversationOptions) {
@@ -19,6 +20,7 @@ export function bootstrapConversation(opts: BootstrapConversationOptions) {
     ...(opts.conversationType && { conversationType: opts.conversationType }),
     ...(opts.source && { source: opts.source }),
     ...(opts.scheduleJobId && { scheduleJobId: opts.scheduleJobId }),
+    ...(opts.groupId && { groupId: opts.groupId }),
   });
   queueGenerateConversationTitle({
     conversationId: conversation.id,