@vellumai/assistant 0.10.2-dev.202606250318.5e7cfb0 → 0.10.2

package/src/daemon/workspace-tools-watcher.ts

@@ -0,0 +1,328 @@
+/**
+ * Filesystem watcher for `<workspaceDir>/tools/`.
+ *
+ * Watches the workspace-tools directory non-recursively using fs.watch.
+ * On any add/change/delete event, debounces per filename stem (= tool
+ * name) and reconciles registry state against on-disk state: registers
+ * newly added tools, re-imports changed tools (cache-busting via the
+ * loader's per-import URL query string), unregisters deleted tools,
+ * strips core tools when a `.removed` sentinel appears, restores them
+ * when the sentinel disappears.
+ *
+ * No assistant restart is required — the file watcher closes the
+ * "edit a file, see the change" loop the same way the apps watcher and
+ * plugin source watcher do for their respective directories.
+ *
+ * ## Why per-stem reconciliation
+ *
+ * The watcher receives `(eventType, filename)` from fs.watch but the
+ * eventType ("rename" vs "change") is unreliable across editors and
+ * platforms — vim atomic-save shows as a rename of the original file
+ * plus an add of a new file, VS Code shows as a change in place, etc.
+ * Rather than route on eventType, we debounce per stem and re-derive
+ * the world: "given what's on disk right now for `<stem>.*`, what
+ * registry state should the assistant be in?"
+ *
+ * This is the same eventual-consistency pattern the plugin mtime cache
+ * uses — the watcher exists to KICK the reconciler, not to be the
+ * source of truth about what changed.
+ *
+ * ## Lifecycle position
+ *
+ * Started after the initial `loadWorkspaceTools()` scan completes
+ * during daemon startup, gated on the `workspace-tools-watcher` feature
+ * flag — when the flag is off the initial scan still runs but no watch
+ * loop is mounted, so workspace tools load from disk once and live edits
+ * need a restart. Stopped on assistant shutdown alongside the
+ * other long-lived watchers. Stoppage during shutdown does not
+ * unregister tools — those go away with the process; the watcher's
+ * only job is to keep the registry fresh while the assistant is up.
+ */
+
+import { existsSync, type FSWatcher, watch } from "node:fs";
+
+import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
+import { getConfig } from "../config/loader.js";
+import {
+  getCoreToolOverride,
+  getTool,
+  getToolOwner,
+  removeCoreToolViaWorkspace,
+  restoreStrippedCoreTool,
+  unregisterWorkspaceTool,
+} from "../tools/registry.js";
+import {
+  classifyWorkspaceToolEntry,
+  findWinningWorkspaceToolPath,
+  loadSingleWorkspaceTool,
+} from "../tools/workspace-tools/loader.js";
+import { DebouncerMap } from "../util/debounce.js";
+import { attachFsWatcherErrorHandler } from "../util/fs-watcher-error.js";
+import { getLogger } from "../util/logger.js";
+import { getWorkspaceToolsDir } from "../util/platform.js";
+
+const log = getLogger("workspace-tools-watcher");
+
+/**
+ * Gates the dynamic hot-reload path. When disabled, workspace tools still
+ * load from disk once at daemon startup via {@link loadWorkspaceTools}; only
+ * the live watch → re-registration loop is suppressed. Read at its point of
+ * use in {@link WorkspaceToolsWatcher.start} so a daemon restart picks up a
+ * changed value.
+ */
+const WORKSPACE_TOOLS_WATCHER_FLAG = "workspace-tools-watcher" as const;
+
+/**
+ * Wait this long after the last fs event for a given stem before
+ * reconciling. Editor saves often emit a burst of 2–4 events; the
+ * debounce collapses them into a single load.
+ */
+const RECONCILE_DEBOUNCE_MS = 250;
+
+export class WorkspaceToolsWatcher {
+  /**
+   * Process-wide singleton. Callers reach the watcher via
+   * {@link WorkspaceToolsWatcher.getInstance} rather than instantiating
+   * directly so the daemon `start()`/`stop()` lifecycle and any future
+   * "trigger a manual reconcile" code paths share one watcher across the
+   * assistant process lifetime.
+   */
+  private static singleton: WorkspaceToolsWatcher | null = null;
+
+  static getInstance(): WorkspaceToolsWatcher {
+    WorkspaceToolsWatcher.singleton ??= new WorkspaceToolsWatcher();
+    return WorkspaceToolsWatcher.singleton;
+  }
+
+  /** Test-only — drops the singleton so the next `getInstance()` rebuilds. */
+  static resetForTests(): void {
+    WorkspaceToolsWatcher.singleton?.stop();
+    WorkspaceToolsWatcher.singleton = null;
+  }
+
+  /** Test-only — whether a live `fs.watch` loop is currently mounted. */
+  isWatchingForTests(): boolean {
+    return this.watcher !== null;
+  }
+
+  private watcher: FSWatcher | null = null;
+  private debouncer = new DebouncerMap({
+    defaultDelayMs: RECONCILE_DEBOUNCE_MS,
+    maxEntries: 100,
+  });
+  /**
+   * Promise queue per stem — guarantees that two events for the same
+   * stem can't run concurrently and corrupt the unregister/load
+   * sequence. The queue is single-deep (the chained promise simply
+   * awaits the in-flight one before running its own work), so the
+   * debouncer's collapsing already does most of the deduplication; this
+   * queue exists for the case where a second event lands during the
+   * in-flight reconcile's `await loadSingleWorkspaceTool`.
+   */
+  private inflight = new Map<string, Promise<void>>();
+
+  start(): void {
+    if (this.watcher) return;
+    if (
+      !isAssistantFeatureFlagEnabled(WORKSPACE_TOOLS_WATCHER_FLAG, getConfig())
+    ) {
+      log.debug(
+        "Workspace tools watcher disabled by feature flag; workspace tools load from disk at startup only (restart required to pick up edits)",
+      );
+      return;
+    }
+    const toolsDir = getWorkspaceToolsDir();
+    if (!existsSync(toolsDir)) {
+      log.debug(
+        { toolsDir },
+        "Workspace tools directory does not exist; watcher not started (will not auto-start on directory creation — restart required)",
+      );
+      return;
+    }
+    try {
+      this.watcher = watch(
+        toolsDir,
+        { recursive: false },
+        (_eventType, filename) => {
+          if (!filename) return;
+          const classified = classifyWorkspaceToolEntry(filename);
+          if (!classified) {
+            // Not a workspace-tool file — ignore (README.md, .DS_Store, etc.)
+            return;
+          }
+          this.debouncer.schedule(`stem:${classified.stem}`, () => {
+            this.scheduleReconcile(classified.stem);
+          });
+        },
+      );
+      // Async FSWatcher errors (e.g. ENOSPC, ENXIO) arrive as an 'error' event;
+      // without a listener they crash the daemon. Degrade to a dead watcher.
+      attachFsWatcherErrorHandler(this.watcher, log, toolsDir);
+      log.info({ toolsDir }, "Workspace tools watcher started");
+    } catch (err) {
+      log.warn(
+        { err, toolsDir },
+        "Failed to start workspace tools watcher — workspace tools will only register at startup",
+      );
+    }
+  }
+
+  stop(): void {
+    this.debouncer.cancelAll();
+    if (this.watcher) {
+      this.watcher.close();
+      this.watcher = null;
+    }
+  }
+
+  /**
+   * Chain `reconcileStem(stem)` after any in-flight reconcile for the
+   * same stem so we never run two `loadSingleWorkspaceTool` calls
+   * concurrently for the same name.
+   */
+  private scheduleReconcile(stem: string): void {
+    const prev = this.inflight.get(stem) ?? Promise.resolve();
+    const next = prev
+      .catch(() => {
+        /* swallow — error already logged in the prior tick */
+      })
+      .then(() => this.reconcileStem(stem))
+      .catch((err) => {
+        const message = err instanceof Error ? err.message : String(err);
+        log.error(
+          { err, stem },
+          `workspace-tools-watcher: reconcile for "${stem}" threw: ${message}`,
+        );
+      })
+      .finally(() => {
+        // Only clear if we're still the current in-flight promise.
+        if (this.inflight.get(stem) === next) {
+          this.inflight.delete(stem);
+        }
+      });
+    this.inflight.set(stem, next);
+  }
+
+  /**
+   * Reconcile a single stem (= tool name) with on-disk state.
+   *
+   * Possible (live, removed) tuples and their resolution:
+   *
+   * - `(present, absent)` → ensure workspace tool is registered using
+   *   the winning live file; re-import if a previous registration
+   *   pointed at a different path
+   * - `(absent, present)` → ensure core tool is stripped (and any
+   *   previous workspace registration torn down first)
+   * - `(absent, absent)` → ensure neither stripped state nor live
+   *   registration remains
+   * - `(present, present)` → ambiguous; tear down both so neither
+   *   state survives (matches the initial-scan contract)
+   */
+  private async reconcileStem(stem: string): Promise<void> {
+    const toolsDir = getWorkspaceToolsDir();
+    if (!existsSync(toolsDir)) {
+      this.teardownStem(stem);
+      return;
+    }
+
+    const { livePath, hasRemovedSentinel } = findWinningWorkspaceToolPath(
+      toolsDir,
+      stem,
+    );
+
+    // Ambiguous: tear down everything for this stem.
+    if (hasRemovedSentinel && livePath !== null) {
+      log.error(
+        { stem, toolsDir, livePath },
+        `workspace-tools-watcher: "${stem}" has both a live file and a .removed sentinel — tearing down both`,
+      );
+      this.teardownStem(stem);
+      return;
+    }
+
+    // Neither: tear down anything we own.
+    if (!hasRemovedSentinel && livePath === null) {
+      this.teardownStem(stem);
+      return;
+    }
+
+    if (hasRemovedSentinel) {
+      // Strip the core tool. Unregister any prior workspace registration
+      // first so removeCoreToolViaWorkspace doesn't throw on the
+      // workspace-owned-name check.
+      if (getToolOwner(stem)?.kind === "workspace") {
+        unregisterWorkspaceTool(stem);
+      }
+      try {
+        removeCoreToolViaWorkspace(stem);
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        log.error(
+          { err, stem },
+          `workspace-tools-watcher: failed to strip "${stem}": ${message}`,
+        );
+      }
+      return;
+    }
+
+    if (livePath !== null) {
+      // If we previously stripped this core tool, restore it before the
+      // workspace registration runs so the override path sees the
+      // expected baseline (core tool present → stash + replace).
+      if (getCoreToolOverride(stem) && !getTool(stem)) {
+        restoreStrippedCoreTool(stem);
+      }
+
+      // If a workspace tool is already registered under this name,
+      // unregister it so the loader can re-import cleanly. This covers
+      // file-change events (same path, new contents) as well as
+      // extension-precedence flips (e.g. user added foo.js next to
+      // foo.ts — now .js wins and the registration must update).
+      if (getToolOwner(stem)?.kind === "workspace") {
+        unregisterWorkspaceTool(stem);
+      }
+
+      const registered = await loadSingleWorkspaceTool(livePath);
+      if (registered) {
+        log.info(
+          { stem, livePath },
+          `Workspace tool "${stem}" registered via watcher`,
+        );
+      }
+      // loadSingleWorkspaceTool already logs failures with attribution.
+    }
+  }
+
+  /**
+   * Tear down any workspace-tool state we own for `stem`: unregister a
+   * live workspace tool, restore a core tool we stripped. Both no-ops
+   * when there's nothing to undo.
+   */
+  private teardownStem(stem: string): void {
+    if (getToolOwner(stem)?.kind === "workspace") {
+      unregisterWorkspaceTool(stem);
+      log.info(
+        { stem },
+        `Workspace tool "${stem}" unregistered via watcher (file removed)`,
+      );
+    }
+    if (getCoreToolOverride(stem) && !getTool(stem)) {
+      restoreStrippedCoreTool(stem);
+    }
+  }
+
+  // ── Test affordances ────────────────────────────────────────────────
+  //
+  // These are intentionally narrow — exposed for the watcher tests so
+  // they can drive reconcile() deterministically without waiting on the
+  // debouncer, and inspect whether an in-flight promise is settled.
+  // Not part of the public lifecycle API.
+
+  /**
+   * Run a reconcile for `stem` synchronously (well, asynchronously, but
+   * without going through the debouncer). For tests.
+   */
+  async _testReconcile(stem: string): Promise<void> {
+    await this.reconcileStem(stem);
+  }
+}

package/src/ipc/__tests__/clients-list-ipc.test.ts

@@ -36,7 +36,7 @@ mock.module("../../config/env.js", () => ({
 }));
 
 mock.module("../../runtime/local-actor-identity.js", () => ({
-  findLocalGuardianPrincipalIdFromStore: () => fakeLocalPrincipalId,
+  findLocalGuardianPrincipalId: () => fakeLocalPrincipalId,
 }));
 
 // ── Real imports (after mocks) ────────────────────────────────────────────

package/src/ipc/assistant-server.ts

@@ -33,7 +33,7 @@ import { createServer, type Server, type Socket } from "node:net";
 
 import { ensureSocketDir, SocketWatchdog } from "@vellumai/ipc-server-utils";
 
-import { findLocalGuardianPrincipalIdFromStore } from "../runtime/local-actor-identity.js";
+import { findLocalGuardianPrincipalId } from "../runtime/local-actor-identity.js";
 import { RouteError } from "../runtime/routes/errors.js";
 import { ROUTES } from "../runtime/routes/index.js";
 import type {
@@ -637,7 +637,7 @@ function injectLocalActorHeader(
   // that require the header will fail-closed on their own.
   let localActor: string | undefined;
   try {
-    localActor = findLocalGuardianPrincipalIdFromStore();
+    localActor = findLocalGuardianPrincipalId();
   } catch (err) {
     log.debug(
       { err },

package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts

@@ -68,7 +68,6 @@ mock.module("../../daemon/mcp-reload-service.js", () => ({
 
 mock.module("../../config/env-registry.js", () => ({
   getIsContainerized: () => false,
-  getWorkspaceDirOverride: () => undefined,
 }));
 
 mock.module("../../util/logger.js", () => ({

package/src/mcp/client.ts

@@ -8,7 +8,6 @@ import { getIsPlatform } from "../config/env-registry.js";
 import type { McpTransport } from "../config/schemas/mcp.js";
 import { getSecureKeyAsync } from "../security/secure-keys.js";
 import { getLogger } from "../util/logger.js";
-import { getMcpHeaders } from "./mcp-header-store.js";
 import { McpOAuthProvider } from "./mcp-oauth-provider.js";
 
 const log = getLogger("mcp-client");
@@ -109,21 +108,8 @@ export class McpClient {
       }
     }
 
-    // Resolve static auth headers from credential store, falling back to
-    // any legacy headers in the transport config for backward compatibility.
-    let effectiveConfig = transportConfig;
-    if (isHttpTransport) {
-      const storedHeaders = await getMcpHeaders(this.serverId);
-      if (storedHeaders) {
-        effectiveConfig = {
-          ...transportConfig,
-          headers: { ...transportConfig.headers, ...storedHeaders },
-        } as McpTransport;
-      }
-    }
-
     log.info({ serverId: this.serverId }, "Connecting to MCP server");
-    this.transport = this.createTransport(effectiveConfig);
+    this.transport = this.createTransport(transportConfig);
 
     try {
       await Promise.race([

package/src/mcp/mcp-auth-orchestrator.ts

@@ -21,7 +21,6 @@ import {
   setMcpAuthError,
   setMcpAuthPending,
 } from "./mcp-auth-state.js";
-import { getMcpHeaders } from "./mcp-header-store.js";
 import {
   type McpOAuthCallbackTransport,
   McpOAuthProvider,
@@ -81,10 +80,6 @@ export async function orchestrateMcpOAuthConnect(args: {
   // Register the pending callback in the daemon heap
   const { codePromise } = await provider.startCallbackServer();
 
-  // Resolve effective headers: credential store takes precedence, then config
-  const storedHeaders = await getMcpHeaders(serverId);
-  const effectiveHeaders = storedHeaders ?? transport.headers;
-
   // Build the MCP transport and client
   const serverUrl = new URL(transport.url);
   const TransportClass =
@@ -93,7 +88,7 @@ export async function orchestrateMcpOAuthConnect(args: {
       : StreamableHTTPClientTransport;
   const mcpTransport = new TransportClass(serverUrl, {
     authProvider: provider,
-    requestInit: effectiveHeaders ? { headers: effectiveHeaders } : undefined,
+    requestInit: transport.headers ? { headers: transport.headers } : undefined,
   });
   const client = new Client({ name: "vellum-assistant", version: "1.0.0" });
 

package/src/mcp/mcp-oauth-provider.ts

@@ -569,21 +569,13 @@ export class McpOAuthProvider implements OAuthClientProvider {
 
 // --- Static helpers ---
 
-/**
- * Check whether OAuth tokens exist in the credential store for a server.
- */
-export async function hasMcpOAuthTokens(serverId: string): Promise<boolean> {
-  const raw = await getSecureKeyAsync(tokensKey(serverId));
-  return raw != null && raw.length > 0;
-}
-
 /**
  * Delete all OAuth credentials for a given MCP server.
  * Used by `mcp remove` for cleanup.
  */
 export async function deleteMcpOAuthCredentials(
   serverId: string,
-): Promise<{ ok: boolean; failedKeys: string[] }> {
+): Promise<void> {
   const [tokensResult, clientResult, discoveryResult] = await Promise.all([
     deleteSecureKeyAsync(tokensKey(serverId)),
     deleteSecureKeyAsync(clientInfoKey(serverId)),
@@ -594,23 +586,20 @@ export async function deleteMcpOAuthCredentials(
     { key: "client_info", result: clientResult },
     { key: "discovery", result: discoveryResult },
   ];
-  const failedKeys = results
-    .filter((r) => r.result === "error")
-    .map((r) => r.key);
-  if (failedKeys.length > 0) {
+  const errors = results.filter((r) => r.result === "error").map((r) => r.key);
+  if (errors.length > 0) {
     log.warn(
-      { serverId, failedKeys },
+      { serverId, failedKeys: errors },
       "Some OAuth credentials could not be deleted from secure storage",
     );
   }
-  const ok = failedKeys.length === 0;
+  const hasErrors = errors.length > 0;
   log.info(
     { serverId },
-    ok
-      ? "OAuth credentials deleted"
-      : "OAuth credential deletion completed with errors",
+    hasErrors
+      ? "OAuth credential deletion completed with errors"
+      : "OAuth credentials deleted",
   );
-  return { ok, failedKeys };
 }
 
 // --- HTML rendering ---

package/src/memory/__tests__/memory-retrospective-job.test.ts

@@ -161,14 +161,6 @@ mock.module("../conversation-crud.js", () => ({
       title: "Source conversation",
     };
   },
-  // Mirrors the real `isConversationProcessing` which reads the persisted
-  // `processing_started_at` column. In tests, the mock registry
-  // (`loadedConversations`) is the source of truth — a conversation id
-  // present with `processing: true` simulates a mid-turn conversation.
-  isConversationProcessing: (id: string) => {
-    const entry = loadedConversations[id];
-    return entry?.processing ?? false;
-  },
   forkConversationForRetrospective: async (params: {
     conversationId: string;
     throughMessageId?: string;

package/src/memory/conversation-crud.ts

@@ -25,7 +25,6 @@ import { parseChannelId, parseInterfaceId } from "../channels/types.js";
 import { CHANNEL_IDS, isChannelId } from "../channels/types.js";
 import { getConfig } from "../config/loader.js";
 import { findDisplayTurnEndIndex } from "../conversations/message-consolidation.js";
-import { findConversation } from "../daemon/conversation-registry.js";
 import { conversationMetadataSyncTag } from "../daemon/message-types/sync.js";
 import type { TrustContext } from "../daemon/trust-context.js";
 import { clearAllConversationIds } from "../home/feed-writer.js";
@@ -305,7 +304,6 @@ export interface ConversationRow {
   inferenceProfileSessionId: string | null;
   inferenceProfileExpiresAt: number | null;
   lastNotifiedInferenceProfile: string | null;
-  processingStartedAt: number | null;
 }
 
 export const parseConversation = createRowMapper<
@@ -341,7 +339,6 @@ export const parseConversation = createRowMapper<
   inferenceProfileSessionId: "inferenceProfileSessionId",
   inferenceProfileExpiresAt: "inferenceProfileExpiresAt",
   lastNotifiedInferenceProfile: "lastNotifiedInferenceProfile",
-  processingStartedAt: "processingStartedAt",
 });
 
 /** Allowed values for the `role` column on `messages`. */
@@ -2183,41 +2180,6 @@ export function unarchiveConversation(id: string): boolean {
   return true;
 }
 
-/**
- * Persist the processing-start timestamp for a conversation. Called by
- * `Conversation.setProcessing(true)` so out-of-process callers can detect
- * mid-turn state by reading the `conversations` row directly. Pass `null`
- * to clear (turn ended).
- */
-export function setConversationProcessingStartedAt(
-  id: string,
-  startedAt: number | null,
-): void {
-  rawRun(
-    "UPDATE conversations SET processing_started_at = ? WHERE id = ?",
-    startedAt,
-    id,
-  );
-}
-
-/**
- * Read whether a conversation is currently processing. Checks the in-memory
- * `Conversation._processing` flag first (hot path for resident conversations),
- * falling back to the persisted `processing_started_at` column for cold
- * (evicted / never-loaded) conversations. This is the single entry point for
- * processing state — callers don't need to layer `findConversation` themselves.
- * Returns `false` when the conversation row doesn't exist.
- */
-export function isConversationProcessing(id: string): boolean {
-  const inMemory = findConversation(id)?.isProcessing();
-  if (inMemory != null) return inMemory;
-  const row = rawGet<{ processing_started_at: number | null }>(
-    "SELECT processing_started_at FROM conversations WHERE id = ?",
-    id,
-  );
-  return row?.processing_started_at != null;
-}
-
 /**
  * Set or clear the `surfaced_at` promotion marker for a conversation.
  *

package/src/memory/db-connection.ts

@@ -8,7 +8,6 @@ import { drizzle } from "drizzle-orm/bun-sqlite";
 import { getLogsDbPath } from "../util/logs-db-path.js";
 import { getMemoryDbPath } from "../util/memory-db-path.js";
 import { ensureDataDir, getDbPath } from "../util/platform.js";
-import { getTelemetryDbPath } from "../util/telemetry-db-path.js";
 import { clearStoredDb, getStoredDb, setStoredDb } from "./db-singleton.js";
 import * as schema from "./schema.js";
 
@@ -137,7 +136,7 @@ export function getSqliteFrom(drizzleDb: DrizzleDb): Database {
  * path so this file stays import-light. Callers tolerate a `null` result.
  */
 function openDedicatedDb(
-  key: "logs" | "memory" | "telemetry",
+  key: "logs" | "memory",
   dbPath: string,
 ): DrizzleDb | null {
   assertTestDbIsIsolated();
@@ -193,28 +192,11 @@ export function getMemorySqlite(): Database | null {
   return db ? getSqliteFrom(db) : null;
 }
 
-/**
- * The telemetry database (`assistant-telemetry.db`), opened lazily as its own
- * connection. Houses telemetry event tables (starting with `watchdog_events`).
- * Returns `null` if the file cannot be opened (logged; the daemon stays up).
- */
-export function getTelemetryDb(): DrizzleDb | null {
-  const existing = getStoredDb<DrizzleDb>("telemetry");
-  if (existing) return existing;
-  return openDedicatedDb("telemetry", getTelemetryDbPath());
-}
-
-/** Underlying bun:sqlite Database for the telemetry connection, or `null`. */
-export function getTelemetrySqlite(): Database | null {
-  const db = getTelemetryDb();
-  return db ? getSqliteFrom(db) : null;
-}
-
 /**
  * Reset all DB singletons. Used by production callers that need to close the
  * live connections so the files can be replaced (post-migration, post-restore,
- * post-vbundle-import) and on graceful shutdown. Clears the main, logs, memory,
- * and telemetry slots together so none lingers open against a swapped-out file.
+ * post-vbundle-import) and on graceful shutdown. Clears the main, logs, and
+ * memory slots together so none lingers open against a swapped-out file.
  *
  * Tests should use `resetDbForTesting()` from
  * `__tests__/db-test-helpers.ts` instead so they don't depend on this
@@ -224,5 +206,4 @@ export function resetDb(): void {
   clearStoredDb("main");
   clearStoredDb("logs");
   clearStoredDb("memory");
-  clearStoredDb("telemetry");
 }