@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/workflows/library.ts

@@ -4,9 +4,10 @@
  *
  * Saved workflows live at `<workspace>/workflows/`. The preferred layout is
  * directory-style `<name>/workflow.ts` (so a workflow and its state can sit in
- * one folder); flat `<name>.workflow.ts` is still supported but discouraged. On
- * a base-name collision the directory wins. Each is a normal workflow script: a
- * leading literal
+ * one folder); flat `<name>.workflow.ts` is still supported but discouraged. A
+ * base-name collision or a duplicate `meta.name` FAILS CLOSED — the ambiguous
+ * name resolves to nothing, so a planted file can't shadow a trusted workflow.
+ * Each is a normal workflow script: a leading literal
  * `export const meta = { name, description }` followed by the script body. The
  * library only reads the STATIC `meta` (via {@link extractWorkflowMeta}, a
  * pure-literal extractor that never executes the untrusted source — only the
@@ -22,7 +23,7 @@ import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 
 import { getLogger } from "../util/logger.js";
-import { getWorkspaceDir } from "../util/platform.js";
+import { getWorkspaceWorkflowsDir } from "../util/platform.js";
 import { extractWorkflowMeta, type WorkflowMeta } from "./engine.js";
 
 const log = getLogger("workflow-library");
@@ -48,11 +49,6 @@ export interface SavedWorkflowSource {
   path: string;
 }
 
-/** Absolute path to the saved-workflows directory (`<workspace>/workflows`). */
-function workflowsDir(): string {
-  return join(getWorkspaceDir(), "workflows");
-}
-
 /** Filename of the entry-point script inside a directory-style workflow. */
 const DIRECTORY_ENTRY = "workflow.ts";
 
@@ -69,66 +65,90 @@ interface RawWorkflowFile {
   source: string;
 }
 
+/** Read one saved workflow off disk; `null` (with a warning) if unreadable. */
+function readRawWorkflow(
+  baseName: string,
+  path: string,
+): RawWorkflowFile | null {
+  try {
+    return { baseName, path, source: readFileSync(path, "utf8") };
+  } catch (err) {
+    log.warn({ err, path }, "Failed to read saved workflow; skipping");
+    return null;
+  }
+}
+
+/** Log the fail-closed refusal to resolve a name declared by multiple files. */
+function logDuplicateMetaName(name: string, paths: string[]): void {
+  log.error(
+    { name, paths },
+    `Multiple saved workflows declare meta.name "${name}"; resolving none. ` +
+      `Delete or rename all but one to disambiguate.`,
+  );
+}
+
 /**
- * Yield every readable saved workflow in a STABLE order (the raw filesystem
- * listing order is not guaranteed, so callers that take "the first match" stay
- * deterministic). Directory-style `<name>/workflow.ts` is yielded first and
- * WINS: a flat `<name>.workflow.ts` with the same base name is shadowed (skipped
- * with a warning). Yields nothing if the directory does not exist — it is never
- * created. An unreadable entry is skipped with a logged warning.
+ * Yield every readable saved workflow in a STABLE order (sorted, so "the first
+ * match" is deterministic). Directory-style `<name>/workflow.ts` is yielded
+ * before flat `<name>.workflow.ts`.
+ *
+ * FAIL CLOSED on a base-name collision: if a base name exists as BOTH a
+ * directory and a flat file, neither is yielded — otherwise a planted
+ * `<name>/workflow.ts` could silently shadow a trusted flat workflow.
+ *
+ * Yields nothing if the directory does not exist. An unreadable entry is
+ * skipped with a logged warning.
  */
 function* readWorkflowFiles(): Generator<RawWorkflowFile> {
-  const dir = workflowsDir();
+  const dir = getWorkspaceWorkflowsDir();
   if (!existsSync(dir)) return;
   const entries = readdirSync(dir).sort();
-  const seen = new Set<string>();
 
-  // Directory-style first, so it wins over a same-base-name flat file.
+  // Resolve each form once, keyed by base name (sorted entries keep Map
+  // iteration deterministic).
+  const dirPaths = new Map<string, string>();
+  const flatPaths = new Map<string, string>();
   for (const entry of entries) {
-    if (entry.endsWith(WORKFLOW_SUFFIX)) continue;
-    const path = join(dir, entry, DIRECTORY_ENTRY);
-    if (!existsSync(path)) continue;
-    try {
-      const source = readFileSync(path, "utf8");
-      seen.add(entry);
-      yield { baseName: entry, path, source };
-    } catch (err) {
-      log.warn({ err, path }, "Failed to read saved workflow; skipping");
+    if (entry.endsWith(WORKFLOW_SUFFIX)) {
+      flatPaths.set(fileBaseName(entry), join(dir, entry));
+      continue;
     }
+    const entrypoint = join(dir, entry, DIRECTORY_ENTRY);
+    if (existsSync(entrypoint)) dirPaths.set(entry, entrypoint);
   }
 
-  // Flat files second; skip any shadowed by a directory of the same base name.
-  for (const entry of entries) {
-    if (!entry.endsWith(WORKFLOW_SUFFIX)) continue;
-    const baseName = fileBaseName(entry);
-    const path = join(dir, entry);
-    if (seen.has(baseName)) {
-      log.warn(
-        { path },
-        `Flat workflow "${entry}" is shadowed by directory "${baseName}/"; skipping`,
-      );
-      continue;
-    }
-    try {
-      const source = readFileSync(path, "utf8");
-      seen.add(baseName);
-      yield { baseName, path, source };
-    } catch (err) {
-      log.warn({ err, path }, "Failed to read saved workflow; skipping");
-    }
+  // Fail closed on a base-name collision (both forms present).
+  const collisions = new Set<string>();
+  for (const baseName of dirPaths.keys()) {
+    if (flatPaths.has(baseName)) collisions.add(baseName);
+  }
+  for (const baseName of collisions) {
+    log.error(
+      { dir, baseName },
+      `Saved workflow "${baseName}" exists as BOTH a directory "${baseName}/" and ` +
+        `a flat file "${baseName}${WORKFLOW_SUFFIX}"; resolving neither. Delete one to disambiguate.`,
+    );
+  }
+
+  // Directory-style first, then flat; a colliding base name is skipped in both.
+  for (const [baseName, path] of [...dirPaths, ...flatPaths]) {
+    if (collisions.has(baseName)) continue;
+    const raw = readRawWorkflow(baseName, path);
+    if (raw) yield raw;
   }
 }
 
 /**
- * List every saved workflow with a statically-extractable `meta`, deduplicated
- * by `meta.name` — the first in {@link readWorkflowFiles}' stable order wins, a
- * later duplicate is skipped with a warning (matching {@link getWorkflow}'s
- * winner). A file whose `meta` cannot be statically extracted is skipped with a
- * warning rather than failing the whole listing.
+ * List every saved workflow with a statically-extractable `meta` (a file whose
+ * `meta` can't be extracted is skipped, not fatal).
+ *
+ * FAIL CLOSED on a duplicate `meta.name`: if two files declare the same name,
+ * neither is listed — matching {@link getWorkflow}, so a planted file can't
+ * hijack the name by iteration order.
  */
 export function listWorkflows(): SavedWorkflowEntry[] {
-  const entries: SavedWorkflowEntry[] = [];
-  const seenNames = new Set<string>();
+  // Group by meta.name so a duplicate can drop all sharers (fail closed).
+  const byName = new Map<string, SavedWorkflowEntry[]>();
   for (const { path, source } of readWorkflowFiles()) {
     let meta: WorkflowMeta;
     try {
@@ -140,32 +160,43 @@ export function listWorkflows(): SavedWorkflowEntry[] {
       );
       continue;
     }
-    if (seenNames.has(meta.name)) {
-      log.warn(
-        { path, name: meta.name },
-        `Duplicate workflow name "${meta.name}"; skipping shadowed entry`,
+    const group = byName.get(meta.name) ?? [];
+    group.push({ name: meta.name, description: meta.description, path });
+    byName.set(meta.name, group);
+  }
+
+  const entries: SavedWorkflowEntry[] = [];
+  for (const [name, group] of byName) {
+    if (group.length > 1) {
+      logDuplicateMetaName(
+        name,
+        group.map((e) => e.path),
       );
       continue;
     }
-    seenNames.add(meta.name);
-    entries.push({ name: meta.name, description: meta.description, path });
+    entries.push(group[0]);
   }
   return entries;
 }
 
 /**
- * Resolve a saved workflow by name, deterministically — {@link readWorkflowFiles}
- * yields in a stable order with directory-over-flat precedence, so "the first
- * match" is well-defined. Matches `meta.name` first (the canonical identity),
- * then falls back to the base name (directory or flat-file). Returns the source
- * + path, or `null` if nothing matches.
+ * Resolve a saved workflow by name: `meta.name` first (canonical), then the
+ * base name (directory or flat file). Returns the source + path, or `null`.
+ *
+ * FAIL CLOSED on a duplicate `meta.name`: if two files declare the requested
+ * name, neither is returned, so a planted file can't hijack it by iteration
+ * order. Base-name collisions already fail closed in {@link readWorkflowFiles}.
  */
 export function getWorkflow(name: string): SavedWorkflowSource | null {
+  const nameMatches: SavedWorkflowSource[] = [];
   let fileMatch: SavedWorkflowSource | null = null;
   for (const { baseName, path, source } of readWorkflowFiles()) {
     // Prefer a `meta.name` match — it is the canonical identity.
     try {
-      if (extractWorkflowMeta(source).name === name) return { source, path };
+      if (extractWorkflowMeta(source).name === name) {
+        nameMatches.push({ source, path });
+        continue;
+      }
     } catch {
       // Fall through: a file with non-extractable meta can still match by name.
     }
@@ -174,5 +205,13 @@ export function getWorkflow(name: string): SavedWorkflowSource | null {
       fileMatch = { source, path };
     }
   }
-  return fileMatch;
+
+  if (nameMatches.length > 1) {
+    logDuplicateMetaName(
+      name,
+      nameMatches.map((m) => m.path),
+    );
+    return null;
+  }
+  return nameMatches[0] ?? fileMatch;
 }

package/src/workspace/git-service.ts

@@ -62,6 +62,7 @@ function cleanGitEnv(workspaceDir: string): Record<string, string> {
 const WORKSPACE_GITIGNORE_RULES = [
   "data/db/",
   "data/qdrant/",
+  "data/monitoring/",
   "plugins/*/node_modules/",
   "logs/",
   "*.log",

package/src/workspace/migrations/011-backfill-installation-id.ts

@@ -2,14 +2,71 @@ import { randomUUID } from "node:crypto";
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+import { Database } from "bun:sqlite";
 
-import {
-  deleteMemoryCheckpoint,
-  getMemoryCheckpoint,
-} from "../../persistence/checkpoints.js";
-import { getExternalAssistantId } from "../../runtime/auth/external-assistant-id.js";
 import type { WorkspaceMigration } from "./types.js";
 
+const INSTALLATION_ID_CHECKPOINT_KEY = "telemetry:installation_id";
+
+/**
+ * Open the assistant SQLite database read-write, or return null when it does
+ * not exist yet (fresh install) or cannot be opened. The path mirrors
+ * getDbPath(): `<workspace>/data/db/assistant.db`.
+ */
+function openAssistantDb(workspaceDir: string): Database | null {
+  const dbPath = join(workspaceDir, "data", "db", "assistant.db");
+  if (!existsSync(dbPath)) {
+    return null;
+  }
+  try {
+    return new Database(dbPath);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read the persisted installation ID from the `memory_checkpoints` table.
+ * Returns null when the database, table, or row is absent — a fresh install
+ * has no prior ID to recover. Never throws.
+ */
+function readInstallationIdCheckpoint(workspaceDir: string): string | null {
+  const db = openAssistantDb(workspaceDir);
+  if (!db) {
+    return null;
+  }
+  try {
+    const row = db
+      .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
+      .get(INSTALLATION_ID_CHECKPOINT_KEY) as { value: string } | null;
+    return row?.value ?? null;
+  } catch {
+    return null;
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Delete the installation-ID checkpoint row. Idempotent and never throws — a
+ * missing table or row is a successful no-op.
+ */
+function deleteInstallationIdCheckpoint(workspaceDir: string): void {
+  const db = openAssistantDb(workspaceDir);
+  if (!db) {
+    return;
+  }
+  try {
+    db.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(
+      INSTALLATION_ID_CHECKPOINT_KEY,
+    );
+  } catch {
+    // Table doesn't exist — nothing to clean up.
+  } finally {
+    db.close();
+  }
+}
+
 export const backfillInstallationIdMigration: WorkspaceMigration = {
   id: "011-backfill-installation-id",
   description:
@@ -25,17 +82,10 @@ export const backfillInstallationIdMigration: WorkspaceMigration = {
     // No-op: leaving installationId in the lockfile is safe and non-disruptive.
   },
 
-  run(_workspaceDir: string): void {
+  run(workspaceDir: string): void {
     // a. Read existing installation ID from SQLite, or generate a new one.
-    //    On fresh installs the memory_checkpoints table may not exist yet,
-    //    so treat errors as null.
-    let existingId: string | null = null;
-    try {
-      existingId = getMemoryCheckpoint("telemetry:installation_id");
-    } catch {
-      // Table doesn't exist yet — fresh install, no prior ID to recover.
-    }
-    const installationId = existingId || randomUUID();
+    const installationId =
+      readInstallationIdCheckpoint(workspaceDir) || randomUUID();
 
     // b. Read the lockfile — check both the current and legacy lockfile paths
     //    to support installs that haven't migrated the filename yet.
@@ -50,7 +100,9 @@ export const backfillInstallationIdMigration: WorkspaceMigration = {
     let lockPath: string | undefined;
     let lockData: Record<string, unknown> | undefined;
     for (const candidate of lockCandidates) {
-      if (!existsSync(candidate)) continue;
+      if (!existsSync(candidate)) {
+        continue;
+      }
       try {
         const raw = JSON.parse(readFileSync(candidate, "utf-8"));
         if (raw && typeof raw === "object" && !Array.isArray(raw)) {
@@ -62,26 +114,30 @@ export const backfillInstallationIdMigration: WorkspaceMigration = {
         // Malformed — try next candidate.
       }
     }
-    if (!lockPath || !lockData) return;
+    if (!lockPath || !lockData) {
+      return;
+    }
 
-    // c. Find the assistant entry that corresponds to this daemon instance
+    // c. Find the assistant entry that corresponds to this daemon instance.
+    //    The external assistant ID comes from the VELLUM_ASSISTANT_NAME env
+    //    var, set by CLI hatch and Docker setup.
     const assistants = lockData.assistants as
       | Array<Record<string, unknown>>
       | undefined;
-    if (!Array.isArray(assistants)) return;
+    if (!Array.isArray(assistants)) {
+      return;
+    }
 
-    const externalId = getExternalAssistantId();
+    const externalId = process.env.VELLUM_ASSISTANT_NAME || undefined;
     const entry = assistants.find((a) => a.assistantId === externalId);
-    if (!entry) return;
+    if (!entry) {
+      return;
+    }
 
     // d. If already has a truthy installationId, skip lockfile write (idempotent)
     if (entry.installationId) {
       // e is skipped for lockfile write, but still clean up SQLite
-      try {
-        deleteMemoryCheckpoint("telemetry:installation_id");
-      } catch {
-        // Table doesn't exist — nothing to clean up.
-      }
+      deleteInstallationIdCheckpoint(workspaceDir);
       return;
     }
 
@@ -90,10 +146,6 @@ export const backfillInstallationIdMigration: WorkspaceMigration = {
     writeFileSync(lockPath, JSON.stringify(lockData, null, 2) + "\n");
 
     // f. Delete the stale SQLite row
-    try {
-      deleteMemoryCheckpoint("telemetry:installation_id");
-    } catch {
-      // Table doesn't exist — nothing to clean up.
-    }
+    deleteInstallationIdCheckpoint(workspaceDir);
   },
 };

package/src/workspace/migrations/041-backfill-google-gmail-settings-scope.ts

@@ -1,4 +1,7 @@
-import { rawGet, rawRun } from "../../persistence/raw-query.js";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+
 import type { WorkspaceMigration } from "./types.js";
 
 const GMAIL_SETTINGS_BASIC_SCOPE =
@@ -18,36 +21,52 @@ export const backfillGoogleGmailSettingsScopeMigration: WorkspaceMigration = {
   id: "041-backfill-google-gmail-settings-scope",
   description:
     "Backfill gmail.settings.basic scope for existing Google provider rows",
-  run(_workspaceDir: string): void {
-    let row: { defaultScopes: string } | null;
+  run(workspaceDir: string): void {
+    const dbPath = join(workspaceDir, "data", "db", "assistant.db");
+    if (!existsSync(dbPath)) return; // DB not created yet — nothing to backfill.
+
+    let db: Database;
     try {
-      row = rawGet<{ defaultScopes: string }>(
-        `SELECT defaultScopes FROM oauth_providers WHERE provider = 'google'`,
-      );
+      db = new Database(dbPath);
     } catch {
-      // DB not initialized yet — nothing to backfill.
+      // Cannot open DB — nothing to backfill.
       return;
     }
 
-    if (!row) return; // No google provider row — seed will create it fresh.
-
-    let scopes: string[];
     try {
-      const parsed = JSON.parse(row.defaultScopes);
-      scopes = Array.isArray(parsed) ? parsed : [];
-    } catch {
-      scopes = [];
-    }
+      let row: { defaultScopes: string } | null;
+      try {
+        row =
+          (db
+            .query(
+              `SELECT defaultScopes FROM oauth_providers WHERE provider = 'google'`,
+            )
+            .get() as { defaultScopes: string }) ?? null;
+      } catch {
+        // DB not initialized yet — nothing to backfill.
+        return;
+      }
+
+      if (!row) return; // No google provider row — seed will create it fresh.
 
-    if (scopes.includes(GMAIL_SETTINGS_BASIC_SCOPE)) return; // Already present.
+      let scopes: string[];
+      try {
+        const parsed = JSON.parse(row.defaultScopes);
+        scopes = Array.isArray(parsed) ? parsed : [];
+      } catch {
+        scopes = [];
+      }
 
-    scopes.push(GMAIL_SETTINGS_BASIC_SCOPE);
+      if (scopes.includes(GMAIL_SETTINGS_BASIC_SCOPE)) return; // Already present.
 
-    rawRun(
-      `UPDATE oauth_providers SET defaultScopes = ?, updatedAt = ? WHERE provider = 'google'`,
-      JSON.stringify(scopes),
-      new Date().toISOString(),
-    );
+      scopes.push(GMAIL_SETTINGS_BASIC_SCOPE);
+
+      db.query(
+        `UPDATE oauth_providers SET defaultScopes = ?, updatedAt = ? WHERE provider = 'google'`,
+      ).run(JSON.stringify(scopes), new Date().toISOString());
+    } finally {
+      db.close();
+    }
   },
   down(_workspaceDir: string): void {
     // Forward-only: removing the scope would break Gmail settings functionality

package/src/workspace/migrations/070-memory-v2-summary-schema-rebuild.ts

@@ -5,7 +5,7 @@ import type { WorkspaceMigration } from "./types.js";
  * #29823 (summary_dense / summary_sparse named vectors). The destructive
  * collection rebuild and reembed enqueue both run inside the daemon at
  * Qdrant init time — see `maybeRebuildMemoryV2Concepts` in
- * `assistant/src/daemon/memory-v2-startup.ts`. The "exactly-once" fence is
+ * `assistant/src/plugins/defaults/memory/v2/memory-v2-startup.ts`. The "exactly-once" fence is
  * per-collection schema introspection, not per-workspace checkpoint, so
  * users who wipe Qdrant separately still get re-rebuilt without resetting
  * any workspace flag.

package/src/workspace/migrations/118-seed-now-md.ts

@@ -0,0 +1,64 @@
+import { existsSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { getLogger } from "../../util/logger.js";
+import type { WorkspaceMigration } from "./types.js";
+
+const log = getLogger("workspace-migration-118");
+
+/**
+ * Stub contents for a freshly-seeded NOW.md. Every line is a `_`-prefixed
+ * comment, so `stripCommentLines` reduces it to empty — the scratchpad injector
+ * sees no content and injects nothing (zero token cost) until the assistant
+ * writes real state. The file existing on disk is the point: the system prompt
+ * tells the assistant it "has a scratchpad file (NOW.md) ... a single file you
+ * overwrite", and `file_edit` requires the target to already exist, so an absent
+ * NOW.md surfaces as "file not found" the first time the assistant follows that
+ * instruction. The comment body mirrors the SOUL.md scratchpad framing so the
+ * assistant has guidance in front of it when it overwrites the file.
+ *
+ * Inlined per the migrations self-containment rule.
+ */
+const NOW_MD_STUB = `_ NOW.md — your scratchpad. Overwrite this whole file with whatever is most
+_ relevant right now. Unlike your journal (retrospective, append-only), this is
+_ a single snapshot you keep current. It's loaded into your context
+_ automatically, so next-you always sees the latest.
+_
+_ What goes here: your current focus and what you're actively working on; threads
+_ you're tracking (waiting on a reply, monitoring something, pending follow-ups);
+_ near-term priorities; temporary context that matters now but won't in a week.
+_
+_ What stays out: permanent facts (those live in memory/); personality and
+_ principles (those live in SOUL.md).
+_
+_ Lines starting with _ are comments — they aren't loaded into your context.
+_ Replace them freely.
+`;
+
+export const seedNowMdMigration: WorkspaceMigration = {
+  id: "118-seed-now-md",
+  description:
+    "Seed a stub NOW.md scratchpad file so the prompt-referenced scratchpad exists on disk for read/edit",
+
+  run(workspaceDir: string): void {
+    // Create-only: write the stub solely when NOW.md is absent. The assistant
+    // owns the file once it exists, so an existing NOW.md — seeded stub or real
+    // content — is never clobbered. Unlike the onboarding-threads seed (069),
+    // this is not gated on `isNewWorkspace`: the assistants hitting the missing
+    // file are existing workspaces, and the comment-only stub injects nothing,
+    // so re-creating it on upgrade is harmless and exactly the intended fix.
+    const filePath = join(workspaceDir, "NOW.md");
+    if (existsSync(filePath)) return;
+    try {
+      writeFileSync(filePath, NOW_MD_STUB, "utf-8");
+    } catch (err) {
+      log.warn({ err }, "Failed to seed NOW.md scratchpad stub");
+    }
+  },
+
+  down(_workspaceDir: string): void {
+    // Forward-only: never delete NOW.md on rollback. By the time a rollback
+    // runs the assistant may have overwritten the stub with real scratchpad
+    // state, and the file's existence is what keeps file_edit/read working.
+  },
+};