@vellumai/assistant 0.6.5 → 0.6.6

package/src/memory/graph/bootstrap.ts

@@ -21,7 +21,7 @@ import { getWorkspaceDir } from "../../util/platform.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "../checkpoints.js";
 import { getDb, rawAll, rawGet, rawRun } from "../db.js";
 import { enqueueMemoryJob, hasActiveJobOfType } from "../jobs-store.js";
-import { initQdrantClient } from "../qdrant-client.js";
+import { initQdrantClient, resolveQdrantUrl } from "../qdrant-client.js";
 import { conversations, memoryGraphNodes, memorySegments } from "../schema.js";
 import { runGraphExtraction } from "./extraction.js";
 import { countNodes } from "./store.js";
@@ -71,7 +71,7 @@ export async function bootstrapFromHistory(
   // Initialize Qdrant client for inline embedding
   try {
     initQdrantClient({
-      url: config.memory.qdrant.url ?? "http://127.0.0.1:6333",
+      url: resolveQdrantUrl(config),
       collection: config.memory.qdrant.collection,
       vectorSize: config.memory.qdrant.vectorSize,
       onDisk: config.memory.qdrant.onDisk ?? true,
@@ -384,6 +384,10 @@ const KIND_TO_PREFIX: Record<string, string> = {
  * schema. ORM-based inserts include every column in the schema definition,
  * so adding a column in a later migration would cause this migration to
  * fail with "table has no column named …" on upgrade paths.
+ *
+ * The INSERT intentionally omits columns added by later migrations (e.g.
+ * `image_refs` from migration 205) since they default to NULL and
+ * including them would couple this migration to those later schema changes.
  */
 export function migrateToolCreatedItems(): void {
   if (getMemoryCheckpoint(MIGRATE_ITEMS_CHECKPOINT)) return;
@@ -446,8 +450,8 @@ export function migrateToolCreatedItems(): void {
         event_date, emotional_charge, fidelity, confidence, significance,
         stability, reinforcement_count, last_reinforced,
         source_conversations, source_type, narrative_role, part_of_story,
-        image_refs, scope_id
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        scope_id
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
       id,
       content,
       "semantic",
@@ -466,7 +470,6 @@ export function migrateToolCreatedItems(): void {
       "direct",
       null,
       null,
-      null,
       row.scope_id || "default",
     );
 

package/src/memory/graph/extraction.ts

@@ -1066,10 +1066,18 @@ export async function runGraphExtraction(
   );
 
   // 7. Handle supersession (inherit durability before applying diff)
+  // TODO: full supersession is not yet implemented. When it lands, iterate
+  // BOTH `diff.createEdges` (existing → existing) AND `deferredEdges`
+  // (new → existing, the typical supersession case).
+  // Tracked by https://github.com/vellum-ai/vellum-assistant/pull/27057 (Devin).
   for (const edge of diff.createEdges) {
     if (edge.relationship === "supersedes") {
-      // Supersession is handled differently — see supersedeNode in store
-      // For now, just mark it; full supersession is applied after node creation
+      // Placeholder — see TODO above.
+    }
+  }
+  for (const de of deferredEdges) {
+    if (de.relationship === "supersedes") {
+      // Placeholder — see TODO above.
     }
   }
 

package/src/memory/graph/graph-search.test.ts

@@ -49,6 +49,7 @@ mock.module("../qdrant-client.js", () => ({
     },
   }),
   initQdrantClient: () => {},
+  resolveQdrantUrl: () => "http://127.0.0.1:6333",
   VellumQdrantClient: class {},
 }));
 

package/src/memory/graph/inspect.ts

@@ -14,7 +14,7 @@
 
 import { getConfig } from "../../config/loader.js";
 import { initializeDb } from "../db-init.js";
-import { initQdrantClient } from "../qdrant-client.js";
+import { initQdrantClient, resolveQdrantUrl } from "../qdrant-client.js";
 import {
   countNodes,
   getEdgesForNode,
@@ -29,7 +29,7 @@ initializeDb();
 const config = getConfig();
 try {
   initQdrantClient({
-    url: config.memory.qdrant.url ?? "http://127.0.0.1:6333",
+    url: resolveQdrantUrl(config),
     collection: config.memory.qdrant.collection,
     vectorSize: config.memory.qdrant.vectorSize,
     onDisk: config.memory.qdrant.onDisk ?? true,

package/src/memory/graph/retriever.ts

@@ -877,9 +877,16 @@ export interface TurnRetrievalResult {
    */
   queryVector?: number[];
   /**
-   * Optional sparse vector passed alongside `queryVector`. Currently always
-   * `undefined` — reserved for future hybrid retrieval that produces a
-   * sparse vector at the call site.
+   * Sparse (TF-IDF) vector of the user's last message, computed once per
+   * turn and reused across every return path of `retrieveForTurn`. Surfaced
+   * so downstream callers (e.g. the PKB hint retriever in
+   * `applyRuntimeInjections`) can pair it with `queryVector` to run a
+   * hybrid dense+sparse query — RRF fusion pulls in lexical matches
+   * (exact filenames, proper nouns, uncommon tokens) that pure dense
+   * embeddings wash out. Computed locally (no embedding-service call), so
+   * it survives even when the dense embed fails via the circuit breaker.
+   * `undefined` when the user's last message is empty/whitespace-only or
+   * yields no TF-IDF tokens.
    */
   sparseVector?: QdrantSparseVector;
 }

package/src/memory/migrations/041-approval-prompt-ts-tracker.ts

@@ -0,0 +1,26 @@
+import type { DrizzleDb } from "../db-connection.js";
+
+/**
+ * Tracker for approval prompt message timestamps.
+ *
+ * Stores the (channel, chat_id, ts) tuples for delivered guardian approval
+ * prompts so only reactions on a known prompt can resolve a pending
+ * request. Persistence (rather than in-memory state) is required because
+ * the guardian approval TTL is 30 minutes, which can span a daemon
+ * restart between prompt delivery and the user's reaction.
+ */
+export function createApprovalPromptTsTrackerTable(database: DrizzleDb): void {
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS approval_prompt_ts_tracker (
+      channel TEXT NOT NULL,
+      chat_id TEXT NOT NULL,
+      ts TEXT NOT NULL,
+      expires_at INTEGER NOT NULL,
+      PRIMARY KEY (channel, chat_id, ts)
+    )
+  `);
+
+  database.run(
+    /*sql*/ `CREATE INDEX IF NOT EXISTS idx_approval_prompt_ts_tracker_expires ON approval_prompt_ts_tracker(expires_at)`,
+  );
+}

package/src/memory/migrations/149-oauth-tables.ts

@@ -15,6 +15,7 @@ export function createOAuthTables(database: DrizzleDb): void {
       base_url TEXT,
       default_scopes TEXT NOT NULL DEFAULT '[]',
       scope_policy TEXT NOT NULL DEFAULT '{}',
+      available_scopes TEXT,
       extra_params TEXT,
       callback_transport TEXT,
       loopback_port INTEGER,

package/src/memory/migrations/223-schedule-script-column.ts

@@ -0,0 +1,11 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+export function migrateScheduleScriptColumn(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  try {
+    raw.exec(`ALTER TABLE cron_jobs ADD COLUMN script TEXT`);
+  } catch {
+    // Column already exists — nothing to do.
+  }
+}

package/src/memory/migrations/224-oauth-providers-managed-service-is-paid.ts

@@ -0,0 +1,24 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Add the `managed_service_is_paid` column to `oauth_providers`.
+ *
+ * Flags whether a provider's managed OAuth variant requires a paid plan
+ * upstream (e.g. Twitter). Defaults to 0 (false) for all existing rows.
+ *
+ * Mirrors the pattern from migration 178. The raw.exec() call below is
+ * Drizzle's SQLite statement runner — NOT a shell exec.
+ */
+export function migrateOAuthProvidersManagedServiceIsPaid(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+  try {
+    raw.exec(
+      /*sql*/ `ALTER TABLE oauth_providers ADD COLUMN managed_service_is_paid INTEGER NOT NULL DEFAULT 0`,
+    );
+  } catch {
+    // Column already exists — nothing to do.
+  }
+}

package/src/memory/migrations/225-oauth-providers-available-scopes.ts

@@ -0,0 +1,13 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+export function migrateOAuthProvidersAvailableScopes(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+  try {
+    raw.exec(`ALTER TABLE oauth_providers ADD COLUMN available_scopes TEXT`);
+  } catch {
+    // Column already exists — nothing to do.
+  }
+}

package/src/memory/migrations/index.ts

@@ -42,6 +42,7 @@ export { migrateVoiceInviteColumns } from "./037-voice-invite-columns.js";
 export { createActorTokenRecordsTable } from "./038-actor-token-records.js";
 export { createActorRefreshTokenRecordsTable } from "./039-actor-refresh-token-records.js";
 export { migrateInviteCodeHashColumn } from "./040-invite-code-hash-column.js";
+export { createApprovalPromptTsTrackerTable } from "./041-approval-prompt-ts-tracker.js";
 export { createCoreTables } from "./100-core-tables.js";
 export { createWatchersAndLogsTables } from "./101-watchers-and-logs.js";
 export { addCoreColumns } from "./102-alter-table-columns.js";
@@ -167,6 +168,9 @@ export {
 } from "./220-normalize-user-file-by-principal.js";
 export { migrateConversationsArchivedAt } from "./221-conversations-archived-at.js";
 export { migrateStripPlaceholderSentinelsFromMessages } from "./222-strip-placeholder-sentinels-from-messages.js";
+export { migrateScheduleScriptColumn } from "./223-schedule-script-column.js";
+export { migrateOAuthProvidersManagedServiceIsPaid } from "./224-oauth-providers-managed-service-is-paid.js";
+export { migrateOAuthProvidersAvailableScopes } from "./225-oauth-providers-available-scopes.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/pkb/pkb-index.test.ts

@@ -96,6 +96,7 @@ mock.module("../qdrant-client.js", () => ({
       return scrollReturnPoints;
     },
   }),
+  resolveQdrantUrl: () => "http://127.0.0.1:6333",
 }));
 
 // The circuit breaker is a thin wrapper; just call the function through.

package/src/memory/pkb/pkb-reconcile.test.ts

@@ -46,6 +46,7 @@ mock.module("../qdrant-client.js", () => ({
       deleteCalls.push({ path, memoryScopeId });
     },
   }),
+  resolveQdrantUrl: () => "http://127.0.0.1:6333",
 }));
 
 // Circuit breaker — pass-through.

package/src/memory/pkb/pkb-search.test.ts

@@ -25,6 +25,8 @@ type ScoredPoint = { id: string; score: number; payload: Payload };
 
 let hybridResults: ScoredPoint[] = [];
 let denseResults: ScoredPoint[] = [];
+let hybridThrows: Error | null = null;
+let denseThrows: Error | null = null;
 
 mock.module("../qdrant-circuit-breaker.js", () => ({
   isQdrantBreakerOpen: () => breakerOpen,
@@ -41,6 +43,7 @@ mock.module("../qdrant-client.js", () => ({
       prefetchLimit?: number;
     }) => {
       hybridSearchCalls.push(params);
+      if (hybridThrows) throw hybridThrows;
       return hybridResults;
     },
     search: async (
@@ -49,9 +52,11 @@ mock.module("../qdrant-client.js", () => ({
       filter?: Record<string, unknown>,
     ) => {
       searchCalls.push({ vector, limit, filter });
+      if (denseThrows) throw denseThrows;
       return denseResults;
     },
   }),
+  resolveQdrantUrl: () => "http://127.0.0.1:6333",
 }));
 
 const { searchPkbFiles } = await import("./pkb-search.js");
@@ -63,6 +68,8 @@ describe("searchPkbFiles", () => {
     searchCalls.length = 0;
     hybridResults = [];
     denseResults = [];
+    hybridThrows = null;
+    denseThrows = null;
   });
 
   test("filter payload targets pkb_file (hybrid path runs both queries)", async () => {
@@ -236,7 +243,7 @@ describe("searchPkbFiles", () => {
     expect(other?.hybridScore).toBeUndefined();
   });
 
-  test("result present only in hybrid returns denseScore=0 and a hybridScore", async () => {
+  test("hybrid-only hits (no dense match) are dropped so they can't evict dense-qualified paths before the slice", async () => {
     denseResults = [
       {
         id: "a",
@@ -248,6 +255,10 @@ describe("searchPkbFiles", () => {
         },
       },
     ];
+    // `b` appears only in the hybrid response (outside the dense prefetch).
+    // It has no cosine score, so it can never pass a downstream threshold
+    // and must not be surfaced — otherwise it could crowd out dense-qualified
+    // hits before the caller gates on denseScore.
     hybridResults = [
       {
         id: "a",
@@ -275,12 +286,62 @@ describe("searchPkbFiles", () => {
       10,
     );
 
+    expect(results).toHaveLength(1);
     const a = results.find((r) => r.path === "/notes/a.md");
-    const b = results.find((r) => r.path === "/notes/b.md");
     expect(a?.denseScore).toBe(0.8);
     expect(a?.hybridScore).toBe(0.03);
-    expect(b?.denseScore).toBe(0);
-    expect(b?.hybridScore).toBe(0.02);
+    expect(results.find((r) => r.path === "/notes/b.md")).toBeUndefined();
+  });
+
+  test("hybrid failure (transient) falls back to dense-only results", async () => {
+    denseResults = [
+      {
+        id: "a",
+        score: 0.8,
+        payload: {
+          target_type: "pkb_file",
+          target_id: "t-1",
+          path: "/notes/a.md",
+        },
+      },
+    ];
+    hybridThrows = new Error("qdrant hybrid transient failure");
+
+    const results = await searchPkbFiles(
+      [0.1],
+      { indices: [1], values: [1] },
+      10,
+    );
+
+    expect(results).toHaveLength(1);
+    expect(results[0]?.path).toBe("/notes/a.md");
+    expect(results[0]?.denseScore).toBe(0.8);
+    expect(results[0]?.hybridScore).toBeUndefined();
+  });
+
+  test("dense failure (transient) falls back to empty results even if hybrid succeeded", async () => {
+    denseThrows = new Error("qdrant dense transient failure");
+    hybridResults = [
+      {
+        id: "a",
+        score: 0.03,
+        payload: {
+          target_type: "pkb_file",
+          target_id: "t-1",
+          path: "/notes/a.md",
+        },
+      },
+    ];
+
+    // Without a dense cosine score there is nothing to gate on, so the
+    // hybrid-only fallback surfaces no results.
+    const results = await searchPkbFiles(
+      [0.1],
+      { indices: [1], values: [1] },
+      10,
+    );
+
+    expect(results).toEqual([]);
   });
 
   test("empty Qdrant response yields []", async () => {

package/src/memory/pkb/pkb-search.ts

@@ -28,9 +28,10 @@ const log = getLogger("pkb-search");
  *
  * PKB files are chunked at index time, so a single path can match on
  * multiple chunks. Scores collapse to the highest per path, per query.
- * The two queries are unioned by path — a path present in only one
- * response still surfaces, with the missing score left `undefined` on
- * its result.
+ * Only paths with a dense cosine score are returned (hybrid-only matches
+ * are dropped because their implicit `denseScore = 0` can never satisfy a
+ * caller's threshold, and keeping them would evict dense-qualifying hits
+ * from the pre-slice top-`limit` window).
  */
 export async function searchPkbFiles(
   queryVector: number[],
@@ -77,29 +78,50 @@ export async function searchPkbFiles(
       )
     : Promise.resolve([]);
 
-  const [denseResults, hybridResults] = await Promise.all([
-    densePromise,
-    hybridPromise,
-  ]);
+  // Silence any hybrid rejection so a short-circuit on dense failure below
+  // does not surface it as an unhandledRejection. We still `await` the
+  // original promise in the non-short-circuit path to observe its outcome.
+  hybridPromise.catch(() => {});
+
+  // Dense is the required signal — only paths with a dense cosine score are
+  // merged, so a dense rejection guarantees `[]` regardless of hybrid. Return
+  // immediately rather than blocking on hybrid latency.
+  let denseResults: QdrantSearchResult[];
+  try {
+    denseResults = await densePromise;
+  } catch (err) {
+    log.warn({ err }, "Dense PKB search failed; returning empty results");
+    return [];
+  }
+
+  let hybridResults: QdrantSearchResult[] = [];
+  if (useHybrid) {
+    try {
+      hybridResults = await hybridPromise;
+    } catch (err) {
+      log.warn(
+        { err },
+        "Hybrid PKB search failed; falling back to dense-only results",
+      );
+    }
+  }
 
   const denseByPath = collapseByPath(denseResults);
   const hybridByPath = useHybrid ? collapseByPath(hybridResults) : new Map();
 
-  const allPaths = new Set<string>([
-    ...denseByPath.keys(),
-    ...hybridByPath.keys(),
-  ]);
-
+  // Only surface paths that have a dense cosine score. Hybrid-only hits
+  // (paths that appeared in the hybrid query but fell outside the dense
+  // prefetch) would have `denseScore = 0`, which is guaranteed to fail any
+  // caller's cosine threshold — and because they'd sort ahead of dense
+  // matches (hybrid-first ranking), they'd evict dense-qualifying hits from
+  // the top-`limit` window before callers ever get a chance to gate on
+  // denseScore. Dropping them keeps gating meaningful with the pre-slice.
   const merged: PkbSearchResult[] = [];
-  for (const path of allPaths) {
-    const denseScore = denseByPath.get(path);
+  for (const [path, denseScore] of denseByPath) {
     const hybridScore = hybridByPath.get(path);
-    // A path that only shows up in the hybrid query (past the dense prefetch
-    // limit) has no cosine score to gate on. Carry denseScore = 0 so callers
-    // see it but can filter it out with their threshold.
     merged.push({
       path,
-      denseScore: denseScore ?? 0,
+      denseScore,
       ...(useHybrid && hybridScore !== undefined ? { hybridScore } : {}),
     });
   }

package/src/memory/qdrant-client.test.ts

@@ -0,0 +1,60 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { applyNestedDefaults } from "../config/loader.js";
+import type { AssistantConfig } from "../config/types.js";
+import { resolveQdrantUrl } from "./qdrant-client.js";
+
+const DEFAULT_CONFIG: AssistantConfig = applyNestedDefaults({});
+
+describe("resolveQdrantUrl", () => {
+  const savedPort = process.env.QDRANT_HTTP_PORT;
+  const savedUrl = process.env.QDRANT_URL;
+
+  beforeEach(() => {
+    delete process.env.QDRANT_HTTP_PORT;
+    delete process.env.QDRANT_URL;
+  });
+
+  afterEach(() => {
+    if (savedPort === undefined) delete process.env.QDRANT_HTTP_PORT;
+    else process.env.QDRANT_HTTP_PORT = savedPort;
+    if (savedUrl === undefined) delete process.env.QDRANT_URL;
+    else process.env.QDRANT_URL = savedUrl;
+  });
+
+  test("falls back to config when no env vars are set", () => {
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe("http://127.0.0.1:6333");
+  });
+
+  test("honours QDRANT_URL when set", () => {
+    process.env.QDRANT_URL = "http://qdrant.example.com:6333";
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe(
+      "http://qdrant.example.com:6333",
+    );
+  });
+
+  test("QDRANT_HTTP_PORT wins over QDRANT_URL", () => {
+    process.env.QDRANT_URL = "http://qdrant.example.com:6333";
+    process.env.QDRANT_HTTP_PORT = "20200";
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe("http://127.0.0.1:20200");
+  });
+
+  test("QDRANT_HTTP_PORT wins over config default", () => {
+    process.env.QDRANT_HTTP_PORT = "20200";
+    expect(resolveQdrantUrl(DEFAULT_CONFIG)).toBe("http://127.0.0.1:20200");
+  });
+
+  test("respects a non-default config URL when no env is set", () => {
+    const config: AssistantConfig = {
+      ...DEFAULT_CONFIG,
+      memory: {
+        ...DEFAULT_CONFIG.memory,
+        qdrant: {
+          ...DEFAULT_CONFIG.memory.qdrant,
+          url: "http://custom-host:9999",
+        },
+      },
+    };
+    expect(resolveQdrantUrl(config)).toBe("http://custom-host:9999");
+  });
+});

package/src/memory/qdrant-client.ts

@@ -1,10 +1,35 @@
 import { QdrantClient as QdrantRestClient } from "@qdrant/js-client-rest";
 import { v4 as uuid } from "uuid";
 
+import { getQdrantHttpPortEnv, getQdrantUrlEnv } from "../config/env.js";
+import type { AssistantConfig } from "../config/types.js";
 import { getLogger } from "../util/logger.js";
 
 const log = getLogger("qdrant-client");
 
+/**
+ * Resolve the Qdrant base URL for this process.
+ *
+ * Precedence (highest first):
+ *   1. `QDRANT_HTTP_PORT` — a locally-spawned Qdrant sidecar on 127.0.0.1:<port>.
+ *      Set by the CLI when spawning the daemon with a non-default port
+ *      (multi-local instances).
+ *   2. `QDRANT_URL` — an external Qdrant instance (K8s sidecar, remote URL).
+ *   3. `config.memory.qdrant.url` — static config (defaults to
+ *      `http://127.0.0.1:6333`).
+ *
+ * Every caller that constructs a Qdrant URL should route through this helper
+ * so the precedence stays consistent across the daemon, CLI, and any
+ * background jobs that re-read the config.
+ */
+export function resolveQdrantUrl(config: AssistantConfig): string {
+  const port = getQdrantHttpPortEnv();
+  if (port) return `http://127.0.0.1:${port}`;
+  const url = getQdrantUrlEnv();
+  if (url) return url;
+  return config.memory.qdrant.url;
+}
+
 export interface QdrantSparseVector {
   indices: number[];
   values: number[];

package/src/memory/schema/infrastructure.ts

@@ -28,6 +28,7 @@ export const cronJobs = sqliteTable("cron_jobs", {
   reuseConversation: integer("reuse_conversation", { mode: "boolean" })
     .notNull()
     .default(false), // reuse the same conversation across runs
+  script: text("script"), // shell command for script mode (nullable, only used when mode = 'script')
   createdAt: integer("created_at").notNull(),
   updatedAt: integer("updated_at").notNull(),
 });

package/src/memory/schema/oauth.ts

@@ -20,7 +20,7 @@ export const oauthProviders = sqliteTable("oauth_providers", {
   userinfoUrl: text("userinfo_url"),
   baseUrl: text("base_url"),
   defaultScopes: text("default_scopes").notNull().default("[]"),
-  scopePolicy: text("scope_policy").notNull().default("{}"),
+  availableScopes: text("available_scopes"),
   scopeSeparator: text("scope_separator").notNull().default(" "),
   authorizeParams: text("extra_params"),
   pingUrl: text("ping_url"),
@@ -30,6 +30,9 @@ export const oauthProviders = sqliteTable("oauth_providers", {
   revokeUrl: text("revoke_url"),
   revokeBodyTemplate: text("revoke_body_template"),
   managedServiceConfigKey: text("managed_service_config_key"),
+  managedServiceIsPaid: integer("managed_service_is_paid", { mode: "boolean" })
+    .notNull()
+    .default(false),
   displayLabel: text("display_name"),
   description: text("description"),
   dashboardUrl: text("dashboard_url"),