@vellumai/assistant 0.5.5 → 0.5.7

package/src/memory/jobs-store.ts

@@ -4,6 +4,10 @@ import { v4 as uuid } from "uuid";
 import { getLogger } from "../util/logger.js";
 import { truncate } from "../util/truncate.js";
 import { getDb, rawAll, rawChanges } from "./db.js";
+import {
+  isQdrantBreakerOpen,
+  shouldAllowQdrantProbe,
+} from "./qdrant-circuit-breaker.js";
 import { memoryJobs } from "./schema.js";
 
 const log = getLogger("memory-jobs-store");
@@ -12,9 +16,6 @@ export type MemoryJobType =
   | "embed_segment"
   | "embed_item"
   | "embed_summary"
-  | "embed_chunk"
-  | "embed_episode"
-  | "embed_observation"
   | "extract_items"
   | "extract_entities"
   | "cleanup_stale_superseded_items"
@@ -30,8 +31,6 @@ export type MemoryJobType =
   | "embed_media"
   | "embed_attachment"
   | "generate_conversation_starters"
-  | "reduce_conversation_memory"
-  | "backfill_simplified_memory"
   | "generate_capability_cards" // legacy compat — silently dropped by worker (capability cards removed)
   | "generate_thread_starters"; // legacy compat — silently dropped by worker (renamed to generate_conversation_starters)
 
@@ -39,9 +38,6 @@ const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_segment",
   "embed_item",
   "embed_summary",
-  "embed_chunk",
-  "embed_episode",
-  "embed_observation",
   "embed_media",
   "embed_attachment",
 ];
@@ -209,14 +205,33 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
     .all();
 
   const remainingSlots = limit - nonEmbedCandidates.length;
+
+  // When the Qdrant circuit breaker is open, skip embed jobs entirely —
+  // they would just be claimed → fail → deferred, wasting CPU cycles.
+  // Exception: if the cooldown has elapsed (breaker ready for half-open probe),
+  // allow exactly 1 embed job through so the breaker can self-heal.
+  const breakerOpen = isQdrantBreakerOpen();
+  const probeAllowed = breakerOpen && shouldAllowQdrantProbe();
+  const skipEmbedJobs = breakerOpen && !probeAllowed;
+  const embedLimit = probeAllowed ? 1 : remainingSlots;
+
+  if (skipEmbedJobs && remainingSlots > 0) {
+    log.debug("Skipping embed job claims — Qdrant circuit breaker is open");
+  }
+  if (probeAllowed && remainingSlots > 0) {
+    log.debug(
+      "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
+    );
+  }
+
   const embedCandidates =
-    remainingSlots > 0
+    remainingSlots > 0 && !skipEmbedJobs
       ? db
           .select()
           .from(memoryJobs)
           .where(and(pendingFilter, inArray(memoryJobs.type, EMBED_JOB_TYPES)))
           .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
-          .limit(remainingSlots)
+          .limit(embedLimit)
           .all()
       : [];
 
@@ -251,8 +266,8 @@ export function completeMemoryJob(id: string): void {
 
 /** Max times a job can be deferred before it is marked as failed. */
 const MAX_DEFERRALS = 50;
-/** Warn when deferrals reach 80% of the limit. */
-const DEFER_WARNING_THRESHOLD = Math.floor(MAX_DEFERRALS * 0.8);
+/** Log warnings at these milestone counts to avoid flooding logs. */
+const DEFERRAL_WARN_MILESTONES = [40, 45];
 /** Base delay in ms for deferred jobs (grows with exponential backoff). */
 const DEFER_BASE_DELAY_MS = 30_000;
 /** Maximum delay cap for deferred jobs (5 minutes). */
@@ -294,7 +309,9 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
     return "failed";
   }
 
-  if (deferrals >= DEFER_WARNING_THRESHOLD) {
+  // Log at milestones only (40, 45) to avoid flooding logs.
+  // At 50, the job fails via the check above, so 40 and 45 are the warnings.
+  if (DEFERRAL_WARN_MILESTONES.includes(deferrals)) {
     log.warn(
       { jobId: id, type: row.type, deferrals, max: MAX_DEFERRALS },
       "Job approaching max deferral limit",

package/src/memory/jobs-worker.ts

@@ -3,7 +3,6 @@ import type { AssistantConfig } from "../config/types.js";
 import { getLogger } from "../util/logger.js";
 import { rawRun } from "./db.js";
 import { backfillJob } from "./job-handlers/backfill.js";
-import { backfillSimplifiedMemoryJob } from "./job-handlers/backfill-simplified-memory.js";
 import {
   cleanupStaleSupersededItemsJob,
   pruneOldConversationsJob,
@@ -12,11 +11,8 @@ import { generateConversationStartersJob } from "./job-handlers/conversation-sta
 // ── Per-job-type handlers ──────────────────────────────────────────
 import {
   embedAttachmentJob,
-  embedChunkJob,
-  embedEpisodeJob,
   embedItemJob,
   embedMediaJob,
-  embedObservationJob,
   embedSegmentJob,
   embedSummaryJob,
 } from "./job-handlers/embedding.js";
@@ -26,7 +22,6 @@ import {
   rebuildIndexJob,
 } from "./job-handlers/index-maintenance.js";
 import { mediaProcessingJob } from "./job-handlers/media-processing.js";
-import { reduceConversationMemoryJob } from "./job-handlers/reduce-conversation-memory.js";
 import { buildConversationSummaryJob } from "./job-handlers/summarization.js";
 import {
   BackendUnavailableError,
@@ -49,6 +44,9 @@ import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
 
 const log = getLogger("memory-jobs-worker");
 
+export const POLL_INTERVAL_MIN_MS = 1_500;
+export const POLL_INTERVAL_MAX_MS = 30_000;
+
 export interface MemoryJobsWorker {
   runOnce(): Promise<number>;
   stop(): void;
@@ -62,24 +60,45 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
 
   let stopped = false;
   let tickRunning = false;
+  let timer: ReturnType<typeof setTimeout>;
+  let currentIntervalMs = POLL_INTERVAL_MIN_MS;
 
   const tick = async () => {
     if (stopped || tickRunning) return;
     tickRunning = true;
     try {
-      await runMemoryJobsOnce({ enableScheduledCleanup: true });
+      const processed = await runMemoryJobsOnce({
+        enableScheduledCleanup: true,
+      });
+      if (processed > 0) {
+        currentIntervalMs = POLL_INTERVAL_MIN_MS;
+      } else {
+        currentIntervalMs = Math.min(
+          currentIntervalMs * 2,
+          POLL_INTERVAL_MAX_MS,
+        );
+      }
     } catch (err) {
       log.error({ err }, "Memory worker tick failed");
+      currentIntervalMs = Math.min(currentIntervalMs * 2, POLL_INTERVAL_MAX_MS);
     } finally {
       tickRunning = false;
     }
   };
 
-  const timer = setInterval(() => {
-    void tick();
-  }, 1500);
-  timer.unref();
-  void tick();
+  const scheduleTick = () => {
+    if (stopped) return;
+    timer = setTimeout(() => {
+      void tick().then(() => {
+        if (!stopped) scheduleTick();
+      });
+    }, currentIntervalMs);
+    (timer as NodeJS.Timeout).unref?.();
+  };
+
+  void tick().then(() => {
+    if (!stopped) scheduleTick();
+  });
 
   return {
     async runOnce(): Promise<number> {
@@ -87,7 +106,7 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
     },
     stop(): void {
       stopped = true;
-      clearInterval(timer);
+      clearTimeout(timer);
     },
   };
 }
@@ -272,15 +291,6 @@ async function processJob(
     case "embed_summary":
       await embedSummaryJob(job, config);
       return;
-    case "embed_chunk":
-      await embedChunkJob(job, config);
-      return;
-    case "embed_episode":
-      await embedEpisodeJob(job, config);
-      return;
-    case "embed_observation":
-      await embedObservationJob(job, config);
-      return;
     case "extract_items":
       await extractItemsJob(job);
       return;
@@ -321,12 +331,6 @@ async function processJob(
     case "embed_attachment":
       await embedAttachmentJob(job, config);
       return;
-    case "reduce_conversation_memory":
-      await reduceConversationMemoryJob(job);
-      return;
-    case "backfill_simplified_memory":
-      await backfillSimplifiedMemoryJob(job);
-      return;
     case "generate_conversation_starters":
       await generateConversationStartersJob(job);
       return;

package/src/memory/migrations/001-job-deferrals.ts

@@ -49,3 +49,22 @@ export function migrateJobDeferrals(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Reverse the deferral reconciliation by moving `deferrals` back into `attempts`
+ * for pending embed jobs. Best-effort: jobs that accumulated real deferral counts
+ * after the forward migration ran cannot be distinguished from migrated ones.
+ */
+export function downJobDeferrals(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `
+    UPDATE memory_jobs
+    SET attempts = deferrals,
+        deferrals = 0,
+        updated_at = ${Date.now()}
+    WHERE status = 'pending'
+      AND deferrals > 0
+      AND attempts = 0
+      AND type IN ('embed_segment', 'embed_item', 'embed_summary')
+  `);
+}

package/src/memory/migrations/004-entity-relation-dedup.ts

@@ -91,3 +91,13 @@ export function migrateMemoryEntityRelationDedup(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * No-op down: deduplication is a lossy operation — deleted duplicate rows
+ * cannot be restored. The forward migration merged rows by keeping the most
+ * recent evidence per (source, target, relation) triple; the discarded rows
+ * are permanently lost.
+ */
+export function downMemoryEntityRelationDedup(_database: DrizzleDb): void {
+  // Intentionally empty — irreversible lossy migration.
+}

package/src/memory/migrations/005-fingerprint-scope-unique.ts

@@ -93,3 +93,79 @@ export function migrateMemoryItemsFingerprintScopeUnique(
     raw.exec("PRAGMA foreign_keys = ON");
   }
 }
+
+/**
+ * Reverse the compound (fingerprint, scope_id) unique index change by rebuilding
+ * memory_items with a column-level UNIQUE on fingerprint.
+ *
+ * WARNING: This is dangerous if data now relies on the compound constraint
+ * (i.e., the same fingerprint exists in multiple scopes). In that case, the
+ * rebuild will fail with a UNIQUE constraint violation. This is intentional —
+ * it prevents silent data loss on rollback.
+ */
+export function downMemoryItemsFingerprintScopeUnique(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  // Check if the column-level UNIQUE already exists — if so, nothing to do.
+  const tableDdl = raw
+    .query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_items'`,
+    )
+    .get() as { sql: string } | null;
+  if (
+    !tableDdl ||
+    tableDdl.sql.match(/fingerprint\s+TEXT\s+NOT\s+NULL\s+UNIQUE/i)
+  ) {
+    return;
+  }
+
+  raw.exec("PRAGMA foreign_keys = OFF");
+  try {
+    raw.exec("BEGIN");
+
+    raw.exec(/*sql*/ `
+      CREATE TABLE memory_items_new (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        subject TEXT NOT NULL,
+        statement TEXT NOT NULL,
+        status TEXT NOT NULL,
+        confidence REAL NOT NULL,
+        fingerprint TEXT NOT NULL UNIQUE,
+        first_seen_at INTEGER NOT NULL,
+        last_seen_at INTEGER NOT NULL,
+        last_used_at INTEGER,
+        importance REAL,
+        access_count INTEGER NOT NULL DEFAULT 0,
+        valid_from INTEGER,
+        invalid_at INTEGER,
+        verification_state TEXT NOT NULL DEFAULT 'assistant_inferred',
+        scope_id TEXT NOT NULL DEFAULT 'default'
+      )
+    `);
+
+    raw.exec(/*sql*/ `
+      INSERT INTO memory_items_new
+      SELECT id, kind, subject, statement, status, confidence, fingerprint,
+             first_seen_at, last_seen_at, last_used_at, importance, access_count,
+             valid_from, invalid_at, verification_state, scope_id
+      FROM memory_items
+    `);
+
+    raw.exec(/*sql*/ `DROP TABLE memory_items`);
+    raw.exec(/*sql*/ `ALTER TABLE memory_items_new RENAME TO memory_items`);
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  } finally {
+    raw.exec("PRAGMA foreign_keys = ON");
+  }
+}

package/src/memory/migrations/006-scope-salted-fingerprints.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import { computeMemoryFingerprint } from "../fingerprint.js";
 
@@ -75,3 +77,51 @@ export function migrateMemoryItemsScopeSaltedFingerprints(
     throw e;
   }
 }
+
+/**
+ * Reverse the scope-salted fingerprint migration by recomputing fingerprints
+ * WITHOUT the scope_id prefix.
+ *
+ * Old format: sha256(`${kind}|${subject.toLowerCase()}|${statement.toLowerCase()}`)
+ */
+export function downMemoryItemsScopeSaltedFingerprints(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  interface ItemRow {
+    id: string;
+    kind: string;
+    subject: string;
+    statement: string;
+  }
+
+  const items = raw
+    .query(`SELECT id, kind, subject, statement FROM memory_items`)
+    .all() as ItemRow[];
+
+  if (items.length === 0) return;
+
+  try {
+    raw.exec("BEGIN");
+
+    const updateStmt = raw.prepare(
+      `UPDATE memory_items SET fingerprint = ? WHERE id = ?`,
+    );
+
+    for (const item of items) {
+      const normalized = `${item.kind}|${item.subject.toLowerCase()}|${item.statement.toLowerCase()}`;
+      const fingerprint = createHash("sha256").update(normalized).digest("hex");
+      updateStmt.run(fingerprint, item.id);
+    }
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  }
+}

package/src/memory/migrations/007-assistant-id-to-self.ts

@@ -265,3 +265,13 @@ export function migrateAssistantIdToSelf(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * No-op down: the original assistant_id values are not recoverable. The forward
+ * migration normalized all assistant_id values to "self" and merged/deduplicated
+ * rows where the same logical entity existed under both the real assistantId and
+ * "self". The original per-assistant IDs are permanently lost.
+ */
+export function downAssistantIdToSelf(_database: DrizzleDb): void {
+  // Intentionally empty — original assistant_id values cannot be restored.
+}

package/src/memory/migrations/008-remove-assistant-id-columns.ts

@@ -228,3 +228,37 @@ export function migrateRemoveAssistantIdColumns(database: DrizzleDb): void {
     raw.exec("PRAGMA foreign_keys = ON");
   }
 }
+
+/**
+ * Add the assistant_id column back to the 4 tables that had it removed.
+ *
+ * NOTE: The data previously stored in assistant_id is lost — all rows will
+ * have assistant_id = 'self' after this down migration. This only restores
+ * the column structure so that older code expecting the column can function.
+ */
+export function downRemoveAssistantIdColumns(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const tables = [
+    "conversation_keys",
+    "attachments",
+    "channel_inbound_events",
+    "message_runs",
+  ];
+
+  for (const table of tables) {
+    // Check if the table exists and lacks assistant_id
+    const ddl = raw
+      .query(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
+      .get(table) as { sql: string } | null;
+    if (!ddl || ddl.sql.includes("assistant_id")) continue;
+
+    try {
+      raw.exec(
+        /*sql*/ `ALTER TABLE ${table} ADD COLUMN assistant_id TEXT NOT NULL DEFAULT 'self'`,
+      );
+    } catch {
+      /* column already exists */
+    }
+  }
+}

package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts

@@ -95,3 +95,29 @@ export function migrateLlmUsageEventsDropAssistantId(
     raw.exec("PRAGMA foreign_keys = ON");
   }
 }
+
+/**
+ * Add the assistant_id column back to llm_usage_events.
+ *
+ * NOTE: The data previously stored in assistant_id is lost — all rows will
+ * have assistant_id = NULL after this down migration. This only restores
+ * the column structure so that older code expecting the column can function.
+ */
+export function downLlmUsageEventsDropAssistantId(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const ddl = raw
+    .query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'llm_usage_events'`,
+    )
+    .get() as { sql: string } | null;
+  if (!ddl || ddl.sql.includes("assistant_id")) return;
+
+  try {
+    raw.exec(
+      /*sql*/ `ALTER TABLE llm_usage_events ADD COLUMN assistant_id TEXT`,
+    );
+  } catch {
+    /* column already exists */
+  }
+}

package/src/memory/migrations/014-backfill-inbox-thread-state.ts

@@ -88,3 +88,13 @@ export function migrateBackfillInboxThreadStateFromBindings(
     throw e;
   }
 }
+
+/**
+ * No-op down: the seeded inbox thread state rows are expected to remain.
+ * The forward migration used INSERT OR IGNORE, so existing rows were never
+ * modified. Removing the seeded rows could leave the inbox empty for
+ * pre-existing conversations, which is worse than keeping them.
+ */
+export function downBackfillInboxThreadState(_database: DrizzleDb): void {
+  // Intentionally empty — seeded data is expected to remain.
+}

package/src/memory/migrations/015-drop-active-search-index.ts

@@ -31,3 +31,20 @@ export function migrateDropActiveSearchIndex(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Recreate the old idx_memory_items_active_search index with its original
+ * covering columns (before the migration added status and invalid_at as
+ * indexed columns).
+ */
+export function downDropActiveSearchIndex(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  // Drop the current index if it exists, then recreate with the old column set.
+  raw.exec(/*sql*/ `DROP INDEX IF EXISTS idx_memory_items_active_search`);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_items_active_search
+    ON memory_items(last_seen_at DESC, subject, statement, id, kind, confidence, importance, first_seen_at, scope_id)
+    WHERE status = 'active' AND invalid_at IS NULL
+  `);
+}

package/src/memory/migrations/019-notification-tables-schema-migration.ts

@@ -82,3 +82,15 @@ export function migrateNotificationTablesSchema(database: DrizzleDb): void {
     }
   }
 }
+
+/**
+ * No-op down: the old enum-based notification tables cannot be recreated
+ * without the original schema definitions (column names, types, constraints,
+ * and enum values). The forward migration dropped these tables entirely.
+ * Any data that was in them is permanently lost. The new signal-contract
+ * schema tables are structurally incompatible with the old enum-based ones.
+ */
+export function downNotificationTablesSchema(_database: DrizzleDb): void {
+  // Intentionally empty — old enum-based tables cannot be recreated without
+  // the original schema, and any data they contained is permanently lost.
+}

package/src/memory/migrations/020-rename-macos-ios-channel-to-vellum.ts

@@ -130,3 +130,124 @@ export function migrateRenameChannelToVellum(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Reverse the channel rename by changing "vellum" back to "macos" in all tables.
+ *
+ * NOTE: The forward migration renamed both "macos" and "ios" to "vellum", so we
+ * cannot distinguish which rows were originally "ios". This down migration
+ * conservatively maps all "vellum" values back to "macos" since that was the
+ * primary desktop channel identifier.
+ */
+export function downRenameChannelToVellum(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  try {
+    raw.exec("BEGIN");
+
+    // guardian_action_deliveries.destination_channel
+    const gadExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'guardian_action_deliveries'`,
+      )
+      .get();
+    if (gadExists) {
+      raw
+        .query(
+          `UPDATE guardian_action_deliveries SET destination_channel = 'macos' WHERE destination_channel = 'vellum'`,
+        )
+        .run();
+    }
+
+    // messages.user_message_channel / assistant_message_channel
+    const msgsExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'`,
+      )
+      .get();
+    if (msgsExists) {
+      const hasUserMsgChannel = raw
+        .query(
+          `SELECT 1 FROM pragma_table_info('messages') WHERE name = 'user_message_channel'`,
+        )
+        .get();
+      if (hasUserMsgChannel) {
+        raw
+          .query(
+            `UPDATE messages SET user_message_channel = 'macos' WHERE user_message_channel = 'vellum'`,
+          )
+          .run();
+      }
+      const hasAssistantMsgChannel = raw
+        .query(
+          `SELECT 1 FROM pragma_table_info('messages') WHERE name = 'assistant_message_channel'`,
+        )
+        .get();
+      if (hasAssistantMsgChannel) {
+        raw
+          .query(
+            `UPDATE messages SET assistant_message_channel = 'macos' WHERE assistant_message_channel = 'vellum'`,
+          )
+          .run();
+      }
+    }
+
+    // external_conversation_bindings.source_channel
+    const ecbExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'external_conversation_bindings'`,
+      )
+      .get();
+    if (ecbExists) {
+      raw
+        .query(
+          `UPDATE external_conversation_bindings SET source_channel = 'macos' WHERE source_channel = 'vellum'`,
+        )
+        .run();
+    }
+
+    // assistant_inbox_thread_state.source_channel
+    const aitsExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'assistant_inbox_thread_state'`,
+      )
+      .get();
+    if (aitsExists) {
+      raw
+        .query(
+          `UPDATE assistant_inbox_thread_state SET source_channel = 'macos' WHERE source_channel = 'vellum'`,
+        )
+        .run();
+    }
+
+    // conversations.origin_channel
+    const convExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'conversations'`,
+      )
+      .get();
+    if (convExists) {
+      const hasOriginChannel = raw
+        .query(
+          `SELECT 1 FROM pragma_table_info('conversations') WHERE name = 'origin_channel'`,
+        )
+        .get();
+      if (hasOriginChannel) {
+        raw
+          .query(
+            `UPDATE conversations SET origin_channel = 'macos' WHERE origin_channel = 'vellum'`,
+          )
+          .run();
+      }
+    }
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  }
+}