@vellumai/assistant 0.5.3 → 0.5.4

package/src/memory/job-handlers/reduce-conversation-memory.ts

@@ -0,0 +1,229 @@
+/**
+ * Job handler for `reduce_conversation_memory`.
+ *
+ * Ties together the reducer service ({@link runReducer}) and the transactional
+ * store ({@link applyReducerResult}) to process unreduced conversation turns
+ * as a background job.
+ *
+ * The handler:
+ *   1. Loads the conversation and validates the dirty tail marker.
+ *   2. Loads the unreduced message span (messages since the dirty tail).
+ *   3. Loads active time contexts and open loops for the conversation's scope.
+ *   4. Includes the current `contextSummary` when present (prepended as a
+ *      synthetic system message so the reducer has compacted context).
+ *   5. Calls `runReducer` with the assembled input.
+ *   6. Applies the result transactionally via `applyReducerResult`.
+ *
+ * If the reducer fails or returns the {@link EMPTY_REDUCER_RESULT} sentinel
+ * (unparseable output), the checkpoint is NOT advanced — the dirty tail stays
+ * in place so the next run retries. A valid-but-empty model response (e.g.
+ * `{}`) returns a normal empty result that advances the checkpoint normally.
+ */
+
+import { and, asc, eq, gte } from "drizzle-orm";
+
+import { getLogger } from "../../util/logger.js";
+import { type ConversationRow, getConversation } from "../conversation-crud.js";
+import { getDb } from "../db.js";
+import { asString } from "../job-utils.js";
+import type { MemoryJob } from "../jobs-store.js";
+import { type ReducerPromptInput, runReducer } from "../reducer.js";
+import {
+  applyReducerResult,
+  getActiveOpenLoops,
+  getActiveTimeContexts,
+} from "../reducer-store.js";
+import { EMPTY_REDUCER_RESULT } from "../reducer-types.js";
+import { messages } from "../schema.js";
+
+const log = getLogger("reduce-conversation-memory-job");
+
+export interface ReduceConversationMemoryPayload {
+  conversationId: string;
+}
+
+/**
+ * Process a `reduce_conversation_memory` job.
+ *
+ * @throws Re-throws reducer errors so the job worker can classify and retry.
+ */
+export async function reduceConversationMemoryJob(
+  job: MemoryJob,
+): Promise<void> {
+  const conversationId = asString(job.payload.conversationId);
+  if (!conversationId) {
+    log.warn({ jobId: job.id }, "Missing conversationId in job payload");
+    return;
+  }
+
+  // ── 1. Load conversation and validate dirty tail ────────────────
+  const conversation = getConversation(conversationId);
+  if (!conversation) {
+    log.warn(
+      { jobId: job.id, conversationId },
+      "Conversation not found, skipping reduction",
+    );
+    return;
+  }
+
+  const dirtyTailMessageId = conversation.memoryDirtyTailSinceMessageId;
+  if (!dirtyTailMessageId) {
+    log.debug(
+      { jobId: job.id, conversationId },
+      "No dirty tail marker — conversation is already fully reduced",
+    );
+    return;
+  }
+
+  // ── 2. Load unreduced message span ──────────────────────────────
+  const unreducedMessages = loadUnreducedMessages(
+    conversationId,
+    dirtyTailMessageId,
+  );
+
+  if (unreducedMessages.length === 0) {
+    log.debug(
+      { jobId: job.id, conversationId, dirtyTailMessageId },
+      "No messages found from dirty tail — nothing to reduce",
+    );
+    return;
+  }
+
+  // ── 3. Load active brief-state context ──────────────────────────
+  const scopeId = conversation.memoryScopeId;
+  const now = Date.now();
+
+  const existingTimeContexts = getActiveTimeContexts(scopeId, now);
+  const existingOpenLoops = getActiveOpenLoops(scopeId);
+
+  // ── 4. Build reducer input ──────────────────────────────────────
+  const newMessages = buildNewMessages(conversation, unreducedMessages);
+
+  const reducerInput: ReducerPromptInput = {
+    conversationId,
+    newMessages,
+    existingTimeContexts: existingTimeContexts.map((tc) => ({
+      id: tc.id,
+      summary: tc.summary,
+    })),
+    existingOpenLoops: existingOpenLoops.map((ol) => ({
+      id: ol.id,
+      summary: ol.summary,
+      status: ol.status,
+    })),
+    nowMs: now,
+    scopeId,
+  };
+
+  // ── 5. Run the reducer ──────────────────────────────────────────
+  const result = await runReducer(reducerInput);
+
+  // If the reducer returns the empty sentinel, skip applying — the dirty
+  // tail stays in place so a future run can retry.
+  if (result === EMPTY_REDUCER_RESULT) {
+    log.warn(
+      { jobId: job.id, conversationId },
+      "Reducer returned empty result — not advancing checkpoint",
+    );
+    return;
+  }
+
+  // ── 6. Apply result transactionally ─────────────────────────────
+  const lastMessage = unreducedMessages[unreducedMessages.length - 1];
+  applyReducerResult({
+    result,
+    conversationId,
+    scopeId,
+    reducedThroughMessageId: lastMessage.id,
+    now,
+  });
+
+  log.info(
+    {
+      jobId: job.id,
+      conversationId,
+      reducedThroughMessageId: lastMessage.id,
+      messageCount: unreducedMessages.length,
+      timeContextOps: result.timeContexts.length,
+      openLoopOps: result.openLoops.length,
+    },
+    "Conversation memory reduction completed",
+  );
+}
+
+// ── Internal helpers ────────────────────────────────────────────────
+
+interface MessageRow {
+  id: string;
+  role: string;
+  content: string;
+  createdAt: number;
+}
+
+/**
+ * Load messages from `dirtyTailMessageId` onward (inclusive), ordered by
+ * createdAt ascending. Uses the message's createdAt as the boundary since
+ * message ordering is timestamp-based.
+ */
+function loadUnreducedMessages(
+  conversationId: string,
+  dirtyTailMessageId: string,
+): MessageRow[] {
+  const db = getDb();
+
+  // First, find the createdAt of the dirty tail message
+  const tailMessage = db
+    .select({ createdAt: messages.createdAt })
+    .from(messages)
+    .where(eq(messages.id, dirtyTailMessageId))
+    .get();
+
+  if (!tailMessage) {
+    return [];
+  }
+
+  // Load all messages from that timestamp onward
+  return db
+    .select({
+      id: messages.id,
+      role: messages.role,
+      content: messages.content,
+      createdAt: messages.createdAt,
+    })
+    .from(messages)
+    .where(
+      and(
+        eq(messages.conversationId, conversationId),
+        gte(messages.createdAt, tailMessage.createdAt),
+      ),
+    )
+    .orderBy(asc(messages.createdAt))
+    .all();
+}
+
+/**
+ * Build the `newMessages` array for the reducer input.
+ *
+ * When the conversation has a `contextSummary` (from context window
+ * compaction), it is prepended as a synthetic `system` message so the
+ * reducer has access to prior compacted context.
+ */
+function buildNewMessages(
+  conversation: ConversationRow,
+  unreducedMessages: MessageRow[],
+): Array<{ role: string; content: string }> {
+  const result: Array<{ role: string; content: string }> = [];
+
+  if (conversation.contextSummary) {
+    result.push({
+      role: "system",
+      content: `[Prior context summary] ${conversation.contextSummary}`,
+    });
+  }
+
+  for (const msg of unreducedMessages) {
+    result.push({ role: msg.role, content: msg.content });
+  }
+
+  return result;
+}

package/src/memory/jobs-store.ts

@@ -30,6 +30,8 @@ export type MemoryJobType =
   | "embed_media"
   | "embed_attachment"
   | "generate_conversation_starters"
+  | "reduce_conversation_memory"
+  | "backfill_simplified_memory"
   | "generate_capability_cards" // legacy compat — silently dropped by worker (capability cards removed)
   | "generate_thread_starters"; // legacy compat — silently dropped by worker (renamed to generate_conversation_starters)
 

package/src/memory/jobs-worker.ts

@@ -3,6 +3,7 @@ import type { AssistantConfig } from "../config/types.js";
 import { getLogger } from "../util/logger.js";
 import { rawRun } from "./db.js";
 import { backfillJob } from "./job-handlers/backfill.js";
+import { backfillSimplifiedMemoryJob } from "./job-handlers/backfill-simplified-memory.js";
 import {
   cleanupStaleSupersededItemsJob,
   pruneOldConversationsJob,
@@ -25,6 +26,7 @@ import {
   rebuildIndexJob,
 } from "./job-handlers/index-maintenance.js";
 import { mediaProcessingJob } from "./job-handlers/media-processing.js";
+import { reduceConversationMemoryJob } from "./job-handlers/reduce-conversation-memory.js";
 import { buildConversationSummaryJob } from "./job-handlers/summarization.js";
 import {
   BackendUnavailableError,
@@ -319,6 +321,12 @@ async function processJob(
     case "embed_attachment":
       await embedAttachmentJob(job, config);
       return;
+    case "reduce_conversation_memory":
+      await reduceConversationMemoryJob(job);
+      return;
+    case "backfill_simplified_memory":
+      await backfillSimplifiedMemoryJob(job);
+      return;
     case "generate_conversation_starters":
       await generateConversationStartersJob(job);
       return;

package/src/memory/migrations/036-normalize-phone-identities.ts

@@ -79,9 +79,27 @@ export function migrateNormalizePhoneIdentities(database: DrizzleDb): void {
       .get(table);
     const orderBy = hasUpdatedAt ? "updated_at DESC, rowid DESC" : "rowid DESC";
 
-    const selectColumns = [`id`, column];
+    // Filter uniqueKeyScope to only include peer columns that actually exist in the table.
+    // If a peer column is missing, its unique index can't exist either, so no collision risk.
+    let effectiveScope = uniqueKeyScope;
     if (uniqueKeyScope) {
-      for (const peer of uniqueKeyScope.peerColumns) {
+      const validPeers = uniqueKeyScope.peerColumns.filter(
+        (col) =>
+          !!raw
+            .query(`SELECT 1 FROM pragma_table_info(?) WHERE name = ?`)
+            .get(table, col),
+      );
+      effectiveScope =
+        validPeers.length === uniqueKeyScope.peerColumns.length
+          ? uniqueKeyScope
+          : validPeers.length > 0
+            ? { ...uniqueKeyScope, peerColumns: validPeers }
+            : undefined;
+    }
+
+    const selectColumns = [`id`, column];
+    if (effectiveScope) {
+      for (const peer of effectiveScope.peerColumns) {
         if (!selectColumns.includes(peer)) selectColumns.push(peer);
       }
     }
@@ -104,14 +122,14 @@ export function migrateNormalizePhoneIdentities(database: DrizzleDb): void {
       if (!original) continue;
       const normalized = normalizePhoneNumber(original);
       if (normalized && normalized !== original) {
-        if (uniqueKeyScope) {
+        if (effectiveScope) {
           // Check if another row already has the normalized value within the same unique-key scope
-          const peerConditions = uniqueKeyScope.peerColumns
+          const peerConditions = effectiveScope.peerColumns
             .map((col) => `${col} = ?`)
             .join(" AND ");
-          const peerValues = uniqueKeyScope.peerColumns.map((col) => row[col]);
-          const whereExtra = uniqueKeyScope.whereClause
-            ? ` AND (${uniqueKeyScope.whereClause})`
+          const peerValues = effectiveScope.peerColumns.map((col) => row[col]);
+          const whereExtra = effectiveScope.whereClause
+            ? ` AND (${effectiveScope.whereClause})`
             : "";
           const existing = raw
             .query(
@@ -154,9 +172,26 @@ export function migrateNormalizePhoneIdentities(database: DrizzleDb): void {
       .get(table);
     const orderBy = hasUpdatedAt ? "updated_at DESC, rowid DESC" : "rowid DESC";
 
-    const selectColumns = [`id`, column];
+    // Filter uniqueKeyScope to only include peer columns that actually exist in the table.
+    let effectiveScope = uniqueKeyScope;
     if (uniqueKeyScope) {
-      for (const peer of uniqueKeyScope.peerColumns) {
+      const validPeers = uniqueKeyScope.peerColumns.filter(
+        (col) =>
+          !!raw
+            .query(`SELECT 1 FROM pragma_table_info(?) WHERE name = ?`)
+            .get(table, col),
+      );
+      effectiveScope =
+        validPeers.length === uniqueKeyScope.peerColumns.length
+          ? uniqueKeyScope
+          : validPeers.length > 0
+            ? { ...uniqueKeyScope, peerColumns: validPeers }
+            : undefined;
+    }
+
+    const selectColumns = [`id`, column];
+    if (effectiveScope) {
+      for (const peer of effectiveScope.peerColumns) {
         if (!selectColumns.includes(peer)) selectColumns.push(peer);
       }
     }
@@ -179,13 +214,13 @@ export function migrateNormalizePhoneIdentities(database: DrizzleDb): void {
       if (!original) continue;
       const normalized = normalizePhoneNumber(original);
       if (normalized && normalized !== original) {
-        if (uniqueKeyScope) {
-          const peerConditions = uniqueKeyScope.peerColumns
+        if (effectiveScope) {
+          const peerConditions = effectiveScope.peerColumns
             .map((col) => `${col} = ?`)
             .join(" AND ");
-          const peerValues = uniqueKeyScope.peerColumns.map((col) => row[col]);
-          const whereExtra = uniqueKeyScope.whereClause
-            ? ` AND (${uniqueKeyScope.whereClause})`
+          const peerValues = effectiveScope.peerColumns.map((col) => row[col]);
+          const whereExtra = effectiveScope.whereClause
+            ? ` AND (${effectiveScope.whereClause})`
             : "";
           const existing = raw
             .query(

package/src/memory/migrations/135-backfill-contact-interaction-stats.ts

@@ -1,4 +1,4 @@
-import type { DrizzleDb } from "../db-connection.js";
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import { withCrashRecovery } from "./validate-migration-state.js";
 
 /**
@@ -7,6 +7,14 @@ import { withCrashRecovery } from "./validate-migration-state.js";
  * existing data, so it stays at 0 and accumulates going forward.
  */
 export function migrateBackfillContactInteractionStats(db: DrizzleDb): void {
+  const raw = getSqliteFrom(db);
+  const colExists = raw
+    .query(
+      `SELECT 1 FROM pragma_table_info('contacts') WHERE name = 'last_interaction'`,
+    )
+    .get();
+  if (!colExists) return;
+
   withCrashRecovery(db, "backfill_contact_interaction_stats", () => {
     db.run(/*sql*/ `
       UPDATE contacts

package/src/memory/migrations/141-rename-verification-table.ts

@@ -18,6 +18,14 @@ export function migrateRenameVerificationTable(database: DrizzleDb): void {
       .get();
     if (!oldTableExists) return;
 
+    // If the new table already exists, the rename would collide — skip
+    const newTableExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'channel_verification_sessions'`,
+      )
+      .get();
+    if (newTableExists) return;
+
     // Rename the physical table
     raw.exec(
       /*sql*/ `ALTER TABLE channel_guardian_verification_challenges RENAME TO channel_verification_sessions`,

package/src/memory/migrations/142-rename-verification-session-id-column.ts

@@ -15,14 +15,19 @@ export function migrateRenameVerificationSessionIdColumn(
     () => {
       const raw = getSqliteFrom(database);
 
-      // Check the old column exists before attempting the rename
+      // Check the old column exists and the new column doesn't before attempting the rename.
+      // Both checks are needed for crash recovery: if the rename succeeded but the checkpoint
+      // didn't commit, the old column is gone and the new one already exists.
       const columns = raw
         .query(`PRAGMA table_info(call_sessions)`)
         .all() as Array<{ name: string }>;
       const hasOldColumn = columns.some(
         (c) => c.name === "guardian_verification_session_id",
       );
-      if (!hasOldColumn) return;
+      const hasNewColumn = columns.some(
+        (c) => c.name === "verification_session_id",
+      );
+      if (!hasOldColumn || hasNewColumn) return;
 
       raw.exec(
         /*sql*/ `ALTER TABLE call_sessions RENAME COLUMN guardian_verification_session_id TO verification_session_id`,

package/src/memory/migrations/174-rename-thread-starters-table.ts

@@ -21,6 +21,14 @@ export function migrateRenameThreadStartersTable(database: DrizzleDb): void {
         .get();
       if (!oldTableExists) return;
 
+      // If the new table already exists (crash recovery), skip the rename
+      const newTableExists = raw
+        .query(
+          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'conversation_starters'`,
+        )
+        .get();
+      if (newTableExists) return;
+
       // Rename the physical table
       raw.exec(
         /*sql*/ `ALTER TABLE thread_starters RENAME TO conversation_starters`,

package/src/memory/migrations/188-schedule-quiet-flag.ts

@@ -0,0 +1,13 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+export function migrateScheduleQuietFlag(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  try {
+    raw.exec(
+      `ALTER TABLE cron_jobs ADD COLUMN quiet INTEGER NOT NULL DEFAULT 0`,
+    );
+  } catch {
+    // Column already exists — nothing to do.
+  }
+}

package/src/memory/migrations/index.ts

@@ -129,6 +129,7 @@ export { migrateLlmRequestLogProvider } from "./184-llm-request-log-provider.js"
 export { migrateMemoryBriefState } from "./185-memory-brief-state.js";
 export { migrateMemoryArchiveTables } from "./186-memory-archive.js";
 export { migrateMemoryReducerCheckpoints } from "./187-memory-reducer-checkpoints.js";
+export { migrateScheduleQuietFlag } from "./188-schedule-quiet-flag.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,