@vellumai/assistant 0.5.5 → 0.5.7

package/src/memory/job-handlers/reduce-conversation-memory.ts

@@ -1,229 +0,0 @@
-/**
- * Job handler for `reduce_conversation_memory`.
- *
- * Ties together the reducer service ({@link runReducer}) and the transactional
- * store ({@link applyReducerResult}) to process unreduced conversation turns
- * as a background job.
- *
- * The handler:
- *   1. Loads the conversation and validates the dirty tail marker.
- *   2. Loads the unreduced message span (messages since the dirty tail).
- *   3. Loads active time contexts and open loops for the conversation's scope.
- *   4. Includes the current `contextSummary` when present (prepended as a
- *      synthetic system message so the reducer has compacted context).
- *   5. Calls `runReducer` with the assembled input.
- *   6. Applies the result transactionally via `applyReducerResult`.
- *
- * If the reducer fails or returns the {@link EMPTY_REDUCER_RESULT} sentinel
- * (unparseable output), the checkpoint is NOT advanced — the dirty tail stays
- * in place so the next run retries. A valid-but-empty model response (e.g.
- * `{}`) returns a normal empty result that advances the checkpoint normally.
- */
-
-import { and, asc, eq, gte } from "drizzle-orm";
-
-import { getLogger } from "../../util/logger.js";
-import { type ConversationRow, getConversation } from "../conversation-crud.js";
-import { getDb } from "../db.js";
-import { asString } from "../job-utils.js";
-import type { MemoryJob } from "../jobs-store.js";
-import { type ReducerPromptInput, runReducer } from "../reducer.js";
-import {
-  applyReducerResult,
-  getActiveOpenLoops,
-  getActiveTimeContexts,
-} from "../reducer-store.js";
-import { EMPTY_REDUCER_RESULT } from "../reducer-types.js";
-import { messages } from "../schema.js";
-
-const log = getLogger("reduce-conversation-memory-job");
-
-export interface ReduceConversationMemoryPayload {
-  conversationId: string;
-}
-
-/**
- * Process a `reduce_conversation_memory` job.
- *
- * @throws Re-throws reducer errors so the job worker can classify and retry.
- */
-export async function reduceConversationMemoryJob(
-  job: MemoryJob,
-): Promise<void> {
-  const conversationId = asString(job.payload.conversationId);
-  if (!conversationId) {
-    log.warn({ jobId: job.id }, "Missing conversationId in job payload");
-    return;
-  }
-
-  // ── 1. Load conversation and validate dirty tail ────────────────
-  const conversation = getConversation(conversationId);
-  if (!conversation) {
-    log.warn(
-      { jobId: job.id, conversationId },
-      "Conversation not found, skipping reduction",
-    );
-    return;
-  }
-
-  const dirtyTailMessageId = conversation.memoryDirtyTailSinceMessageId;
-  if (!dirtyTailMessageId) {
-    log.debug(
-      { jobId: job.id, conversationId },
-      "No dirty tail marker — conversation is already fully reduced",
-    );
-    return;
-  }
-
-  // ── 2. Load unreduced message span ──────────────────────────────
-  const unreducedMessages = loadUnreducedMessages(
-    conversationId,
-    dirtyTailMessageId,
-  );
-
-  if (unreducedMessages.length === 0) {
-    log.debug(
-      { jobId: job.id, conversationId, dirtyTailMessageId },
-      "No messages found from dirty tail — nothing to reduce",
-    );
-    return;
-  }
-
-  // ── 3. Load active brief-state context ──────────────────────────
-  const scopeId = conversation.memoryScopeId;
-  const now = Date.now();
-
-  const existingTimeContexts = getActiveTimeContexts(scopeId, now);
-  const existingOpenLoops = getActiveOpenLoops(scopeId);
-
-  // ── 4. Build reducer input ──────────────────────────────────────
-  const newMessages = buildNewMessages(conversation, unreducedMessages);
-
-  const reducerInput: ReducerPromptInput = {
-    conversationId,
-    newMessages,
-    existingTimeContexts: existingTimeContexts.map((tc) => ({
-      id: tc.id,
-      summary: tc.summary,
-    })),
-    existingOpenLoops: existingOpenLoops.map((ol) => ({
-      id: ol.id,
-      summary: ol.summary,
-      status: ol.status,
-    })),
-    nowMs: now,
-    scopeId,
-  };
-
-  // ── 5. Run the reducer ──────────────────────────────────────────
-  const result = await runReducer(reducerInput);
-
-  // If the reducer returns the empty sentinel, skip applying — the dirty
-  // tail stays in place so a future run can retry.
-  if (result === EMPTY_REDUCER_RESULT) {
-    log.warn(
-      { jobId: job.id, conversationId },
-      "Reducer returned empty result — not advancing checkpoint",
-    );
-    return;
-  }
-
-  // ── 6. Apply result transactionally ─────────────────────────────
-  const lastMessage = unreducedMessages[unreducedMessages.length - 1];
-  applyReducerResult({
-    result,
-    conversationId,
-    scopeId,
-    reducedThroughMessageId: lastMessage.id,
-    now,
-  });
-
-  log.info(
-    {
-      jobId: job.id,
-      conversationId,
-      reducedThroughMessageId: lastMessage.id,
-      messageCount: unreducedMessages.length,
-      timeContextOps: result.timeContexts.length,
-      openLoopOps: result.openLoops.length,
-    },
-    "Conversation memory reduction completed",
-  );
-}
-
-// ── Internal helpers ────────────────────────────────────────────────
-
-interface MessageRow {
-  id: string;
-  role: string;
-  content: string;
-  createdAt: number;
-}
-
-/**
- * Load messages from `dirtyTailMessageId` onward (inclusive), ordered by
- * createdAt ascending. Uses the message's createdAt as the boundary since
- * message ordering is timestamp-based.
- */
-function loadUnreducedMessages(
-  conversationId: string,
-  dirtyTailMessageId: string,
-): MessageRow[] {
-  const db = getDb();
-
-  // First, find the createdAt of the dirty tail message
-  const tailMessage = db
-    .select({ createdAt: messages.createdAt })
-    .from(messages)
-    .where(eq(messages.id, dirtyTailMessageId))
-    .get();
-
-  if (!tailMessage) {
-    return [];
-  }
-
-  // Load all messages from that timestamp onward
-  return db
-    .select({
-      id: messages.id,
-      role: messages.role,
-      content: messages.content,
-      createdAt: messages.createdAt,
-    })
-    .from(messages)
-    .where(
-      and(
-        eq(messages.conversationId, conversationId),
-        gte(messages.createdAt, tailMessage.createdAt),
-      ),
-    )
-    .orderBy(asc(messages.createdAt))
-    .all();
-}
-
-/**
- * Build the `newMessages` array for the reducer input.
- *
- * When the conversation has a `contextSummary` (from context window
- * compaction), it is prepended as a synthetic `system` message so the
- * reducer has access to prior compacted context.
- */
-function buildNewMessages(
-  conversation: ConversationRow,
-  unreducedMessages: MessageRow[],
-): Array<{ role: string; content: string }> {
-  const result: Array<{ role: string; content: string }> = [];
-
-  if (conversation.contextSummary) {
-    result.push({
-      role: "system",
-      content: `[Prior context summary] ${conversation.contextSummary}`,
-    });
-  }
-
-  for (const msg of unreducedMessages) {
-    result.push({ role: msg.role, content: msg.content });
-  }
-
-  return result;
-}

package/src/memory/migrations/185-memory-brief-state.ts

@@ -1,52 +0,0 @@
-import type { DrizzleDb } from "../db-connection.js";
-import { getSqliteFrom } from "../db-connection.js";
-
-/**
- * Create the memory brief state tables: time_contexts and open_loops.
- *
- * Both tables use CREATE TABLE IF NOT EXISTS and CREATE INDEX IF NOT EXISTS,
- * making this migration inherently idempotent — safe to re-run on every startup
- * without a checkpoint guard.
- */
-export function migrateMemoryBriefState(database: DrizzleDb): void {
-  const raw = getSqliteFrom(database);
-
-  // -- time_contexts: bounded temporal windows for the brief --
-  raw.exec(/*sql*/ `
-    CREATE TABLE IF NOT EXISTS time_contexts (
-      id TEXT PRIMARY KEY,
-      scope_id TEXT NOT NULL,
-      summary TEXT NOT NULL,
-      source TEXT NOT NULL,
-      active_from INTEGER NOT NULL,
-      active_until INTEGER NOT NULL,
-      created_at INTEGER NOT NULL,
-      updated_at INTEGER NOT NULL
-    )
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_time_contexts_scope_active_until
-    ON time_contexts (scope_id, active_until)
-  `);
-
-  // -- open_loops: unresolved items the brief should surface --
-  raw.exec(/*sql*/ `
-    CREATE TABLE IF NOT EXISTS open_loops (
-      id TEXT PRIMARY KEY,
-      scope_id TEXT NOT NULL,
-      summary TEXT NOT NULL,
-      status TEXT NOT NULL DEFAULT 'open',
-      source TEXT NOT NULL,
-      due_at INTEGER,
-      surfaced_at INTEGER,
-      created_at INTEGER NOT NULL,
-      updated_at INTEGER NOT NULL
-    )
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_open_loops_scope_status_due
-    ON open_loops (scope_id, status, due_at)
-  `);
-}

package/src/memory/migrations/186-memory-archive.ts

@@ -1,109 +0,0 @@
-import type { DrizzleDb } from "../db-connection.js";
-import { getSqliteFrom } from "../db-connection.js";
-
-/**
- * Create the memory archive tables (memory_observations, memory_chunks,
- * memory_episodes) with prefetch indexes on scopeId, conversationId, and
- * createdAt.
- *
- * All statements use IF NOT EXISTS / IF NOT EXISTS guards so the migration
- * is safe to re-run on every startup.
- */
-export function migrateMemoryArchiveTables(database: DrizzleDb): void {
-  const raw = getSqliteFrom(database);
-
-  // -- memory_observations --------------------------------------------------
-  raw.exec(/*sql*/ `
-    CREATE TABLE IF NOT EXISTS memory_observations (
-      id TEXT PRIMARY KEY,
-      scope_id TEXT NOT NULL DEFAULT 'default',
-      conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
-      message_id TEXT REFERENCES messages(id) ON DELETE SET NULL,
-      role TEXT NOT NULL,
-      content TEXT NOT NULL,
-      modality TEXT NOT NULL DEFAULT 'text',
-      source TEXT,
-      created_at INTEGER NOT NULL
-    )
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_observations_scope_id
-    ON memory_observations (scope_id)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_observations_conversation_id
-    ON memory_observations (conversation_id)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_observations_created_at
-    ON memory_observations (created_at)
-  `);
-
-  // -- memory_chunks --------------------------------------------------------
-  raw.exec(/*sql*/ `
-    CREATE TABLE IF NOT EXISTS memory_chunks (
-      id TEXT PRIMARY KEY,
-      scope_id TEXT NOT NULL DEFAULT 'default',
-      observation_id TEXT NOT NULL REFERENCES memory_observations(id) ON DELETE CASCADE,
-      content TEXT NOT NULL,
-      token_estimate INTEGER NOT NULL,
-      content_hash TEXT NOT NULL,
-      created_at INTEGER NOT NULL
-    )
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_chunks_scope_id
-    ON memory_chunks (scope_id)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_chunks_observation_id
-    ON memory_chunks (observation_id)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_chunks_content_hash
-    ON memory_chunks (scope_id, content_hash)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_chunks_created_at
-    ON memory_chunks (created_at)
-  `);
-
-  // -- memory_episodes ------------------------------------------------------
-  raw.exec(/*sql*/ `
-    CREATE TABLE IF NOT EXISTS memory_episodes (
-      id TEXT PRIMARY KEY,
-      scope_id TEXT NOT NULL DEFAULT 'default',
-      conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
-      title TEXT NOT NULL,
-      summary TEXT NOT NULL,
-      token_estimate INTEGER NOT NULL,
-      source TEXT,
-      start_at INTEGER NOT NULL,
-      end_at INTEGER NOT NULL,
-      created_at INTEGER NOT NULL,
-      updated_at INTEGER NOT NULL
-    )
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_episodes_scope_id
-    ON memory_episodes (scope_id)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_episodes_conversation_id
-    ON memory_episodes (conversation_id)
-  `);
-
-  raw.exec(/*sql*/ `
-    CREATE INDEX IF NOT EXISTS idx_memory_episodes_created_at
-    ON memory_episodes (created_at)
-  `);
-}

package/src/memory/migrations/187-memory-reducer-checkpoints.ts

@@ -1,19 +0,0 @@
-import type { DrizzleDb } from "../db-connection.js";
-import { getSqliteFrom } from "../db-connection.js";
-
-export function migrateMemoryReducerCheckpoints(database: DrizzleDb): void {
-  const raw = getSqliteFrom(database);
-  const columns = [
-    "memory_reduced_through_message_id TEXT",
-    "memory_dirty_tail_since_message_id TEXT",
-    "memory_last_reduced_at INTEGER",
-  ];
-
-  for (const column of columns) {
-    try {
-      raw.exec(`ALTER TABLE conversations ADD COLUMN ${column}`);
-    } catch {
-      // Column already exists — nothing to do.
-    }
-  }
-}

package/src/memory/reducer-scheduler.ts

@@ -1,242 +0,0 @@
-/**
- * Reducer scheduler — synchronous pre-switch/create reduction of the most
- * recently updated dirty conversation.
- *
- * When the user switches conversations or starts a new one, we want the
- * *previous* conversation's memory to be reduced before the next memory
- * read. This module exposes {@link reduceBeforeSwitch} which:
- *
- *   1. Finds the single most recently updated dirty conversation (excluding
- *      the target conversation).
- *   2. Runs the same reduction pipeline the background job uses (load
- *      unreduced messages, call {@link runReducer}, apply via
- *      {@link applyReducerResult}).
- *   3. Awaits the result so the caller can proceed knowing memory is fresh.
- *
- * If no eligible dirty conversation exists, the function returns immediately.
- */
-
-import { and, asc, desc, eq, gte, isNotNull, ne } from "drizzle-orm";
-
-import { getLogger } from "../util/logger.js";
-import { type ConversationRow, getConversation } from "./conversation-crud.js";
-import { getDb } from "./db.js";
-import { type ReducerPromptInput, runReducer } from "./reducer.js";
-import {
-  applyReducerResult,
-  getActiveOpenLoops,
-  getActiveTimeContexts,
-} from "./reducer-store.js";
-import { EMPTY_REDUCER_RESULT } from "./reducer-types.js";
-import { conversations, messages } from "./schema.js";
-
-const log = getLogger("reducer-scheduler");
-
-// ── Internal helpers ────────────────────────────────────────────────
-
-interface MessageRow {
-  id: string;
-  role: string;
-  content: string;
-  createdAt: number;
-}
-
-/**
- * Find the single most recently updated dirty conversation, excluding
- * the target conversation. Returns the conversation ID or null if none.
- */
-export function findMostRecentDirtyConversation(
-  excludeConversationId: string,
-): string | null {
-  const db = getDb();
-  const row = db
-    .select({ id: conversations.id })
-    .from(conversations)
-    .where(
-      and(
-        isNotNull(conversations.memoryDirtyTailSinceMessageId),
-        ne(conversations.id, excludeConversationId),
-      ),
-    )
-    .orderBy(desc(conversations.updatedAt))
-    .limit(1)
-    .get();
-
-  return row?.id ?? null;
-}
-
-/**
- * Load messages from `dirtyTailMessageId` onward (inclusive), ordered by
- * createdAt ascending.
- */
-function loadUnreducedMessages(
-  conversationId: string,
-  dirtyTailMessageId: string,
-): MessageRow[] {
-  const db = getDb();
-
-  const tailMessage = db
-    .select({ createdAt: messages.createdAt })
-    .from(messages)
-    .where(eq(messages.id, dirtyTailMessageId))
-    .get();
-
-  if (!tailMessage) {
-    return [];
-  }
-
-  return db
-    .select({
-      id: messages.id,
-      role: messages.role,
-      content: messages.content,
-      createdAt: messages.createdAt,
-    })
-    .from(messages)
-    .where(
-      and(
-        eq(messages.conversationId, conversationId),
-        gte(messages.createdAt, tailMessage.createdAt),
-      ),
-    )
-    .orderBy(asc(messages.createdAt))
-    .all();
-}
-
-/**
- * Build the `newMessages` array for the reducer input, optionally
- * prepending the conversation's contextSummary as a synthetic system message.
- */
-function buildNewMessages(
-  conversation: ConversationRow,
-  unreducedMessages: MessageRow[],
-): Array<{ role: string; content: string }> {
-  const result: Array<{ role: string; content: string }> = [];
-
-  if (conversation.contextSummary) {
-    result.push({
-      role: "system",
-      content: `[Prior context summary] ${conversation.contextSummary}`,
-    });
-  }
-
-  for (const msg of unreducedMessages) {
-    result.push({ role: msg.role, content: msg.content });
-  }
-
-  return result;
-}
-
-// ── Public API ──────────────────────────────────────────────────────
-
-/**
- * Reduce the most recently updated dirty conversation (excluding
- * `targetConversationId`) before a conversation switch or create.
- *
- * This runs the full reduction pipeline synchronously (awaiting the
- * provider call) so the caller can proceed knowing memory is fresh.
- *
- * Returns the conversation ID that was reduced, or null if none were eligible.
- */
-export async function reduceBeforeSwitch(
-  targetConversationId: string,
-): Promise<string | null> {
-  const dirtyConversationId =
-    findMostRecentDirtyConversation(targetConversationId);
-
-  if (!dirtyConversationId) {
-    return null;
-  }
-
-  const conversation = getConversation(dirtyConversationId);
-  if (!conversation) {
-    return null;
-  }
-
-  const dirtyTailMessageId = conversation.memoryDirtyTailSinceMessageId;
-  if (!dirtyTailMessageId) {
-    return null;
-  }
-
-  // ── Load unreduced messages ──────────────────────────────────
-  const unreducedMessages = loadUnreducedMessages(
-    dirtyConversationId,
-    dirtyTailMessageId,
-  );
-
-  if (unreducedMessages.length === 0) {
-    log.debug(
-      { conversationId: dirtyConversationId, dirtyTailMessageId },
-      "No messages found from dirty tail — nothing to reduce on switch",
-    );
-    return null;
-  }
-
-  // ── Load active brief-state context ──────────────────────────
-  const scopeId = conversation.memoryScopeId;
-  const now = Date.now();
-
-  const existingTimeContexts = getActiveTimeContexts(scopeId, now);
-  const existingOpenLoops = getActiveOpenLoops(scopeId);
-
-  // ── Build reducer input ──────────────────────────────────────
-  const newMessages = buildNewMessages(conversation, unreducedMessages);
-
-  const reducerInput: ReducerPromptInput = {
-    conversationId: dirtyConversationId,
-    newMessages,
-    existingTimeContexts: existingTimeContexts.map((tc) => ({
-      id: tc.id,
-      summary: tc.summary,
-    })),
-    existingOpenLoops: existingOpenLoops.map((ol) => ({
-      id: ol.id,
-      summary: ol.summary,
-      status: ol.status,
-    })),
-    nowMs: now,
-    scopeId,
-  };
-
-  // ── Run the reducer ──────────────────────────────────────────
-  try {
-    const result = await runReducer(reducerInput);
-
-    if (result === EMPTY_REDUCER_RESULT) {
-      log.debug(
-        { conversationId: dirtyConversationId },
-        "Reducer returned empty result on switch — not advancing checkpoint",
-      );
-      return null;
-    }
-
-    // ── Apply result transactionally ───────────────────────────
-    const lastMessage = unreducedMessages[unreducedMessages.length - 1];
-    applyReducerResult({
-      result,
-      conversationId: dirtyConversationId,
-      scopeId,
-      reducedThroughMessageId: lastMessage.id,
-      now,
-    });
-
-    log.info(
-      {
-        conversationId: dirtyConversationId,
-        reducedThroughMessageId: lastMessage.id,
-        messageCount: unreducedMessages.length,
-        timeContextOps: result.timeContexts.length,
-        openLoopOps: result.openLoops.length,
-      },
-      "Pre-switch memory reduction completed",
-    );
-
-    return dirtyConversationId;
-  } catch (err) {
-    log.warn(
-      { err, conversationId: dirtyConversationId },
-      "Pre-switch memory reduction failed — continuing with switch",
-    );
-    return null;
-  }
-}