@vellumai/assistant 0.5.2 → 0.5.4

package/src/memory/migrations/188-schedule-quiet-flag.ts

@@ -0,0 +1,13 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+
+export function migrateScheduleQuietFlag(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  try {
+    raw.exec(
+      `ALTER TABLE cron_jobs ADD COLUMN quiet INTEGER NOT NULL DEFAULT 0`,
+    );
+  } catch {
+    // Column already exists — nothing to do.
+  }
+}

package/src/memory/migrations/index.ts

@@ -126,6 +126,10 @@ export { migrateRenameThreadStartersCheckpoints } from "./181-rename-thread-star
 export { migrateOAuthProvidersDisplayMetadata } from "./182-oauth-providers-display-metadata.js";
 export { migrateConversationForkLineage } from "./183-add-conversation-fork-lineage.js";
 export { migrateLlmRequestLogProvider } from "./184-llm-request-log-provider.js";
+export { migrateMemoryBriefState } from "./185-memory-brief-state.js";
+export { migrateMemoryArchiveTables } from "./186-memory-archive.js";
+export { migrateMemoryReducerCheckpoints } from "./187-memory-reducer-checkpoints.js";
+export { migrateScheduleQuietFlag } from "./188-schedule-quiet-flag.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/qdrant-client.ts

@@ -20,7 +20,7 @@ export interface QdrantClientConfig {
 }
 
 export interface QdrantPointPayload {
-  target_type: "segment" | "item" | "summary" | "media";
+  target_type: "segment" | "item" | "summary" | "observation" | "chunk" | "episode" | "media";
   target_id: string;
   text: string;
   kind?: string;
@@ -230,7 +230,7 @@ export class VellumQdrantClient {
   }
 
   async upsert(
-    targetType: "segment" | "item" | "summary" | "media",
+    targetType: "segment" | "item" | "summary" | "observation" | "chunk" | "episode" | "media",
     targetId: string,
     vector: number[],
     payload: Omit<QdrantPointPayload, "target_type" | "target_id">,
@@ -324,8 +324,11 @@ export class VellumQdrantClient {
   async searchWithFilter(
     vector: number[],
     limit: number,
-    targetTypes: Array<"segment" | "item" | "summary" | "media">,
+    targetTypes: Array<
+      "segment" | "item" | "summary" | "media" | "chunk" | "episode"
+    >,
     excludeMessageIds?: string[],
+    scopeIds?: string[],
   ): Promise<QdrantSearchResult[]> {
     const mustConditions: Array<Record<string, unknown>> = [
       {
@@ -346,12 +349,24 @@ export class VellumQdrantClient {
           },
           {
             key: "target_type",
-            match: { any: ["segment", "summary", "media"] },
+            match: { any: ["segment", "summary", "media", "chunk"] },
           },
         ],
       });
     }
 
+    // Scope filtering: accept points whose memory_scope_id matches one of the
+    // allowed scopes, OR points that lack the field entirely (legacy data).
+    // Post-query DB filtering remains as defense-in-depth for legacy points.
+    if (scopeIds && scopeIds.length > 0) {
+      mustConditions.push({
+        should: [
+          { key: "memory_scope_id", match: { any: scopeIds } },
+          { is_empty: { key: "memory_scope_id" } },
+        ],
+      });
+    }
+
     const mustNotConditions: Array<Record<string, unknown>> = [
       { key: "_meta", match: { value: true } },
     ];
@@ -561,6 +576,10 @@ export class VellumQdrantClient {
         field_name: "modality",
         field_schema: "keyword",
       }),
+      this.client.createPayloadIndex(this.collection, {
+        field_name: "memory_scope_id",
+        field_schema: "keyword",
+      }),
     ]);
   }
 

package/src/memory/reducer-scheduler.ts

@@ -0,0 +1,242 @@
+/**
+ * Reducer scheduler — synchronous pre-switch/create reduction of the most
+ * recently updated dirty conversation.
+ *
+ * When the user switches conversations or starts a new one, we want the
+ * *previous* conversation's memory to be reduced before the next memory
+ * read. This module exposes {@link reduceBeforeSwitch} which:
+ *
+ *   1. Finds the single most recently updated dirty conversation (excluding
+ *      the target conversation).
+ *   2. Runs the same reduction pipeline the background job uses (load
+ *      unreduced messages, call {@link runReducer}, apply via
+ *      {@link applyReducerResult}).
+ *   3. Awaits the result so the caller can proceed knowing memory is fresh.
+ *
+ * If no eligible dirty conversation exists, the function returns immediately.
+ */
+
+import { and, asc, desc, eq, gte, isNotNull, ne } from "drizzle-orm";
+
+import { getLogger } from "../util/logger.js";
+import { type ConversationRow, getConversation } from "./conversation-crud.js";
+import { getDb } from "./db.js";
+import { type ReducerPromptInput, runReducer } from "./reducer.js";
+import {
+  applyReducerResult,
+  getActiveOpenLoops,
+  getActiveTimeContexts,
+} from "./reducer-store.js";
+import { EMPTY_REDUCER_RESULT } from "./reducer-types.js";
+import { conversations, messages } from "./schema.js";
+
+const log = getLogger("reducer-scheduler");
+
+// ── Internal helpers ────────────────────────────────────────────────
+
+interface MessageRow {
+  id: string;
+  role: string;
+  content: string;
+  createdAt: number;
+}
+
+/**
+ * Find the single most recently updated dirty conversation, excluding
+ * the target conversation. Returns the conversation ID or null if none.
+ */
+export function findMostRecentDirtyConversation(
+  excludeConversationId: string,
+): string | null {
+  const db = getDb();
+  const row = db
+    .select({ id: conversations.id })
+    .from(conversations)
+    .where(
+      and(
+        isNotNull(conversations.memoryDirtyTailSinceMessageId),
+        ne(conversations.id, excludeConversationId),
+      ),
+    )
+    .orderBy(desc(conversations.updatedAt))
+    .limit(1)
+    .get();
+
+  return row?.id ?? null;
+}
+
+/**
+ * Load messages from `dirtyTailMessageId` onward (inclusive), ordered by
+ * createdAt ascending.
+ */
+function loadUnreducedMessages(
+  conversationId: string,
+  dirtyTailMessageId: string,
+): MessageRow[] {
+  const db = getDb();
+
+  const tailMessage = db
+    .select({ createdAt: messages.createdAt })
+    .from(messages)
+    .where(eq(messages.id, dirtyTailMessageId))
+    .get();
+
+  if (!tailMessage) {
+    return [];
+  }
+
+  return db
+    .select({
+      id: messages.id,
+      role: messages.role,
+      content: messages.content,
+      createdAt: messages.createdAt,
+    })
+    .from(messages)
+    .where(
+      and(
+        eq(messages.conversationId, conversationId),
+        gte(messages.createdAt, tailMessage.createdAt),
+      ),
+    )
+    .orderBy(asc(messages.createdAt))
+    .all();
+}
+
+/**
+ * Build the `newMessages` array for the reducer input, optionally
+ * prepending the conversation's contextSummary as a synthetic system message.
+ */
+function buildNewMessages(
+  conversation: ConversationRow,
+  unreducedMessages: MessageRow[],
+): Array<{ role: string; content: string }> {
+  const result: Array<{ role: string; content: string }> = [];
+
+  if (conversation.contextSummary) {
+    result.push({
+      role: "system",
+      content: `[Prior context summary] ${conversation.contextSummary}`,
+    });
+  }
+
+  for (const msg of unreducedMessages) {
+    result.push({ role: msg.role, content: msg.content });
+  }
+
+  return result;
+}
+
+// ── Public API ──────────────────────────────────────────────────────
+
+/**
+ * Reduce the most recently updated dirty conversation (excluding
+ * `targetConversationId`) before a conversation switch or create.
+ *
+ * This runs the full reduction pipeline synchronously (awaiting the
+ * provider call) so the caller can proceed knowing memory is fresh.
+ *
+ * Returns the conversation ID that was reduced, or null if none were eligible.
+ */
+export async function reduceBeforeSwitch(
+  targetConversationId: string,
+): Promise<string | null> {
+  const dirtyConversationId =
+    findMostRecentDirtyConversation(targetConversationId);
+
+  if (!dirtyConversationId) {
+    return null;
+  }
+
+  const conversation = getConversation(dirtyConversationId);
+  if (!conversation) {
+    return null;
+  }
+
+  const dirtyTailMessageId = conversation.memoryDirtyTailSinceMessageId;
+  if (!dirtyTailMessageId) {
+    return null;
+  }
+
+  // ── Load unreduced messages ──────────────────────────────────
+  const unreducedMessages = loadUnreducedMessages(
+    dirtyConversationId,
+    dirtyTailMessageId,
+  );
+
+  if (unreducedMessages.length === 0) {
+    log.debug(
+      { conversationId: dirtyConversationId, dirtyTailMessageId },
+      "No messages found from dirty tail — nothing to reduce on switch",
+    );
+    return null;
+  }
+
+  // ── Load active brief-state context ──────────────────────────
+  const scopeId = conversation.memoryScopeId;
+  const now = Date.now();
+
+  const existingTimeContexts = getActiveTimeContexts(scopeId, now);
+  const existingOpenLoops = getActiveOpenLoops(scopeId);
+
+  // ── Build reducer input ──────────────────────────────────────
+  const newMessages = buildNewMessages(conversation, unreducedMessages);
+
+  const reducerInput: ReducerPromptInput = {
+    conversationId: dirtyConversationId,
+    newMessages,
+    existingTimeContexts: existingTimeContexts.map((tc) => ({
+      id: tc.id,
+      summary: tc.summary,
+    })),
+    existingOpenLoops: existingOpenLoops.map((ol) => ({
+      id: ol.id,
+      summary: ol.summary,
+      status: ol.status,
+    })),
+    nowMs: now,
+    scopeId,
+  };
+
+  // ── Run the reducer ──────────────────────────────────────────
+  try {
+    const result = await runReducer(reducerInput);
+
+    if (result === EMPTY_REDUCER_RESULT) {
+      log.debug(
+        { conversationId: dirtyConversationId },
+        "Reducer returned empty result on switch — not advancing checkpoint",
+      );
+      return null;
+    }
+
+    // ── Apply result transactionally ───────────────────────────
+    const lastMessage = unreducedMessages[unreducedMessages.length - 1];
+    applyReducerResult({
+      result,
+      conversationId: dirtyConversationId,
+      scopeId,
+      reducedThroughMessageId: lastMessage.id,
+      now,
+    });
+
+    log.info(
+      {
+        conversationId: dirtyConversationId,
+        reducedThroughMessageId: lastMessage.id,
+        messageCount: unreducedMessages.length,
+        timeContextOps: result.timeContexts.length,
+        openLoopOps: result.openLoops.length,
+      },
+      "Pre-switch memory reduction completed",
+    );
+
+    return dirtyConversationId;
+  } catch (err) {
+    log.warn(
+      { err, conversationId: dirtyConversationId },
+      "Pre-switch memory reduction failed — continuing with switch",
+    );
+    return null;
+  }
+}

package/src/memory/reducer-store.ts

@@ -0,0 +1,271 @@
+/**
+ * Reducer store — transactional application of reducer results to brief-state
+ * tables (time_contexts, open_loops) and conversation reducer checkpoints.
+ *
+ * The `applyReducerResult` helper is the single entry point for persisting
+ * reducer output. It runs all upserts, resolves, and checkpoint advances
+ * inside a single SQLite transaction so the DB is never left in a
+ * partially-applied state.
+ *
+ * Archive writes are intentionally out of scope — they have their own
+ * lifecycle and can be tested independently.
+ */
+
+import { and, eq, gt } from "drizzle-orm";
+import { v4 as uuid } from "uuid";
+
+import { getLogger } from "../util/logger.js";
+import { getDb } from "./db.js";
+import type { ReducerResult } from "./reducer-types.js";
+import { conversations, messages, openLoops, timeContexts } from "./schema.js";
+
+const log = getLogger("reducer-store");
+
+// ── Read helpers ─────────────────────────────────────────────────────
+
+/**
+ * Return all active (non-expired) time contexts for a memory scope.
+ * "Active" means `activeUntil` is in the future relative to `now`.
+ */
+export function getActiveTimeContexts(
+  scopeId: string,
+  now: number = Date.now(),
+): Array<{
+  id: string;
+  summary: string;
+  activeFrom: number;
+  activeUntil: number;
+}> {
+  const db = getDb();
+  return db
+    .select({
+      id: timeContexts.id,
+      summary: timeContexts.summary,
+      activeFrom: timeContexts.activeFrom,
+      activeUntil: timeContexts.activeUntil,
+    })
+    .from(timeContexts)
+    .where(
+      and(eq(timeContexts.scopeId, scopeId), gt(timeContexts.activeUntil, now)),
+    )
+    .all();
+}
+
+/**
+ * Return all open loops for a memory scope.
+ */
+export function getActiveOpenLoops(
+  scopeId: string,
+): Array<{ id: string; summary: string; status: string }> {
+  const db = getDb();
+  return db
+    .select({
+      id: openLoops.id,
+      summary: openLoops.summary,
+      status: openLoops.status,
+    })
+    .from(openLoops)
+    .where(and(eq(openLoops.scopeId, scopeId), eq(openLoops.status, "open")))
+    .all();
+}
+
+// ── Brief-compiler helper ────────────────────────────────────────────
+
+/**
+ * Update the `surfaced_at` timestamp on a single open loop.
+ *
+ * Called by the brief compiler after resurfacing a low-salience loop
+ * so it is not immediately resurfaced again on the next turn.
+ */
+export function updateLastSurfacedAt(loopId: string, surfacedAt: number): void {
+  const db = getDb();
+  db.update(openLoops)
+    .set({ surfacedAt, updatedAt: surfacedAt })
+    .where(eq(openLoops.id, loopId))
+    .run();
+}
+
+// ── Transactional apply ──────────────────────────────────────────────
+
+export interface ApplyReducerResultParams {
+  /** The validated reducer result to persist. */
+  result: ReducerResult;
+  /** Conversation that was reduced. */
+  conversationId: string;
+  /** Memory scope for new rows (e.g. assistant instance ID). */
+  scopeId: string;
+  /** ID of the last message that was included in this reducer run. */
+  reducedThroughMessageId: string;
+  /** Current timestamp in epoch ms (injectable for testing). */
+  now?: number;
+}
+
+/**
+ * Atomically apply a reducer result to the database.
+ *
+ * Within a single transaction this function:
+ *   1. Upserts time_contexts (create / update / resolve)
+ *   2. Upserts open_loops (create / update / resolve)
+ *   3. Advances the conversation's reducer checkpoint columns
+ *   4. Clears `memoryDirtyTailSinceMessageId` when the conversation is
+ *      fully caught up (no messages exist after `reducedThroughMessageId`)
+ *
+ * Archive candidates in the result are intentionally ignored — they are
+ * handled by a separate pipeline.
+ *
+ * The function is idempotent: applying the same result twice leaves the
+ * database in the same state. Create operations use deterministic IDs
+ * derived from the reducer output position so re-application produces
+ * the same rows.
+ */
+export function applyReducerResult(params: ApplyReducerResultParams): void {
+  const {
+    result,
+    conversationId,
+    scopeId,
+    reducedThroughMessageId,
+    now = Date.now(),
+  } = params;
+
+  const db = getDb();
+
+  db.transaction((tx) => {
+    // ── 1. Time contexts ───────────────────────────────────────────
+    for (let i = 0; i < result.timeContexts.length; i++) {
+      const op = result.timeContexts[i];
+
+      if (op.action === "create") {
+        const id = uuid();
+        tx.insert(timeContexts)
+          .values({
+            id,
+            scopeId,
+            summary: op.summary,
+            source: op.source,
+            activeFrom: op.activeFrom,
+            activeUntil: op.activeUntil,
+            createdAt: now,
+            updatedAt: now,
+          })
+          .run();
+      } else if (op.action === "update") {
+        const setFields: Record<string, unknown> = { updatedAt: now };
+        if (op.summary !== undefined) setFields.summary = op.summary;
+        if (op.activeFrom !== undefined) setFields.activeFrom = op.activeFrom;
+        if (op.activeUntil !== undefined)
+          setFields.activeUntil = op.activeUntil;
+
+        tx.update(timeContexts)
+          .set(setFields)
+          .where(eq(timeContexts.id, op.id))
+          .run();
+      } else {
+        // resolve — delete the row (resolved time contexts are no longer relevant)
+        tx.delete(timeContexts).where(eq(timeContexts.id, op.id)).run();
+      }
+    }
+
+    // ── 2. Open loops ──────────────────────────────────────────────
+    for (let i = 0; i < result.openLoops.length; i++) {
+      const op = result.openLoops[i];
+
+      if (op.action === "create") {
+        const id = uuid();
+        tx.insert(openLoops)
+          .values({
+            id,
+            scopeId,
+            summary: op.summary,
+            source: op.source,
+            status: "open",
+            dueAt: op.dueAt ?? null,
+            createdAt: now,
+            updatedAt: now,
+          })
+          .run();
+      } else if (op.action === "update") {
+        const setFields: Record<string, unknown> = { updatedAt: now };
+        if (op.summary !== undefined) setFields.summary = op.summary;
+        if (op.dueAt !== undefined) setFields.dueAt = op.dueAt;
+
+        tx.update(openLoops)
+          .set(setFields)
+          .where(eq(openLoops.id, op.id))
+          .run();
+      } else {
+        // resolve — mark status (resolved | expired)
+        tx.update(openLoops)
+          .set({ status: op.status, updatedAt: now })
+          .where(eq(openLoops.id, op.id))
+          .run();
+      }
+    }
+
+    // ── 3. Advance reducer checkpoint ──────────────────────────────
+    //
+    // Check whether the conversation is fully caught up: no messages
+    // exist after the one we just reduced through. If caught up, clear
+    // the dirty tail marker so the reducer knows there's nothing left
+    // to process.
+    const laterMessage = tx
+      .select({ id: messages.id })
+      .from(messages)
+      .where(
+        and(
+          eq(messages.conversationId, conversationId),
+          gt(
+            messages.createdAt,
+            getMessageCreatedAt(tx, reducedThroughMessageId),
+          ),
+        ),
+      )
+      .limit(1)
+      .get();
+
+    const isCaughtUp = !laterMessage;
+
+    const checkpointUpdate: Record<string, unknown> = {
+      memoryReducedThroughMessageId: reducedThroughMessageId,
+      memoryLastReducedAt: now,
+    };
+
+    if (isCaughtUp) {
+      checkpointUpdate.memoryDirtyTailSinceMessageId = null;
+    }
+
+    tx.update(conversations)
+      .set(checkpointUpdate)
+      .where(eq(conversations.id, conversationId))
+      .run();
+
+    log.debug(
+      {
+        conversationId,
+        reducedThroughMessageId,
+        timeContextOps: result.timeContexts.length,
+        openLoopOps: result.openLoops.length,
+        isCaughtUp,
+      },
+      "Applied reducer result",
+    );
+  });
+}
+
+// ── Internal helpers ─────────────────────────────────────────────────
+
+/**
+ * Get the createdAt timestamp for a message by ID.
+ * Returns 0 if the message doesn't exist (which means the gt() comparison
+ * will match all messages — safe fallback that prevents clearing dirty tail).
+ */
+function getMessageCreatedAt(
+  tx: Parameters<Parameters<ReturnType<typeof getDb>["transaction"]>[0]>[0],
+  messageId: string,
+): number {
+  const row = tx
+    .select({ createdAt: messages.createdAt })
+    .from(messages)
+    .where(eq(messages.id, messageId))
+    .get();
+  return row?.createdAt ?? 0;
+}