@vellumai/assistant 0.8.6 → 0.8.7-dev.202606052118.34cd356

package/src/memory/jobs-store.ts

@@ -44,6 +44,8 @@ export type MemoryJobType =
   | "memory_v2_migrate"
   | "memory_v2_reembed"
   | "memory_v2_activation_recompute"
+  | "memory_v3_maintain"
+  // Retired/legacy — no live handler; persisted rows drop via LEGACY_JOB_TYPES.
   | "memory_v3_consolidate"
   | "memory_v3_index_maintenance"
   | "memory_v3_edge_learning"
@@ -70,13 +72,18 @@ export const SLOW_LLM_JOB_TYPES: MemoryJobType[] = [
   "generate_conversation_starters",
   "memory_v2_sweep",
   "memory_v2_consolidate",
-  "memory_v3_consolidate",
+  "memory_v3_maintain",
   "memory_v2_migrate",
   "memory_retrospective",
   "backfill",
   "graph_bootstrap",
 ];
 
+export const MEMORY_V2_CONSOLIDATION_JOB_TRIGGERS = {
+  automatic: "automatic",
+  manual: "manual",
+} as const;
+
 /** Returns `false` only when `config.memory.enabled` is explicitly `false`; defaults to `true` on missing config or load errors. */
 export function isMemoryEnabled(): boolean {
   try {
@@ -335,6 +342,34 @@ export function hasActiveJobOfType(type: MemoryJobType): boolean {
   );
 }
 
+export function isAutomaticConsolidationJob(job: MemoryJob): boolean {
+  return job.payload.trigger !== MEMORY_V2_CONSOLIDATION_JOB_TRIGGERS.manual;
+}
+
+export function cancelPendingAutomaticConsolidationJobs(): number {
+  const db = getDb();
+  db.update(memoryJobs)
+    .set({
+      status: "failed",
+      lastError: "automatic_consolidation_disabled",
+      updatedAt: Date.now(),
+    })
+    .where(
+      and(
+        eq(memoryJobs.type, "memory_v2_consolidate"),
+        eq(memoryJobs.status, "pending"),
+        or(
+          sql`json_extract(${memoryJobs.payload}, '$.trigger') = ${MEMORY_V2_CONSOLIDATION_JOB_TRIGGERS.automatic}`,
+          // Legacy rows predate trigger markers and are indistinguishable from
+          // automatic enqueues, so disabling the schedule treats them as auto.
+          sql`json_extract(${memoryJobs.payload}, '$.trigger') IS NULL`,
+        ),
+      ),
+    )
+    .run();
+  return rawChanges();
+}
+
 export function enqueuePruneOldLlmRequestLogsJob(retentionMs?: number): string {
   const db = getDb();
   const existing = db

package/src/memory/jobs-worker.ts

@@ -1,5 +1,6 @@
 import { join } from "node:path";
 
+import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
 import { getConfig } from "../config/loader.js";
 import type { AssistantConfig } from "../config/types.js";
 import {
@@ -7,6 +8,7 @@ import {
   diskPressureBackgroundSkipLogFields,
   shouldLogDiskPressureBackgroundSkip,
 } from "../daemon/disk-pressure-background-gate.js";
+import { maintainJob as memoryV3MaintainJob } from "../plugins/defaults/memory-v3-shadow/maintain-job.js";
 import { getLogger } from "../util/logger.js";
 import { getWorkspaceDir } from "../util/platform.js";
 import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
@@ -66,6 +68,8 @@ import {
   failMemoryJob,
   failStalledJobs,
   hasActiveJobOfType,
+  isAutomaticConsolidationJob,
+  MEMORY_V2_CONSOLIDATION_JOB_TRIGGERS,
   type MemoryJob,
   type MemoryJobType,
   resetRunningJobsToPending,
@@ -84,12 +88,13 @@ import {
   memoryV2ConsolidateJob,
 } from "./v2/consolidation-job.js";
 import { memoryV2SweepJob } from "./v2/sweep-job.js";
-import { memoryV3ConsolidateJob } from "./v3/consolidation-job.js";
-import { memoryV3EdgeLearningJob } from "./v3/edge-learning-job.js";
-import { memoryV3IndexMaintenanceJob } from "./v3/maintenance.js";
 
 const log = getLogger("memory-jobs-worker");
 
+const AUTOMATIC_CONSOLIDATION_JOB_PAYLOAD = {
+  trigger: MEMORY_V2_CONSOLIDATION_JOB_TRIGGERS.automatic,
+} as const;
+
 /**
  * V1 job types that read or write the v1 Qdrant collection via
  * `getQdrantClient()`. When `memory.v2.enabled` is true, the v1 client is
@@ -126,6 +131,11 @@ const LEGACY_JOB_TYPES = new Set([
   "generate_capability_cards",
   "generate_thread_starters",
   "memory_v2_rebuild_edges",
+  // Retired memory-v3 job types — handlers were removed in the v3 rip. Kept
+  // here so pre-upgrade rows enqueued by the old write path drop gracefully.
+  "memory_v3_consolidate",
+  "memory_v3_index_maintenance",
+  "memory_v3_edge_learning",
 ]);
 
 export const POLL_INTERVAL_MIN_MS = 1_500;
@@ -605,18 +615,18 @@ async function processJob(
       await memoryV2SweepJob(job, config);
       return;
     case "memory_v2_consolidate":
+      if (
+        isAutomaticConsolidationJob(job) &&
+        !config.memory.v2.consolidation_enabled
+      ) {
+        log.info(
+          { jobId: job.id },
+          "Skipping automatic memory v2 consolidation because scheduled consolidation is disabled",
+        );
+        return;
+      }
       await memoryV2ConsolidateJob(job, config);
       return;
-    case "memory_v3_consolidate":
-      await memoryV3ConsolidateJob(job, config);
-      return;
-    case "memory_v3_index_maintenance":
-      await memoryV3IndexMaintenanceJob(job);
-      return;
-    case "memory_v3_edge_learning":
-      // Fast lane: bounded DB work (decay + reinforce + read), no LLM.
-      memoryV3EdgeLearningJob(job);
-      return;
     case "memory_v2_migrate":
       await memoryV2MigrateJob(job, config);
       return;
@@ -626,6 +636,9 @@ async function processJob(
     case "memory_v2_activation_recompute":
       await memoryV2ActivationRecomputeJob(job, config);
       return;
+    case "memory_v3_maintain":
+      await memoryV3MaintainJob(job, config);
+      return;
     case "memory_retrospective":
       await memoryRetrospectiveJob(job, config);
       return;
@@ -688,6 +701,12 @@ const GRAPH_DECAY_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
 const GRAPH_CONSOLIDATE_INTERVAL_MS = 4 * 60 * 60 * 1000; // 4 hours
 const GRAPH_PATTERN_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 1 day
 const GRAPH_NARRATIVE_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week
+// Backstop cadence for v3 self-maintenance. The primary trigger is the
+// post-consolidation follow-up (see `consolidation-job.ts`); this interval only
+// covers the case where that follow-up is missed (enqueue failure, or a v3-on
+// install with v2 consolidation disabled). A conservative cadence is fine since
+// the maintenance pass is idempotent and cheap when there's nothing to do.
+const GRAPH_V3_MAINTAIN_INTERVAL_MS = 6 * 60 * 60 * 1000; // 6 hours
 
 export const GRAPH_MAINTENANCE_CHECKPOINTS = {
   decay: "graph_maintenance:decay:last_run",
@@ -695,7 +714,7 @@ export const GRAPH_MAINTENANCE_CHECKPOINTS = {
   patternScan: "graph_maintenance:pattern_scan:last_run",
   narrative: "graph_maintenance:narrative:last_run",
   memoryV2Consolidate: "memory_v2_consolidate_last_run",
-  memoryV3Consolidate: "memory_v3_consolidate_last_run",
+  memoryV3Maintain: "memory_v3_maintain_last_run",
 } as const;
 
 /**
@@ -707,15 +726,9 @@ export const GRAPH_MAINTENANCE_CHECKPOINTS = {
  *   - v2 inactive → the four v1 entries (decay, consolidate, pattern_scan,
  *     narrative) are scheduled instead.
  *
- * **Buffer-drainer retarget (v2 vs v3).** The `memory/buffer.md` is shared, so
- * exactly one consolidator may own the drain at a time. When
- * `memory.v3.write.enabled` is on, the v3 consolidator (`memory_v3_consolidate`)
- * is scheduled INSTEAD of `memory_v2_consolidate` — same shared buffer +
- * standing-context files, additionally authored into the v3 tree. When the v3
- * write flag is off (default) the v2 consolidator stays the sole drainer,
- * unchanged. The retarget is a clean conditional, fully reversible via the flag.
- * Concept pages stay the shared canonical store, so the v2 router keeps working
- * off pages v3 writes regardless of which consolidator ran.
+ * The `memory/buffer.md` is shared, so exactly one consolidator owns the drain
+ * at a time. When v2 is active, the v2 consolidator (`memory_v2_consolidate`)
+ * is the sole buffer-drainer.
  *
  * Read/write paths route to v2 when the flag is on, so v1 graph data goes
  * unread; running v1 maintenance alongside v2 is wasted compute and LLM
@@ -727,35 +740,35 @@ export const GRAPH_MAINTENANCE_CHECKPOINTS = {
  * Sweep is intentionally not on this schedule: it is debounced from the
  * live `graph_extract` trigger path (see `indexMessageNow` in `indexer.ts`)
  * so it runs on the same idle/message-count cadence.
+ *
+ * Independently of the v1/v2 split, a flag-gated `memory_v3_maintain` backstop
+ * is appended when a v3 path is active so the topic tree self-heals even if the
+ * primary post-consolidation follow-up enqueue is missed.
  */
 export function maybeEnqueueGraphMaintenanceJobs(
   config: AssistantConfig,
   nowMs = Date.now(),
 ): void {
   const v2Active = config.memory.v2.enabled;
-  const v3WriteActive = config.memory.v3.write.enabled;
-
-  // The single buffer-drainer entry for the v2-active branch: v3 when the v3
-  // write flag owns the drain, v2 otherwise. Same shared buffer either way.
-  const consolidateEntry = v3WriteActive
-    ? {
-        key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV3Consolidate,
-        intervalMs: config.memory.v3.write.consolidateIntervalMs,
-        jobType: "memory_v3_consolidate" as MemoryJobType,
-      }
-    : {
-        key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV2Consolidate,
-        intervalMs:
-          config.memory.v2.consolidation_interval_hours * 60 * 60 * 1000,
-        jobType: "memory_v2_consolidate" as MemoryJobType,
-      };
+  const v2ConsolidationAutomaticEnabled =
+    v2Active && config.memory.v2.consolidation_enabled;
+
+  // The single buffer-drainer entry for the v2-active branch. Referenced again
+  // below by the size-based trigger.
+  const consolidateEntry = {
+    key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV2Consolidate,
+    intervalMs: config.memory.v2.consolidation_interval_hours * 60 * 60 * 1000,
+    jobType: "memory_v2_consolidate" as MemoryJobType,
+  };
 
   const schedule: Array<{
     key: string;
     intervalMs: number;
     jobType: MemoryJobType;
   }> = v2Active
-    ? [consolidateEntry]
+    ? v2ConsolidationAutomaticEnabled
+      ? [consolidateEntry]
+      : []
     : [
         {
           key: GRAPH_MAINTENANCE_CHECKPOINTS.decay,
@@ -779,11 +792,34 @@ export function maybeEnqueueGraphMaintenanceJobs(
         },
       ];
 
+  // v3 self-maintenance backstop. Orthogonal to the v1/v2 mutual exclusion
+  // above: it owns its own checkpoint and operates on the v3 topic tree, so it
+  // runs under either branch. Gated on the same flags that gate the v3 plugin
+  // so it stays inert when v3 is off. The post-consolidation follow-up in
+  // `consolidation-job.ts` remains the primary trigger; this interval only
+  // self-heals when that follow-up is missed (failed enqueue, or v3-on with v2
+  // consolidation disabled). The job handler itself no-ops when v3 is off, so
+  // this guard is belt-and-suspenders that also avoids a wasted enqueue.
+  if (
+    isAssistantFeatureFlagEnabled("memory-v3-shadow", config) ||
+    isAssistantFeatureFlagEnabled("memory-v3-live", config)
+  ) {
+    schedule.push({
+      key: GRAPH_MAINTENANCE_CHECKPOINTS.memoryV3Maintain,
+      intervalMs: GRAPH_V3_MAINTAIN_INTERVAL_MS,
+      jobType: "memory_v3_maintain",
+    });
+  }
+
   let enqueuedConsolidate = false;
   for (const { key, intervalMs, jobType } of schedule) {
     const lastRun = parseInt(getMemoryCheckpoint(key) ?? "0", 10);
     if (nowMs - lastRun >= intervalMs) {
-      enqueueMemoryJob(jobType, {});
+      const payload =
+        jobType === consolidateEntry.jobType
+          ? AUTOMATIC_CONSOLIDATION_JOB_PAYLOAD
+          : {};
+      enqueueMemoryJob(jobType, payload);
       setMemoryCheckpoint(key, String(nowMs));
       if (jobType === consolidateEntry.jobType) enqueuedConsolidate = true;
     }
@@ -799,14 +835,17 @@ export function maybeEnqueueGraphMaintenanceJobs(
   // buffer stays over threshold, flooding the queue with redundant LLM work.
   const maxLines = config.memory.v2.consolidation_max_buffer_lines;
   if (
-    v2Active &&
+    v2ConsolidationAutomaticEnabled &&
     !enqueuedConsolidate &&
     maxLines !== null &&
     !hasActiveJobOfType(consolidateEntry.jobType)
   ) {
     const bufferPath = join(getWorkspaceDir(), "memory", "buffer.md");
     if (countBufferLines(bufferPath) >= maxLines) {
-      enqueueMemoryJob(consolidateEntry.jobType, {});
+      enqueueMemoryJob(
+        consolidateEntry.jobType,
+        AUTOMATIC_CONSOLIDATION_JOB_PAYLOAD,
+      );
       setMemoryCheckpoint(consolidateEntry.key, String(nowMs));
     }
   }

package/src/memory/llm-request-log-source-clickhouse.ts

@@ -259,36 +259,6 @@ export class ClickHouseLlmRequestLogSource implements LlmRequestLogSource {
     return rows.map((r) => this.toLogRow(r));
   }
 
-  async getPreviousNonCompactionCallCreatedAt(
-    conversationId: string,
-    beforeCreatedAt: number,
-  ): Promise<number | null> {
-    const aid = await this.assistantId();
-    // "Non-compactionAgent" includes empty-string `call_site`, which is
-    // what the CH mirror writes for pre-migration-264 rows whose SQLite
-    // value was NULL (CH columns are `DEFAULT ''`, not Nullable — see
-    // `toLogRow`). The local store treats NULL `callSite` the same way,
-    // so the predicate stays in sync.
-    const sql = `SELECT
-        toUnixTimestamp64Milli(created_at) AS created_at
-      FROM ${this.tableRef()}
-      WHERE assistant_id = {assistant_id:String}
-        AND conversation_id = {conversation_id:String}
-        AND created_at < fromUnixTimestamp64Milli({before_created_at:Int64})
-        AND call_site != {call_site:String}
-      ORDER BY created_at DESC, id DESC
-      LIMIT 1
-      FORMAT JSONEachRow`;
-    const rows = await this.exec(sql, {
-      assistant_id: aid,
-      conversation_id: conversationId,
-      call_site: "compactionAgent",
-      before_created_at: String(beforeCreatedAt),
-    });
-    const value = rows[0]?.created_at;
-    return value === undefined ? null : Number(value);
-  }
-
   private async selectByMessageIds(ids: string[]): Promise<LogRow[]> {
     if (ids.length === 0) return [];
     const aid = await this.assistantId();
@@ -367,7 +337,11 @@ export class ClickHouseLlmRequestLogSource implements LlmRequestLogSource {
     if (!res.ok) {
       const body = await res.text().catch(() => "");
       log.error(
-        { status: res.status, table: this.config.table, bodySnippet: body.slice(0, 200) },
+        {
+          status: res.status,
+          table: this.config.table,
+          bodySnippet: body.slice(0, 200),
+        },
         "ClickHouse query failed",
       );
       throw new Error(

package/src/memory/llm-request-log-source-local.ts

@@ -11,7 +11,6 @@
 import type { LlmRequestLogSource } from "./llm-request-log-source.js";
 import {
   getCompactionLogsBetween,
-  getPreviousNonCompactionCallCreatedAt,
   getRequestLogById,
   getRequestLogsByConversationId,
   getRequestLogsByMessageId,
@@ -44,14 +43,4 @@ export class LocalLlmRequestLogSource implements LlmRequestLogSource {
       beforeCreatedAt,
     );
   }
-
-  async getPreviousNonCompactionCallCreatedAt(
-    conversationId: string,
-    beforeCreatedAt: number,
-  ): Promise<number | null> {
-    return getPreviousNonCompactionCallCreatedAt(
-      conversationId,
-      beforeCreatedAt,
-    );
-  }
 }

package/src/memory/llm-request-log-source.ts

@@ -44,31 +44,17 @@ export interface LlmRequestLogSource {
    * whose `createdAt` falls in the **open window**
    * `(afterCreatedAt, beforeCreatedAt)`, ordered chronologically.
    *
-   * Drives the Inspector's Compaction tab. The route handler resolves
-   * both bounds: the selected LLM call's `createdAt` (ceiling) and the
-   * previous non-`compactionAgent` call's `createdAt` (floor) — passing
-   * `null` for the floor when the selected call is the first real call
-   * in the conversation. See `getCompactionLogsBetween` and
-   * `getPreviousNonCompactionCallCreatedAt` in `llm-request-log-store.ts`
-   * for the full bound semantics.
+   * Drives the Inspector's Compaction tab. The route handler computes
+   * the window from the turn the selected call belongs to via
+   * `getTurnTimeBounds` in `memory/conversation-crud.ts`; this source
+   * method is bound-agnostic. See `getCompactionLogsBetween` in
+   * `llm-request-log-store.ts` for the SQL.
    */
   getCompactionLogsBetween(
     conversationId: string,
     afterCreatedAt: number | null,
     beforeCreatedAt: number,
   ): Promise<LogRow[]>;
-
-  /**
-   * Find the `createdAt` of the most recent non-`compactionAgent` LLM
-   * call in the conversation strictly before `beforeCreatedAt`, or
-   * `null` when no such call exists. Pairs with
-   * `getCompactionLogsBetween` to bound the compaction trail to the
-   * window between the prior real call and the selected call.
-   */
-  getPreviousNonCompactionCallCreatedAt(
-    conversationId: string,
-    beforeCreatedAt: number,
-  ): Promise<number | null>;
 }
 
 /**
@@ -89,14 +75,12 @@ export async function getLlmRequestLogSource(): Promise<LlmRequestLogSource> {
   const cfg = config.llmRequestLogs ?? { readSource: "local" as const };
 
   if (cfg.readSource === "clickhouse") {
-    const { ClickHouseLlmRequestLogSource } = await import(
-      "./llm-request-log-source-clickhouse.js"
-    );
+    const { ClickHouseLlmRequestLogSource } =
+      await import("./llm-request-log-source-clickhouse.js");
     return new ClickHouseLlmRequestLogSource(cfg.clickhouse);
   }
 
-  const { LocalLlmRequestLogSource } = await import(
-    "./llm-request-log-source-local.js"
-  );
+  const { LocalLlmRequestLogSource } =
+    await import("./llm-request-log-source-local.js");
   return new LocalLlmRequestLogSource();
 }

package/src/memory/llm-request-log-store.ts

@@ -9,8 +9,6 @@ import {
   isNull,
   lt,
   lte,
-  ne,
-  or,
   sql,
 } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
@@ -456,45 +454,6 @@ export function getCompactionLogsBetween(
     .all();
 }
 
-/**
- * Find the `createdAt` of the most recent non-`compactionAgent` LLM
- * call in the conversation strictly before `beforeCreatedAt`, or `null`
- * when no such call exists (i.e. the cutoff is the first real call).
- *
- * Pairs with `getCompactionLogsBetween` to bound the compaction trail
- * to the window between the prior real call and the selected call.
- *
- * "Non-compactionAgent" means `callSite IS NULL OR callSite !=
- * 'compactionAgent'`. NULL rows are pre-migration-264 (no backfill) and
- * are treated as real agent calls — they were `mainAgent` in practice.
- * The OR-with-IS NULL is required because SQL's three-valued logic
- * makes `callSite != 'compactionAgent'` return UNKNOWN (not TRUE) for
- * NULL rows, which would otherwise filter them out.
- */
-export function getPreviousNonCompactionCallCreatedAt(
-  conversationId: string,
-  beforeCreatedAt: number,
-): number | null {
-  const db = getDb();
-  const row = db
-    .select({ createdAt: llmRequestLogs.createdAt })
-    .from(llmRequestLogs)
-    .where(
-      and(
-        eq(llmRequestLogs.conversationId, conversationId),
-        lt(llmRequestLogs.createdAt, beforeCreatedAt),
-        or(
-          isNull(llmRequestLogs.callSite),
-          ne(llmRequestLogs.callSite, "compactionAgent"),
-        ),
-      ),
-    )
-    .orderBy(desc(llmRequestLogs.createdAt), desc(llmRequestLogs.id))
-    .limit(1)
-    .get();
-  return row?.createdAt ?? null;
-}
-
 export function getRequestLogById(logId: string): LogRow | null {
   const db = getDb();
   return (