@vellumai/assistant 0.5.16 → 0.6.0

package/src/memory/jobs-store.ts

@@ -19,7 +19,7 @@ export type MemoryJobType =
   | "extract_items"
   | "extract_entities"
   | "batch_extract"
-  | "cleanup_stale_superseded_items"
+  | "cleanup_stale_superseded_items" // legacy compat — silently dropped by worker (memory_items table dropped)
   | "prune_old_conversations"
   | "backfill_entity_relations"
   | "refresh_weekly_summary"
@@ -33,6 +33,14 @@ export type MemoryJobType =
   | "embed_attachment"
   | "generate_conversation_starters"
   | "journal_carry_forward"
+  | "embed_graph_node"
+  | "graph_extract"
+  | "graph_decay"
+  | "graph_consolidate"
+  | "graph_pattern_scan"
+  | "graph_narrative_refine"
+  | "graph_trigger_embed"
+  | "graph_bootstrap"
   | "generate_capability_cards" // legacy compat — silently dropped by worker (capability cards removed)
   | "generate_thread_starters"; // legacy compat — silently dropped by worker (renamed to generate_conversation_starters)
 
@@ -42,6 +50,8 @@ const EMBED_JOB_TYPES: MemoryJobType[] = [
   "embed_summary",
   "embed_media",
   "embed_attachment",
+  "embed_graph_node",
+  "graph_trigger_embed",
 ];
 
 export interface MemoryJob<T = Record<string, unknown>> {
@@ -63,10 +73,10 @@ export function enqueueMemoryJob(
   payload: Record<string, unknown>,
   runAfter = Date.now(),
   dbOverride?: Parameters<ReturnType<typeof getDb>["transaction"]>[0] extends (
-    tx: infer T,
+    tx: infer T
   ) => unknown
     ? T
-    : never,
+    : never
 ): string {
   const db = dbOverride ?? getDb();
   const id = uuid();
@@ -101,10 +111,10 @@ export function upsertDebouncedJob(
   payload: { conversationId: string },
   runAfter: number,
   dbOverride?: Parameters<ReturnType<typeof getDb>["transaction"]>[0] extends (
-    tx: infer T,
+    tx: infer T
   ) => unknown
     ? T
-    : never,
+    : never
 ): void {
   const db = dbOverride ?? getDb();
   const existing = db
@@ -114,8 +124,8 @@ export function upsertDebouncedJob(
       and(
         eq(memoryJobs.type, type),
         eq(memoryJobs.status, "pending"),
-        sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}`,
-      ),
+        sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}`
+      )
     )
     .get();
   if (existing) {
@@ -135,7 +145,7 @@ export function upsertDebouncedJob(
  */
 export function hasActiveCarryForwardJob(
   filename: string,
-  userSlug: string,
+  userSlug: string
 ): boolean {
   const db = getDb();
   return (
@@ -147,65 +157,35 @@ export function hasActiveCarryForwardJob(
           eq(memoryJobs.type, "journal_carry_forward"),
           inArray(memoryJobs.status, ["pending", "running"]),
           sql`json_extract(${memoryJobs.payload}, '$.filename') = ${filename}`,
-          sql`json_extract(${memoryJobs.payload}, '$.userSlug') = ${userSlug}`,
-        ),
+          sql`json_extract(${memoryJobs.payload}, '$.userSlug') = ${userSlug}`
+        )
       )
       .get() != null
   );
 }
 
-export function enqueueCleanupStaleSupersededItemsJob(
-  retentionMs?: number,
-): string {
+/**
+ * Check whether a pending or running job of the given type already exists.
+ * Used to prevent duplicate enqueues for long-running maintenance jobs.
+ */
+export function hasActiveJobOfType(type: MemoryJobType): boolean {
   const db = getDb();
-  const now = Date.now();
-  const existing = db
-    .select()
-    .from(memoryJobs)
-    .where(
-      and(
-        eq(memoryJobs.type, "cleanup_stale_superseded_items"),
-        inArray(memoryJobs.status, ["pending", "running"]),
-      ),
-    )
-    .orderBy(asc(memoryJobs.createdAt))
-    .get();
-  if (existing) {
-    if (
-      existing.status === "pending" &&
-      typeof retentionMs === "number" &&
-      Number.isFinite(retentionMs) &&
-      retentionMs > 0
-    ) {
-      let payload: Record<string, unknown> = {};
-      try {
-        payload = JSON.parse(existing.payload) as Record<string, unknown>;
-      } catch {
-        payload = {};
-      }
-      if (payload.retentionMs !== retentionMs) {
-        db.update(memoryJobs)
-          .set({
-            payload: JSON.stringify({ ...payload, retentionMs }),
-            updatedAt: now,
-          })
-          .where(eq(memoryJobs.id, existing.id))
-          .run();
-      }
-    }
-    return existing.id;
-  }
-  const payload =
-    typeof retentionMs === "number" &&
-    Number.isFinite(retentionMs) &&
-    retentionMs > 0
-      ? { retentionMs }
-      : {};
-  return enqueueMemoryJob("cleanup_stale_superseded_items", payload);
+  return (
+    db
+      .select({ id: memoryJobs.id })
+      .from(memoryJobs)
+      .where(
+        and(
+          eq(memoryJobs.type, type),
+          inArray(memoryJobs.status, ["pending", "running"])
+        )
+      )
+      .get() != null
+  );
 }
 
 export function enqueuePruneOldConversationsJob(
-  retentionDays?: number,
+  retentionDays?: number
 ): string {
   const db = getDb();
   const existing = db
@@ -214,8 +194,8 @@ export function enqueuePruneOldConversationsJob(
     .where(
       and(
         eq(memoryJobs.type, "prune_old_conversations"),
-        inArray(memoryJobs.status, ["pending", "running"]),
-      ),
+        inArray(memoryJobs.status, ["pending", "running"])
+      )
     )
     .orderBy(asc(memoryJobs.createdAt))
     .get();
@@ -259,7 +239,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
   const now = Date.now();
   const pendingFilter = and(
     eq(memoryJobs.status, "pending"),
-    lte(memoryJobs.runAfter, now),
+    lte(memoryJobs.runAfter, now)
   );
 
   // Claim non-embed jobs first, then fill remaining slots with embed jobs.
@@ -288,7 +268,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
   }
   if (probeAllowed && remainingSlots > 0) {
     log.debug(
-      "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
+      "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed"
     );
   }
 
@@ -318,7 +298,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
         status: "running",
         startedAt: now,
         updatedAt: now,
-      }),
+      })
     );
   }
   return claimed;
@@ -363,7 +343,7 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
   if (deferrals >= MAX_DEFERRALS) {
     log.error(
       { jobId: id, type: row.type, deferrals },
-      "Job exceeded max deferrals, marking as failed",
+      "Job exceeded max deferrals, marking as failed"
     );
     db.update(memoryJobs)
       .set({
@@ -382,14 +362,14 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
   if (DEFERRAL_WARN_MILESTONES.includes(deferrals)) {
     log.warn(
       { jobId: id, type: row.type, deferrals, max: MAX_DEFERRALS },
-      "Job approaching max deferral limit",
+      "Job approaching max deferral limit"
     );
   }
 
   // Exponential backoff: 30s, 60s, 120s, ... capped at 5 minutes
   const delay = Math.min(
     DEFER_BASE_DELAY_MS * Math.pow(2, Math.min(deferrals - 1, 10)),
-    DEFER_MAX_DELAY_MS,
+    DEFER_MAX_DELAY_MS
   );
   db.update(memoryJobs)
     .set({
@@ -406,7 +386,7 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
 export function failMemoryJob(
   id: string,
   error: string,
-  options?: { retryDelayMs?: number; maxAttempts?: number },
+  options?: { retryDelayMs?: number; maxAttempts?: number }
 ): void {
   const retryDelayMs = options?.retryDelayMs ?? 30_000;
   const maxAttempts = options?.maxAttempts ?? 5;
@@ -463,7 +443,7 @@ export function failStalledJobs(timeoutMs: number): number {
       AND started_at IS NOT NULL
       AND started_at < ?
   `,
-    cutoff,
+    cutoff
   );
   if (stalled.length === 0) return 0;
 
@@ -474,14 +454,14 @@ export function failStalledJobs(timeoutMs: number): number {
         status: "failed",
         updatedAt: now,
         lastError: `Job timed out after ${Math.round(
-          timeoutMs / 60_000,
+          timeoutMs / 60_000
         )} minutes`,
       })
       .where(and(eq(memoryJobs.id, row.id), eq(memoryJobs.status, "running")))
       .run();
     log.warn(
       { jobId: row.id, type: row.type, timeoutMs },
-      "Failed stalled memory job due to timeout",
+      "Failed stalled memory job due to timeout"
     );
   }
   return stalled.length;

package/src/memory/jobs-worker.ts

@@ -1,29 +1,32 @@
 import { getConfig } from "../config/loader.js";
 import type { AssistantConfig } from "../config/types.js";
 import { getLogger } from "../util/logger.js";
-import { rawAll, rawRun } from "./db.js";
-import { backfillJob } from "./job-handlers/backfill.js";
-import { batchExtractJob } from "./job-handlers/batch-extraction.js";
+import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
+import { runConsolidation } from "./graph/consolidation.js";
+import { runDecayTick } from "./graph/decay.js";
+import { graphExtractJob } from "./graph/extraction-job.js";
 import {
-  cleanupStaleSupersededItemsJob,
-  pruneOldConversationsJob,
-} from "./job-handlers/cleanup.js";
+  embedGraphNodeJob,
+  embedGraphTriggerJob,
+} from "./graph/graph-search.js";
+import { runNarrativeRefinement } from "./graph/narrative.js";
+import { runPatternScan } from "./graph/pattern-scan.js";
+import { backfillJob } from "./job-handlers/backfill.js";
+import { pruneOldConversationsJob } from "./job-handlers/cleanup.js";
 import { generateConversationStartersJob } from "./job-handlers/conversation-starters.js";
 // ── Per-job-type handlers ──────────────────────────────────────────
 import {
   embedAttachmentJob,
-  embedItemJob,
   embedMediaJob,
   embedSegmentJob,
   embedSummaryJob,
 } from "./job-handlers/embedding.js";
-import { extractItemsJob } from "./job-handlers/extraction.js";
 import {
   deleteQdrantVectorsJob,
   rebuildIndexJob,
 } from "./job-handlers/index-maintenance.js";
-import { journalCarryForwardJob } from "./job-handlers/journal-carry-forward.js";
 import { mediaProcessingJob } from "./job-handlers/media-processing.js";
+import { buildConversationSummaryJob } from "./job-handlers/summarization.js";
 import {
   BackendUnavailableError,
   classifyError,
@@ -34,12 +37,12 @@ import {
   claimMemoryJobs,
   completeMemoryJob,
   deferMemoryJob,
-  enqueueCleanupStaleSupersededItemsJob,
   enqueueMemoryJob,
   enqueuePruneOldConversationsJob,
   failMemoryJob,
   failStalledJobs,
   type MemoryJob,
+  type MemoryJobType,
   resetRunningJobsToPending,
 } from "./jobs-store.js";
 import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
@@ -60,31 +63,6 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
     log.info({ recovered }, "Recovered stale running memory jobs");
   }
 
-  // Startup recovery: enqueue batch_extract for conversations with pending
-  // unextracted messages (e.g. after a crash mid-conversation).
-  try {
-    const pendingRows = rawAll<{ key: string; value: string }>(
-      `SELECT key, value FROM memory_checkpoints WHERE key LIKE 'batch_extract:%:pending_count' AND CAST(value AS INTEGER) > 0`,
-    );
-    for (const row of pendingRows) {
-      // Extract conversationId from key: "batch_extract:<conversationId>:pending_count"
-      const parts = row.key.split(":");
-      if (parts.length >= 3) {
-        const conversationId = parts.slice(1, -1).join(":");
-        enqueueMemoryJob("batch_extract", { conversationId });
-        log.info(
-          { conversationId, pendingCount: row.value },
-          "Recovered pending batch extraction on startup",
-        );
-      }
-    }
-  } catch (err) {
-    log.warn(
-      { err: err instanceof Error ? err.message : String(err) },
-      "Failed to recover pending batch extractions on startup",
-    );
-  }
-
   let stopped = false;
   let tickRunning = false;
   let timer: ReturnType<typeof setTimeout>;
@@ -145,9 +123,6 @@ export async function runMemoryJobsOnce(
   if (!config.memory.enabled) return 0;
   const enableScheduledCleanup = options.enableScheduledCleanup === true;
 
-  // Periodic stale item sweep (throttled to at most once per hour)
-  sweepStaleItems(config);
-
   // Fail jobs that have been running longer than the configured timeout
   const timedOut = failStalledJobs(config.memory.jobs.stalledJobTimeoutMs);
   if (timedOut > 0) {
@@ -161,6 +136,7 @@ export async function runMemoryJobsOnce(
     if (enableScheduledCleanup) {
       maybeEnqueueScheduledCleanupJobs(config);
     }
+    maybeEnqueueGraphMaintenanceJobs();
     return 0;
   }
 
@@ -256,9 +232,67 @@ export async function runMemoryJobsOnce(
   if (enableScheduledCleanup) {
     maybeEnqueueScheduledCleanupJobs(config);
   }
+  maybeEnqueueGraphMaintenanceJobs();
   return processed;
 }
 
+// ── Graph lifecycle job handlers ──────────────────────────────────
+
+function graphDecayJob(job: MemoryJob): void {
+  const scopeId = (job.payload as { scopeId?: string })?.scopeId ?? "default";
+  const result = runDecayTick(scopeId);
+  log.info({ jobId: job.id, ...result }, "Graph decay tick complete");
+}
+
+async function graphConsolidateJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const scopeId = (job.payload as { scopeId?: string })?.scopeId ?? "default";
+  const result = await runConsolidation(scopeId, config);
+  log.info(
+    {
+      jobId: job.id,
+      updated: result.totalUpdated,
+      deleted: result.totalDeleted,
+      mergeEdges: result.totalMergeEdges,
+    },
+    "Graph consolidation complete",
+  );
+}
+
+async function graphPatternScanJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const scopeId = (job.payload as { scopeId?: string })?.scopeId ?? "default";
+  const result = await runPatternScan(scopeId, config);
+  log.info(
+    {
+      jobId: job.id,
+      patterns: result.patternsDetected,
+      edges: result.edgesCreated,
+    },
+    "Graph pattern scan complete",
+  );
+}
+
+async function graphNarrativeRefineJob(
+  job: MemoryJob,
+  config: AssistantConfig,
+): Promise<void> {
+  const scopeId = (job.payload as { scopeId?: string })?.scopeId ?? "default";
+  const result = await runNarrativeRefinement(scopeId, config);
+  log.info(
+    {
+      jobId: job.id,
+      updated: result.nodesUpdated,
+      arcs: result.arcsIdentified,
+    },
+    "Graph narrative refinement complete",
+  );
+}
+
 // ── Job error handling ─────────────────────────────────────────────
 
 function handleJobError(job: MemoryJob, err: unknown): void {
@@ -322,33 +356,20 @@ async function processJob(
       await embedSegmentJob(job, config);
       return;
     case "embed_item":
-      await embedItemJob(job, config);
-      return;
-    case "embed_summary":
-      await embedSummaryJob(job, config);
-      return;
     case "extract_items":
-      await extractItemsJob(job);
-      return;
     case "batch_extract":
-      await batchExtractJob(job);
-      return;
     case "extract_entities":
-      // Entity extraction has been removed — silently drop legacy jobs
-      return;
     case "cleanup_stale_superseded_items":
-      cleanupStaleSupersededItemsJob(job, config);
+      // Old extraction pipeline replaced by memory graph — silently drop
+      return;
+    case "embed_summary":
+      await embedSummaryJob(job, config);
       return;
     case "prune_old_conversations":
       pruneOldConversationsJob(job, config);
       return;
     case "build_conversation_summary":
-      // Deprecated: conversation summaries are now produced as a side-effect
-      // of batch extraction. Silently skip legacy jobs.
-      log.debug(
-        { jobId: job.id },
-        "Skipping deprecated build_conversation_summary job — handled by batch extraction",
-      );
+      await buildConversationSummaryJob(job, config);
       return;
     case "backfill":
       await backfillJob(job, config);
@@ -376,7 +397,28 @@ async function processJob(
       await embedAttachmentJob(job, config);
       return;
     case "journal_carry_forward":
-      await journalCarryForwardJob(job);
+      // Journal carry-forward replaced by graph extraction — silently drop
+      return;
+    case "embed_graph_node":
+      await embedGraphNodeJob(job, config);
+      return;
+    case "graph_trigger_embed":
+      await embedGraphTriggerJob(job, config);
+      return;
+    case "graph_extract":
+      await graphExtractJob(job, config);
+      return;
+    case "graph_decay":
+      graphDecayJob(job);
+      return;
+    case "graph_consolidate":
+      await graphConsolidateJob(job, config);
+      return;
+    case "graph_pattern_scan":
+      await graphPatternScanJob(job, config);
+      return;
+    case "graph_narrative_refine":
+      await graphNarrativeRefineJob(job, config);
       return;
     case "generate_conversation_starters":
       await generateConversationStartersJob(job);
@@ -416,9 +458,6 @@ export function maybeEnqueueScheduledCleanupJobs(
   if (nowMs - lastScheduledCleanupEnqueueMs < cleanup.enqueueIntervalMs)
     return false;
 
-  const staleSupersededItemsJobId = enqueueCleanupStaleSupersededItemsJob(
-    cleanup.supersededItemRetentionMs,
-  );
   const pruneConversationsJobId =
     cleanup.conversationRetentionDays > 0
       ? enqueuePruneOldConversationsJob(cleanup.conversationRetentionDays)
@@ -426,10 +465,8 @@ export function maybeEnqueueScheduledCleanupJobs(
   lastScheduledCleanupEnqueueMs = nowMs;
   log.debug(
     {
-      staleSupersededItemsJobId,
       pruneConversationsJobId,
       enqueueIntervalMs: cleanup.enqueueIntervalMs,
-      supersededItemRetentionMs: cleanup.supersededItemRetentionMs,
       conversationRetentionDays: cleanup.conversationRetentionDays,
     },
     "Enqueued scheduled memory cleanup jobs",
@@ -437,61 +474,59 @@ export function maybeEnqueueScheduledCleanupJobs(
   return true;
 }
 
-// ── Stale item sweep ───────────────────────────────────────────────
+// ── Graph maintenance scheduling ──────────────────────────────────
 
-const STALE_SWEEP_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
-let lastStaleSweepMs = 0;
+const GRAPH_DECAY_INTERVAL_MS = 60 * 60 * 1000; // 1 hour
+const GRAPH_CONSOLIDATE_INTERVAL_MS = 4 * 60 * 60 * 1000; // 4 hours
+const GRAPH_PATTERN_SCAN_INTERVAL_MS = 24 * 60 * 60 * 1000; // 1 day
+const GRAPH_NARRATIVE_INTERVAL_MS = 7 * 24 * 60 * 60 * 1000; // 1 week
 
-/** Reset the sweep throttle so tests can call sweepStaleItems back-to-back. */
-export function resetStaleSweepThrottle(): void {
-  lastStaleSweepMs = 0;
-}
+const GRAPH_MAINTENANCE_CHECKPOINTS = {
+  decay: "graph_maintenance:decay:last_run",
+  consolidate: "graph_maintenance:consolidate:last_run",
+  patternScan: "graph_maintenance:pattern_scan:last_run",
+  narrative: "graph_maintenance:narrative:last_run",
+} as const;
 
 /**
- * Mark deeply stale memory items as invalid. An item is considered deeply
- * stale when it has exceeded 2x its freshness window for its kind and has
- * not been recently accessed.
- *
- * This is non-destructive: items keep their data but get an `invalid_at`
- * timestamp that excludes them from retrieval queries.
+ * Enqueue periodic graph maintenance jobs (decay, consolidation, pattern scan, narrative).
+ * Uses durable checkpoints so intervals survive daemon restarts — jobs only fire
+ * when the actual elapsed time since last run exceeds the interval.
  */
-export function sweepStaleItems(config: AssistantConfig): number {
-  const freshness = config.memory.retrieval.freshness;
-  if (!freshness.enabled) return 0;
-
-  const now = Date.now();
-  // Throttle: at most once per hour
-  if (now - lastStaleSweepMs < STALE_SWEEP_INTERVAL_MS) return 0;
-  lastStaleSweepMs = now;
-
-  let totalMarked = 0;
-  for (const [kind, maxAgeDays] of Object.entries(freshness.maxAgeDays)) {
-    if (maxAgeDays <= 0) continue;
-    // Mark invalid if: past 2x window, no access in the shield period, and not already invalid
-    const cutoffMs = now - maxAgeDays * 2 * 86_400_000;
-    const shieldCutoffMs = now - freshness.reinforcementShieldDays * 86_400_000;
-    const changes = rawRun(
-      `
-      UPDATE memory_items
-      SET invalid_at = ?
-      WHERE kind = ?
-        AND status = 'active'
-        AND invalid_at IS NULL
-        AND last_seen_at < ?
-        AND (access_count = 0 OR COALESCE(last_used_at, 0) < ?)
-    `,
-      now,
-      kind,
-      cutoffMs,
-      shieldCutoffMs,
-    );
-    if (changes > 0) {
-      log.info(
-        { kind, marked: changes, cutoffMs },
-        "Marked stale memory items as invalid",
-      );
-      totalMarked += changes;
+function maybeEnqueueGraphMaintenanceJobs(nowMs = Date.now()): void {
+  const schedule: Array<{
+    key: string;
+    intervalMs: number;
+    jobType: MemoryJobType;
+  }> = [
+    {
+      key: GRAPH_MAINTENANCE_CHECKPOINTS.decay,
+      intervalMs: GRAPH_DECAY_INTERVAL_MS,
+      jobType: "graph_decay",
+    },
+    {
+      key: GRAPH_MAINTENANCE_CHECKPOINTS.consolidate,
+      intervalMs: GRAPH_CONSOLIDATE_INTERVAL_MS,
+      jobType: "graph_consolidate",
+    },
+    {
+      key: GRAPH_MAINTENANCE_CHECKPOINTS.patternScan,
+      intervalMs: GRAPH_PATTERN_SCAN_INTERVAL_MS,
+      jobType: "graph_pattern_scan",
+    },
+    {
+      key: GRAPH_MAINTENANCE_CHECKPOINTS.narrative,
+      intervalMs: GRAPH_NARRATIVE_INTERVAL_MS,
+      jobType: "graph_narrative_refine",
+    },
+  ];
+
+  for (const { key, intervalMs, jobType } of schedule) {
+    const lastRun = parseInt(getMemoryCheckpoint(key) ?? "0", 10);
+    if (nowMs - lastRun >= intervalMs) {
+      enqueueMemoryJob(jobType, {});
+      setMemoryCheckpoint(key, String(nowMs));
     }
   }
-  return totalMarked;
 }
+

package/src/memory/message-content.ts

@@ -121,6 +121,7 @@ export function extractMediaBlockMeta(
   }
 }
 
+
 function stableJson(value: unknown): string {
   try {
     return JSON.stringify(value);