@vellumai/assistant 0.6.3 → 0.6.4

package/src/memory/cleanup-schedule-state.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared state for the cleanup-scheduler throttle.
+ *
+ * `maybeEnqueueScheduledCleanupJobs` in jobs-worker.ts gates cleanup-job
+ * enqueueing behind a 6-hour window (configurable via
+ * memory.cleanup.enqueueIntervalMs). This module owns the "last enqueue"
+ * timestamp so that code paths outside jobs-worker — notably
+ * ConfigWatcher.refreshConfigFromSources — can reset the throttle without
+ * pulling in jobs-worker's large transitive import graph.
+ *
+ * The ConfigWatcher uses resetCleanupScheduleThrottle() to ensure that
+ * retention changes made via the UI (which flow through config.json →
+ * invalidateConfigCache → refreshConfigFromSources) take effect on the
+ * very next scheduler tick instead of waiting out the remaining window.
+ */
+
+let lastScheduledCleanupEnqueueMs = 0;
+
+/** Read the timestamp of the most recent enqueue (0 if never/reset). */
+export function getLastScheduledCleanupEnqueueMs(): number {
+  return lastScheduledCleanupEnqueueMs;
+}
+
+/** Record that an enqueue just happened at `nowMs`. */
+export function markScheduledCleanupEnqueued(nowMs: number): void {
+  lastScheduledCleanupEnqueueMs = nowMs;
+}
+
+/**
+ * Clear the throttle so the next `maybeEnqueueScheduledCleanupJobs` call
+ * bypasses the `enqueueIntervalMs` window. Used by ConfigWatcher when
+ * retention settings change, and by tests that need deterministic
+ * scheduling.
+ */
+export function resetCleanupScheduleThrottle(): void {
+  lastScheduledCleanupEnqueueMs = 0;
+}

package/src/memory/conversation-analyze-job.ts

@@ -0,0 +1,73 @@
+// ---------------------------------------------------------------------------
+// Auto-analyze — conversation_analyze job handler
+//
+// Bridges the jobs worker to the shared analyzeConversation() service. The
+// deps bundle is stashed on a module singleton during daemon startup; if it
+// isn't set yet we skip this iteration. The next batch / idle / lifecycle
+// trigger from `enqueueAutoAnalysisIfEnabled()` will produce a fresh job
+// once the daemon has fully started.
+//
+// The service itself distinguishes manual vs. auto triggers: this handler
+// always invokes with `trigger: "auto"`, so the rolling analysis conversation
+// logic and recursion guard apply.
+// ---------------------------------------------------------------------------
+
+import type { AssistantConfig } from "../config/types.js";
+import { analyzeConversation } from "../runtime/services/analyze-conversation.js";
+import { getAnalysisDeps } from "../runtime/services/analyze-deps-singleton.js";
+import { getLogger } from "../util/logger.js";
+import { enqueueAutoAnalysisIfEnabled } from "./auto-analysis-enqueue.js";
+import type { MemoryJob } from "./jobs-store.js";
+
+const log = getLogger("conversation-analyze-job");
+
+export async function conversationAnalyzeJob(
+  job: MemoryJob<{ conversationId?: string }>,
+  _config: AssistantConfig,
+): Promise<void> {
+  const { conversationId } = job.payload;
+  if (!conversationId) {
+    log.warn({ jobId: job.id }, "Skipping job: missing conversationId");
+    return;
+  }
+
+  const deps = getAnalysisDeps();
+  if (!deps) {
+    // Daemon hasn't finished startup. Return without throwing — a plain
+    // Error here would be classified as fatal by `classifyError()` and the
+    // worker would mark the job permanently failed. Throwing
+    // `BackendUnavailableError` would defer, but defer counters cap out and
+    // would still permanently fail in the worst case. Since
+    // `enqueueAutoAnalysisIfEnabled()` re-enqueues on the next batch / idle
+    // / lifecycle trigger, dropping this iteration is the safest choice and
+    // avoids retry storms during slow daemon startup.
+    log.warn(
+      { jobId: job.id, conversationId },
+      "Skipping job: analysis deps not yet initialized; will retrigger",
+    );
+    return;
+  }
+
+  const result = await analyzeConversation(conversationId, deps, {
+    trigger: "auto",
+  });
+  if ("error" in result) {
+    log.warn(
+      { jobId: job.id, conversationId, error: result.error },
+      "Auto-analysis service rejected source conversation",
+    );
+    return;
+  }
+  if (result.skipped) {
+    // The rolling analysis conversation was still processing a prior run, so
+    // this invocation was a no-op. Schedule a debounced follow-up ourselves
+    // — otherwise, if no later batch/idle/lifecycle trigger arrives (e.g.
+    // the conversation goes quiet after a long in-flight analysis), new
+    // source messages would stay un-analyzed indefinitely.
+    enqueueAutoAnalysisIfEnabled({ conversationId, trigger: "idle" });
+    log.debug(
+      { jobId: job.id, conversationId },
+      "Auto-analysis skipped (rolling conversation busy); requeued follow-up",
+    );
+  }
+}

package/src/memory/conversation-crud.ts

@@ -30,6 +30,7 @@ import {
   deleteOrphanAttachments,
   linkAttachmentToMessage,
 } from "./attachments-store.js";
+import { AUTO_ANALYSIS_SOURCE } from "./auto-analysis-guard.js";
 import {
   projectAssistantMessage,
   seedForkedConversationAttention,
@@ -174,6 +175,7 @@ export interface ConversationRow {
   isAutoTitle: number;
   scheduleJobId: string | null;
   lastMessageAt: number | null;
+  archivedAt: number | null;
 }
 
 export const parseConversation = createRowMapper<
@@ -201,6 +203,7 @@ export const parseConversation = createRowMapper<
   isAutoTitle: "isAutoTitle",
   scheduleJobId: "scheduleJobId",
   lastMessageAt: "lastMessageAt",
+  archivedAt: "archivedAt",
 });
 
 export interface MessageRow {
@@ -248,6 +251,7 @@ export function createConversation(
         scheduleJobId?: string;
         groupId?: string;
         hostAccess?: boolean;
+        forkParentConversationId?: string;
       },
 ) {
   const db = getDb();
@@ -284,6 +288,7 @@ export function createConversation(
     source,
     memoryScopeId,
     scheduleJobId: opts.scheduleJobId ?? null,
+    forkParentConversationId: opts.forkParentConversationId ?? null,
   };
 
   // Retry on SQLITE_BUSY and SQLITE_IOERR — transient disk I/O errors or WAL
@@ -379,6 +384,54 @@ export function countConversationsByScheduleJobId(
   );
 }
 
+/**
+ * Find the rolling analysis conversation for a given source conversation,
+ * or null if none exists yet. Used by the auto-analyze loop to append
+ * to an existing analysis conversation rather than creating a new one
+ * each time the analyze job fires.
+ *
+ * Returns the most recently updated match if multiple exist (defensive —
+ * shouldn't happen in normal operation but the contract is well-defined).
+ *
+ * Hits `idx_conversations_fork_parent_conversation_id` for the
+ * `forkParentConversationId` lookup.
+ */
+export function findAnalysisConversationFor(
+  parentConversationId: string,
+): { id: string } | null {
+  const db = getDb();
+  const row = db
+    .select({ id: conversations.id })
+    .from(conversations)
+    .where(
+      and(
+        eq(conversations.source, AUTO_ANALYSIS_SOURCE),
+        eq(conversations.forkParentConversationId, parentConversationId),
+      ),
+    )
+    .orderBy(desc(conversations.updatedAt))
+    .limit(1)
+    .get();
+  return row ? { id: row.id } : null;
+}
+
+/**
+ * Returns the `source` column for the given conversation, or null if
+ * not found. Tiny convenience used by the recursion guard in the
+ * auto-analyze loop.
+ */
+export function getConversationSource(
+  conversationId: string,
+): string | null {
+  const db = getDb();
+  const row = db
+    .select({ source: conversations.source })
+    .from(conversations)
+    .where(eq(conversations.id, conversationId))
+    .get();
+  return row?.source ?? null;
+}
+
 export function getConversationType(
   conversationId: string,
 ): "standard" | "private" {
@@ -1062,6 +1115,27 @@ export function getLastAssistantTimestampBefore(
   return row?.createdAt ?? 0;
 }
 
+export function getLastUserTimestampBefore(
+  conversationId: string,
+  beforeTimestamp: number,
+): number {
+  const db = getDb();
+  const row = db
+    .select({ createdAt: messages.createdAt })
+    .from(messages)
+    .where(
+      and(
+        eq(messages.conversationId, conversationId),
+        eq(messages.role, "user"),
+        lt(messages.createdAt, beforeTimestamp),
+      ),
+    )
+    .orderBy(desc(messages.createdAt))
+    .limit(1)
+    .get();
+  return row?.createdAt ?? 0;
+}
+
 /** Fetch a single message by ID, optionally scoped to a specific conversation. */
 export function getMessageById(
   messageId: string,
@@ -1146,6 +1220,31 @@ export function updateConversationHostAccess(
     .run();
 }
 
+export function archiveConversation(id: string): boolean {
+  const conv = getConversation(id);
+  if (!conv) return false;
+  const now = Date.now();
+  rawRun(
+    "UPDATE conversations SET archived_at = ?, updated_at = ? WHERE id = ?",
+    now,
+    now,
+    id,
+  );
+  return true;
+}
+
+export function unarchiveConversation(id: string): boolean {
+  const conv = getConversation(id);
+  if (!conv) return false;
+  const now = Date.now();
+  rawRun(
+    "UPDATE conversations SET archived_at = NULL, updated_at = ? WHERE id = ?",
+    now,
+    id,
+  );
+  return true;
+}
+
 /**
  * Delete all conversations, messages, and related data (tool invocations,
  * memory segments, etc.) from the daemon database.

package/src/memory/conversation-disk-view.ts

@@ -310,6 +310,13 @@ export function syncMessageToDisk(
     if (toolResults.length > 0) record.toolResults = toolResults;
     if (attachmentFilenames.length > 0)
       record.attachments = attachmentFilenames;
+    if (message.metadata) {
+      try {
+        record.metadata = JSON.parse(message.metadata);
+      } catch {
+        // Invalid JSON — omit metadata from disk record
+      }
+    }
 
     appendFileSync(
       join(dirPath, "messages.jsonl"),

package/src/memory/conversation-group-migration.ts

@@ -57,7 +57,11 @@ export function ensureGroupMigration(): void {
     }
   }
 
-  // 3. Seed system groups (four: pinned, scheduled, background, all)
+  // 3. Seed system groups.
+  //
+  // `system:reflections` is a legacy group kept for backward compatibility
+  // with existing installations. New auto-analysis conversations are assigned
+  // to `system:background`; the migration in step 6 moves existing ones.
   const now = Math.floor(Date.now() / 1000);
   rawExec(`
     INSERT OR IGNORE INTO conversation_groups (id, name, sort_position, is_system_group, created_at, updated_at)
@@ -65,7 +69,8 @@ export function ensureGroupMigration(): void {
       ('system:pinned', 'Pinned', 0, TRUE, ${now}, ${now}),
       ('system:scheduled', 'Scheduled', 1, TRUE, ${now}, ${now}),
       ('system:background', 'Background', 2, TRUE, ${now}, ${now}),
-      ('system:all', 'Recents', 3, TRUE, ${now}, ${now})
+      ('system:all', 'Recents', 3, TRUE, ${now}, ${now}),
+      ('system:reflections', 'Reflections', 100, TRUE, ${now}, ${now})
   `);
 
   // One-time migration: move system:all to sortPosition 3 (from 999999).
@@ -213,5 +218,32 @@ export function ensureGroupMigration(): void {
     }
   }
 
+  // 6. One-time migration: move auto-analysis conversations from system:reflections to system:background
+  const reflectionsMigrateDone = rawGet<{ id: string }>(
+    "SELECT id FROM conversation_groups WHERE id = '_reflections_to_background_complete'",
+  );
+
+  if (!reflectionsMigrateDone) {
+    try {
+      rawExec("BEGIN");
+
+      rawExec(`
+        UPDATE conversations SET group_id = 'system:background'
+        WHERE group_id = 'system:reflections'
+      `);
+
+      rawExec(`
+        INSERT OR IGNORE INTO conversation_groups (id, name, sort_position, is_system_group)
+        VALUES ('_reflections_to_background_complete', '_reflections_to_background_complete', -1, TRUE)
+      `);
+
+      rawExec("COMMIT");
+    } catch (err) {
+      rawExec("ROLLBACK");
+      log.error({ err }, "reflections-to-background migration failed, rolled back");
+      throw err;
+    }
+  }
+
   migrated = true;
 }

package/src/memory/conversation-queries.ts

@@ -218,7 +218,7 @@ export function searchConversations(
         FROM messages_fts f
         JOIN messages m ON m.id = f.message_id
         JOIN conversations c ON c.id = m.conversation_id
-        WHERE messages_fts MATCH ? AND c.conversation_type NOT IN ('background', 'private', 'scheduled')
+        WHERE messages_fts MATCH ? AND c.conversation_type NOT IN ('background', 'private', 'scheduled') AND c.archived_at IS NULL
         LIMIT 1000
       `,
         ftsMatch,
@@ -243,7 +243,7 @@ export function searchConversations(
       SELECT DISTINCT m.conversation_id
       FROM messages m
       JOIN conversations c ON c.id = m.conversation_id
-      WHERE m.content LIKE ? ESCAPE '\\' AND c.conversation_type NOT IN ('background', 'private', 'scheduled')
+      WHERE m.content LIKE ? ESCAPE '\\' AND c.conversation_type NOT IN ('background', 'private', 'scheduled') AND c.archived_at IS NULL
       LIMIT 1000
     `,
       likePattern,
@@ -257,9 +257,10 @@ export function searchConversations(
     .from(conversations)
     .where(
       and(
-        sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`,
-        sql`${conversations.title} LIKE ${titlePattern} ESCAPE '\\'`,
-      ),
+          sql`${conversations.conversationType} NOT IN ('background', 'private', 'scheduled')`,
+          sql`${conversations.title} LIKE ${titlePattern} ESCAPE '\\'`,
+          sql`${conversations.archivedAt} IS NULL`,
+        ),
     )
     .all();
   for (const row of titleMatchConvs) ftsConvIds.add(row.id);

package/src/memory/db-init.ts

@@ -59,6 +59,7 @@ import {
   migrateContactsUserFileColumn,
   migrateConversationForkLineage,
   migrateConversationHostAccess,
+  migrateConversationsArchivedAt,
   migrateConversationsLastMessageAt,
   migrateConversationsThreadTypeIndex,
   migrateCreateConversationGraphMemoryState,
@@ -106,6 +107,7 @@ import {
   migrateMessagesConversationCreatedAtIndex,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
+  migrateNormalizeUserFileByPrincipal,
   migrateNotificationDeliveryThreadDecision,
   migrateOAuthAppsClientSecretPath,
   migrateOAuthProvidersBehaviorColumns,
@@ -119,6 +121,7 @@ import {
   migrateOAuthProvidersRevoke,
   migrateOAuthProvidersScopeSeparator,
   migrateOAuthProvidersTokenAuthMethodDefault,
+  migrateOAuthProvidersTokenExchangeBodyFormat,
   migrateReminderRoutingIntent,
   migrateRemindersToSchedules,
   migrateRenameConversationTypeColumn,
@@ -364,6 +367,9 @@ export function initializeDb(): void {
     migrateOAuthProvidersTokenAuthMethodDefault,
     migrateConversationHostAccess,
     migrateOAuthProvidersLogoUrl,
+    migrateOAuthProvidersTokenExchangeBodyFormat,
+    migrateNormalizeUserFileByPrincipal,
+    migrateConversationsArchivedAt,
   ];
 
   // Run each migration step, catching and logging individual failures so one

package/src/memory/db-maintenance.ts

@@ -0,0 +1,108 @@
+import { statSync } from "node:fs";
+
+import { getLogger } from "../util/logger.js";
+import { getDbPath } from "../util/platform.js";
+import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
+import { getSqlite } from "./db-connection.js";
+
+const log = getLogger("db-maintenance");
+
+const DB_MAINTENANCE_CHECKPOINT_KEY = "db_maintenance:last_run";
+const DB_MAINTENANCE_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+interface DbStats {
+  pageSizeBytes: number;
+  pageCount: number;
+  freelistCount: number;
+  fileSizeBytes: number | null;
+}
+
+function getDbStats(): DbStats {
+  const sqlite = getSqlite();
+  const pageSizeBytes = (
+    sqlite.query("PRAGMA page_size").get() as { page_size: number }
+  ).page_size;
+  const pageCount = (
+    sqlite.query("PRAGMA page_count").get() as { page_count: number }
+  ).page_count;
+  const freelistCount = (
+    sqlite.query("PRAGMA freelist_count").get() as { freelist_count: number }
+  ).freelist_count;
+  let fileSizeBytes: number | null = null;
+  try {
+    fileSizeBytes = statSync(getDbPath()).size;
+  } catch {
+    /* non-fatal */
+  }
+  return { pageSizeBytes, pageCount, freelistCount, fileSizeBytes };
+}
+
+export function runDbMaintenance(): void {
+  const before = getDbStats();
+  const freelistPct =
+    before.pageCount > 0
+      ? ((before.freelistCount / before.pageCount) * 100).toFixed(1)
+      : "0";
+
+  log.info(
+    {
+      pageCount: before.pageCount,
+      freelistCount: before.freelistCount,
+      freelistPct,
+      fileSizeBytes: before.fileSizeBytes,
+    },
+    "Starting database maintenance",
+  );
+
+  try {
+    getSqlite().exec("VACUUM");
+  } catch (err) {
+    log.warn({ err }, "VACUUM failed (non-fatal)");
+    try {
+      getSqlite().exec("PRAGMA optimize");
+    } catch (optErr) {
+      log.warn({ err: optErr }, "PRAGMA optimize failed (non-fatal)");
+    }
+    return;
+  }
+
+  try {
+    getSqlite().exec("PRAGMA optimize");
+  } catch (err) {
+    log.warn({ err }, "PRAGMA optimize failed (non-fatal)");
+  }
+
+  const after = getDbStats();
+  const reclaimedPages = before.pageCount - after.pageCount;
+  const reclaimedBytes =
+    before.fileSizeBytes != null && after.fileSizeBytes != null
+      ? before.fileSizeBytes - after.fileSizeBytes
+      : null;
+
+  log.info(
+    {
+      beforePageCount: before.pageCount,
+      afterPageCount: after.pageCount,
+      reclaimedPages,
+      reclaimedBytes,
+      afterFileSizeBytes: after.fileSizeBytes,
+    },
+    "Database maintenance complete",
+  );
+}
+
+export function maybeRunDbMaintenance(nowMs = Date.now()): void {
+  const lastRun = parseInt(
+    getMemoryCheckpoint(DB_MAINTENANCE_CHECKPOINT_KEY) ?? "0",
+    10,
+  );
+  if (nowMs - lastRun < DB_MAINTENANCE_INTERVAL_MS) return;
+
+  try {
+    runDbMaintenance();
+  } catch (err) {
+    log.error({ err }, "Database maintenance failed unexpectedly");
+  }
+  // Always set checkpoint — even on failure — to avoid retry-hammering every tick.
+  setMemoryCheckpoint(DB_MAINTENANCE_CHECKPOINT_KEY, String(nowMs));
+}

package/src/memory/db.ts

@@ -6,6 +6,7 @@ export {
   resetDb,
 } from "./db-connection.js";
 export { initializeDb } from "./db-init.js";
+export { maybeRunDbMaintenance, runDbMaintenance } from "./db-maintenance.js";
 export {
   rawAll,
   rawAllFrom,

package/src/memory/graph/conversation-graph-memory.ts

@@ -252,6 +252,21 @@ export class ConversationGraphMemory {
     };
   }
 
+  /**
+   * Re-register cached node IDs with the InContextTracker after compaction
+   * WITHOUT modifying messages. Use this at post-agent-loop compaction sites
+   * where the memory block already survives on the original user message
+   * (since `<memory __injected>` is not stripped by stripInjectionsForCompaction).
+   *
+   * Calling reinjectCachedMemory at these sites would inject a duplicate
+   * onto the last user message — which after tool calls is a tool_result,
+   * not the original user message.
+   */
+  retrackCachedNodes(): void {
+    if (this.lastInjectedNodeIds.length === 0) return;
+    this.tracker.add(this.lastInjectedNodeIds);
+  }
+
   /**
    * Main entry point — called on every turn before the LLM sees the messages.
    *

package/src/memory/graph/extraction.test.ts

@@ -111,6 +111,29 @@ describe("parseExtractionResponse — node creation", () => {
     expect(diff.createNodes[0].stability).toBe(14);
   });
 
+  test("procedural nodes get stability=60", () => {
+    const { diff } = parse({
+      create_nodes: [
+        {
+          content: "ffmpeg needs -ac 2 to force stereo output.",
+          type: "procedural",
+          emotional_charge: {
+            valence: 0,
+            intensity: 0,
+            decay_curve: "linear",
+            decay_rate: 0.05,
+          },
+          significance: 0.5,
+          confidence: 0.9,
+          source_type: "direct",
+        },
+      ],
+      reinforce_node_ids: [],
+    });
+
+    expect(diff.createNodes[0].stability).toBe(60);
+  });
+
   test("clamps significance to [0, 1]", () => {
     const { diff } = parse({
       create_nodes: [

package/src/memory/graph/extraction.ts

@@ -596,6 +596,14 @@ export function parseExtractionResponse(
       node.stability = 5;
     }
 
+    // Procedural nodes (learned skills, how-to knowledge — "ffmpeg needs -ac 2
+    // for stereo") encode facts that stay useful long after the moment they
+    // were learned. Higher initial stability slows Ebbinghaus decay so they
+    // remain retrievable months later without needing explicit reinforcement.
+    if (node.type === "procedural") {
+      node.stability = 60;
+    }
+
     diff.createNodes.push(node);
     const nodeIndex = diff.createNodes.length - 1;
 

package/src/memory/graph/retriever.ts

@@ -25,6 +25,7 @@ import {
   computeTemporalBoost,
   PER_TURN_WEIGHTS,
   scoreCandidate,
+  weightsForContextLoad,
 } from "./scoring.js";
 import { sampleSerendipity } from "./serendipity.js";
 import { getEdgesForNode, getNodesByIds, queryNodes } from "./store.js";
@@ -530,15 +531,19 @@ export async function loadContextMemory(
     const recency = computeRecencyBoost(node, nowMs);
 
     scored.push(
-      scoreCandidate(node, {
-        semanticSimilarity: semanticSim,
-        effectiveSignificance: effectiveSig,
-        emotionalIntensity: node.emotionalCharge.intensity,
-        temporalBoost: normalizedTemporal,
-        recencyBoost: recency,
-        triggerBoost,
-        activationBoost: activation,
-      }),
+      scoreCandidate(
+        node,
+        {
+          semanticSimilarity: semanticSim,
+          effectiveSignificance: effectiveSig,
+          emotionalIntensity: node.emotionalCharge.intensity,
+          temporalBoost: normalizedTemporal,
+          recencyBoost: recency,
+          triggerBoost,
+          activationBoost: activation,
+        },
+        weightsForContextLoad(node),
+      ),
     );
   }
 
@@ -592,15 +597,19 @@ export async function loadContextMemory(
     (node) => {
       const existing = scored.find((s) => s.node.id === node.id);
       if (existing) return existing;
-      return scoreCandidate(node, {
-        semanticSimilarity: semanticCandidateIds.get(node.id) ?? 0,
-        effectiveSignificance: computeEffectiveSignificance(node, nowMs),
-        emotionalIntensity: node.emotionalCharge.intensity,
-        temporalBoost: (computeTemporalBoost(node, now) + 1) / 2,
-        recencyBoost: 0,
-        triggerBoost: 0,
-        activationBoost: 0,
-      });
+      return scoreCandidate(
+        node,
+        {
+          semanticSimilarity: semanticCandidateIds.get(node.id) ?? 0,
+          effectiveSignificance: computeEffectiveSignificance(node, nowMs),
+          emotionalIntensity: node.emotionalCharge.intensity,
+          temporalBoost: (computeTemporalBoost(node, now) + 1) / 2,
+          recencyBoost: 0,
+          triggerBoost: 0,
+          activationBoost: 0,
+        },
+        weightsForContextLoad(node),
+      );
     },
   );