@vellumai/assistant 0.5.6 → 0.5.8

package/src/memory/job-handlers/conversation-starters.ts

@@ -9,6 +9,7 @@ import { and, desc, eq } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import { loadSkillCatalog } from "../../config/skills.js";
+import { resolveGuardianPersona } from "../../prompts/persona-resolver.js";
 import { buildCoreIdentityContext } from "../../prompts/system-prompt.js";
 import {
   createTimeout,
@@ -171,7 +172,9 @@ async function generateStarters(scopeId: string): Promise<GeneratedStarter[]> {
 
   // Truncate identity context to prevent oversized prompts when SOUL.md /
   // IDENTITY.md / USER.md are large.
-  const rawIdentityContext = buildCoreIdentityContext();
+  const rawIdentityContext = buildCoreIdentityContext({
+    userPersona: resolveGuardianPersona(),
+  });
   const identityContext = rawIdentityContext
     ? truncate(rawIdentityContext, 2000, "\n…[truncated]")
     : null;

package/src/memory/job-handlers/summarization.ts

@@ -19,18 +19,25 @@ import { memorySegments, memorySummaries } from "../schema.js";
 const log = getLogger("memory-jobs-worker");
 
 const SUMMARY_LLM_TIMEOUT_MS = 20_000;
-const SUMMARY_MAX_TOKENS = 800;
+const SUMMARY_MAX_TOKENS = 500;
 
 const CONVERSATION_SUMMARY_SYSTEM_PROMPT = [
-  "You are a memory summarization system. Your job is to produce a compact, information-dense summary of a conversation.",
+  "You compress conversation transcripts into compact summaries for semantic search and memory retrieval.",
+  "Focus on durable facts, not transient discussion.",
+  "Preserve: goals, decisions, constraints, preferences, names, technical details, actions taken.",
+  "Remove: filler, pleasantries, tool invocation details, transient status updates.",
   "",
-  "Guidelines:",
-  "- Focus on key facts, decisions, user preferences, and actionable information.",
-  "- Preserve concrete details: names, file paths, tool choices, technical decisions, constraints.",
-  "- Remove filler, pleasantries, and transient discussion that has no lasting value.",
-  "- Use concise bullet points grouped by topic.",
-  "- Target 400-600 tokens. Be dense but readable.",
-  "- If updating an existing summary with new data, merge new information and remove anything that was superseded.",
+  "Return concise markdown:",
+  "## Topic",
+  "One-line description of what the conversation is about.",
+  "## Key Facts",
+  "Bullet points of concrete facts, names, decisions, preferences.",
+  "## Outcomes",
+  "What was decided, resolved, or accomplished.",
+  "## Open Items",
+  "Unresolved questions, pending tasks, or follow-ups (omit section if none).",
+  "",
+  "Target 200-400 tokens. Be dense.",
 ].join("\n");
 
 export async function buildConversationSummaryJob(
@@ -62,9 +69,8 @@ export async function buildConversationSummaryJob(
 
   // Build segment text for LLM input (chronological order)
   const segmentTexts = rows
-    .slice(0, 30)
     .reverse()
-    .map((row) => `[${row.role}] ${truncate(row.text, 400)}`)
+    .map((row) => `[${row.role}] ${truncate(row.text, 600)}`)
     .join("\n\n");
 
   const summaryText = await summarizeWithLLM(
@@ -208,14 +214,18 @@ async function summarizeWithLLM(
 }
 
 function buildFallbackSummary(
-  _existingSummary: string | null,
+  existingSummary: string | null,
   newContent: string,
   label: string,
 ): string {
   const lines = newContent.split("\n").filter((l) => l.trim().length > 0);
-  const snippets = lines
-    .slice(0, 20)
-    .map((l) => `- ${truncate(l.trim(), 180)}`);
-  const parts: string[] = [`${label} summary`, "", ...snippets];
+  if (lines.length === 0) return existingSummary ?? `${label} (no content)`;
+  const head = lines.slice(0, 3).map((l) => `- ${truncate(l.trim(), 200)}`);
+  const tail =
+    lines.length > 6
+      ? lines.slice(-3).map((l) => `- ${truncate(l.trim(), 200)}`)
+      : [];
+  const parts = [`${label} summary`, "", ...head];
+  if (tail.length > 0) parts.push("", "...", "", ...tail);
   return parts.join("\n");
 }

package/src/memory/jobs-store.ts

@@ -1,9 +1,13 @@
-import { and, asc, eq, inArray, lte, notInArray } from "drizzle-orm";
+import { and, asc, eq, inArray, lte, notInArray, sql } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
 import { getLogger } from "../util/logger.js";
 import { truncate } from "../util/truncate.js";
 import { getDb, rawAll, rawChanges } from "./db.js";
+import {
+  isQdrantBreakerOpen,
+  shouldAllowQdrantProbe,
+} from "./qdrant-circuit-breaker.js";
 import { memoryJobs } from "./schema.js";
 
 const log = getLogger("memory-jobs-store");
@@ -82,6 +86,38 @@ export function enqueueMemoryJob(
   return id;
 }
 
+/**
+ * Upsert a debounced job: if a pending job of the same type and conversation
+ * already exists, push its `runAfter` forward instead of creating a duplicate.
+ * This prevents rapid message indexing from spawning redundant jobs.
+ */
+export function upsertDebouncedJob(
+  type: MemoryJobType,
+  payload: { conversationId: string },
+  runAfter: number,
+): void {
+  const db = getDb();
+  const existing = db
+    .select()
+    .from(memoryJobs)
+    .where(
+      and(
+        eq(memoryJobs.type, type),
+        eq(memoryJobs.status, "pending"),
+        sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}`,
+      ),
+    )
+    .get();
+  if (existing) {
+    db.update(memoryJobs)
+      .set({ runAfter, updatedAt: Date.now() })
+      .where(eq(memoryJobs.id, existing.id))
+      .run();
+  } else {
+    enqueueMemoryJob(type, payload, runAfter);
+  }
+}
+
 export function enqueueCleanupStaleSupersededItemsJob(
   retentionMs?: number,
 ): string {
@@ -201,14 +237,33 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
     .all();
 
   const remainingSlots = limit - nonEmbedCandidates.length;
+
+  // When the Qdrant circuit breaker is open, skip embed jobs entirely —
+  // they would just be claimed → fail → deferred, wasting CPU cycles.
+  // Exception: if the cooldown has elapsed (breaker ready for half-open probe),
+  // allow exactly 1 embed job through so the breaker can self-heal.
+  const breakerOpen = isQdrantBreakerOpen();
+  const probeAllowed = breakerOpen && shouldAllowQdrantProbe();
+  const skipEmbedJobs = breakerOpen && !probeAllowed;
+  const embedLimit = probeAllowed ? 1 : remainingSlots;
+
+  if (skipEmbedJobs && remainingSlots > 0) {
+    log.debug("Skipping embed job claims — Qdrant circuit breaker is open");
+  }
+  if (probeAllowed && remainingSlots > 0) {
+    log.debug(
+      "Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
+    );
+  }
+
   const embedCandidates =
-    remainingSlots > 0
+    remainingSlots > 0 && !skipEmbedJobs
       ? db
           .select()
           .from(memoryJobs)
           .where(and(pendingFilter, inArray(memoryJobs.type, EMBED_JOB_TYPES)))
           .orderBy(asc(memoryJobs.runAfter), asc(memoryJobs.createdAt))
-          .limit(remainingSlots)
+          .limit(embedLimit)
           .all()
       : [];
 
@@ -243,8 +298,8 @@ export function completeMemoryJob(id: string): void {
 
 /** Max times a job can be deferred before it is marked as failed. */
 const MAX_DEFERRALS = 50;
-/** Warn when deferrals reach 80% of the limit. */
-const DEFER_WARNING_THRESHOLD = Math.floor(MAX_DEFERRALS * 0.8);
+/** Log warnings at these milestone counts to avoid flooding logs. */
+const DEFERRAL_WARN_MILESTONES = [40, 45];
 /** Base delay in ms for deferred jobs (grows with exponential backoff). */
 const DEFER_BASE_DELAY_MS = 30_000;
 /** Maximum delay cap for deferred jobs (5 minutes). */
@@ -286,7 +341,9 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
     return "failed";
   }
 
-  if (deferrals >= DEFER_WARNING_THRESHOLD) {
+  // Log at milestones only (40, 45) to avoid flooding logs.
+  // At 50, the job fails via the check above, so 40 and 45 are the warnings.
+  if (DEFERRAL_WARN_MILESTONES.includes(deferrals)) {
     log.warn(
       { jobId: id, type: row.type, deferrals, max: MAX_DEFERRALS },
       "Job approaching max deferral limit",

package/src/memory/jobs-worker.ts

@@ -44,6 +44,9 @@ import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
 
 const log = getLogger("memory-jobs-worker");
 
+export const POLL_INTERVAL_MIN_MS = 1_500;
+export const POLL_INTERVAL_MAX_MS = 30_000;
+
 export interface MemoryJobsWorker {
   runOnce(): Promise<number>;
   stop(): void;
@@ -57,24 +60,45 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
 
   let stopped = false;
   let tickRunning = false;
+  let timer: ReturnType<typeof setTimeout>;
+  let currentIntervalMs = POLL_INTERVAL_MIN_MS;
 
   const tick = async () => {
     if (stopped || tickRunning) return;
     tickRunning = true;
     try {
-      await runMemoryJobsOnce({ enableScheduledCleanup: true });
+      const processed = await runMemoryJobsOnce({
+        enableScheduledCleanup: true,
+      });
+      if (processed > 0) {
+        currentIntervalMs = POLL_INTERVAL_MIN_MS;
+      } else {
+        currentIntervalMs = Math.min(
+          currentIntervalMs * 2,
+          POLL_INTERVAL_MAX_MS,
+        );
+      }
     } catch (err) {
       log.error({ err }, "Memory worker tick failed");
+      currentIntervalMs = Math.min(currentIntervalMs * 2, POLL_INTERVAL_MAX_MS);
     } finally {
       tickRunning = false;
     }
   };
 
-  const timer = setInterval(() => {
-    void tick();
-  }, 1500);
-  timer.unref();
-  void tick();
+  const scheduleTick = () => {
+    if (stopped) return;
+    timer = setTimeout(() => {
+      void tick().then(() => {
+        if (!stopped) scheduleTick();
+      });
+    }, currentIntervalMs);
+    (timer as NodeJS.Timeout).unref?.();
+  };
+
+  void tick().then(() => {
+    if (!stopped) scheduleTick();
+  });
 
   return {
     async runOnce(): Promise<number> {
@@ -82,7 +106,7 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
     },
     stop(): void {
       stopped = true;
-      clearInterval(timer);
+      clearTimeout(timer);
     },
   };
 }

package/src/memory/journal-memory.ts

@@ -0,0 +1,214 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+import { and, eq } from "drizzle-orm";
+import { v4 as uuid } from "uuid";
+
+import { getLogger } from "../util/logger.js";
+import { getWorkspaceDir } from "../util/platform.js";
+import { type DrizzleDb, getDb } from "./db.js";
+import { computeMemoryFingerprint } from "./fingerprint.js";
+import { enqueueMemoryJob } from "./jobs-store.js";
+import { memoryItems, memoryItemSources } from "./schema.js";
+
+const log = getLogger("memory-journal");
+
+/**
+ * Process a single journal `.md` file: read content, derive subject, compute
+ * fingerprint, upsert to DB, and enqueue an embed job.
+ *
+ * Returns `true` if a new memory item was inserted, `false` if it already
+ * existed (or was skipped).
+ */
+function upsertSingleJournalFile(
+  filepath: string,
+  filename: string,
+  messageCreatedAt: number,
+  scopeId: string,
+  messageId: string,
+  db: DrizzleDb,
+): boolean {
+  const content = readFileSync(filepath, "utf-8");
+
+  // Derive subject from filename:
+  // strip .md extension, strip leading date prefix, replace hyphens with spaces, capitalize first letter
+  const basename = filename.replace(/\.md$/, "");
+  const withoutDate = basename.replace(/^\d{4}-\d{2}-\d{2}-?/, "");
+  const withSpaces = withoutDate.replace(/-/g, " ");
+  const subject =
+    withSpaces.length > 0
+      ? withSpaces.charAt(0).toUpperCase() + withSpaces.slice(1)
+      : basename;
+
+  const fingerprint = computeMemoryFingerprint(
+    scopeId,
+    "journal",
+    subject,
+    content,
+  );
+
+  const existing = db
+    .select()
+    .from(memoryItems)
+    .where(
+      and(
+        eq(memoryItems.fingerprint, fingerprint),
+        eq(memoryItems.scopeId, scopeId),
+      ),
+    )
+    .get();
+
+  let memoryItemId: string;
+  let inserted = false;
+
+  if (existing) {
+    memoryItemId = existing.id;
+    db.update(memoryItems)
+      .set({
+        lastSeenAt: messageCreatedAt,
+        status: "active",
+      })
+      .where(eq(memoryItems.id, existing.id))
+      .run();
+  } else {
+    memoryItemId = uuid();
+    db.insert(memoryItems)
+      .values({
+        id: memoryItemId,
+        kind: "journal",
+        subject,
+        statement: content,
+        status: "active",
+        confidence: 0.95,
+        importance: 0.8,
+        fingerprint,
+        sourceType: "extraction",
+        sourceMessageRole: "assistant",
+        verificationState: "assistant_inferred",
+        scopeId,
+        firstSeenAt: messageCreatedAt,
+        lastSeenAt: messageCreatedAt,
+        lastUsedAt: null,
+        supersedes: null,
+        overrideConfidence: null,
+      })
+      .run();
+    inserted = true;
+  }
+
+  db.insert(memoryItemSources)
+    .values({
+      memoryItemId,
+      messageId,
+      evidence: content,
+      createdAt: Date.now(),
+    })
+    .onConflictDoNothing()
+    .run();
+
+  enqueueMemoryJob("embed_item", { itemId: memoryItemId });
+
+  return inserted;
+}
+
+/**
+ * Scan the journal directory for `.md` files created during (or after) the
+ * given message timestamp and upsert them as journal memory items with the
+ * raw, unedited file content as the `statement`.
+ *
+ * Also scans immediate subdirectories (e.g. per-user folders like
+ * `journal/sidd/`) so that user-scoped journal entries are indexed alongside
+ * root-level files.
+ *
+ * This bypasses the LLM extraction layer entirely — journal memories are
+ * stored verbatim so they are never summarised or rewritten.
+ *
+ * Returns the number of newly inserted items.
+ */
+export function upsertJournalMemoriesFromDisk(
+  messageCreatedAt: number,
+  scopeId: string,
+  messageId: string,
+): number {
+  try {
+    const journalDir = join(getWorkspaceDir(), "journal");
+
+    let files: string[];
+    try {
+      files = readdirSync(journalDir);
+    } catch {
+      // Directory doesn't exist — no journal entries
+      return 0;
+    }
+
+    // Filter for .md files, excluding readme.md (case-insensitive)
+    const mdFiles = files.filter(
+      (f) => f.endsWith(".md") && f.toLowerCase() !== "readme.md",
+    );
+
+    let upserted = 0;
+    const db = getDb();
+
+    for (const filename of mdFiles) {
+      try {
+        const filepath = join(journalDir, filename);
+        const stat = statSync(filepath);
+        if (!stat.isFile()) continue;
+
+        // Only process files created during or after this message
+        if (stat.birthtimeMs < messageCreatedAt) continue;
+
+        if (upsertSingleJournalFile(filepath, filename, messageCreatedAt, scopeId, messageId, db)) {
+          upserted += 1;
+        }
+      } catch (err) {
+        log.warn(
+          { filename, err: err instanceof Error ? err.message : String(err) },
+          "Failed to process journal file for memory — skipping",
+        );
+      }
+    }
+
+    // Scan per-user journal subdirectories
+    for (const entry of files) {
+      try {
+        const subdirPath = join(journalDir, entry);
+        if (!statSync(subdirPath).isDirectory()) continue;
+
+        const subFiles = readdirSync(subdirPath).filter(
+          (f) => f.endsWith(".md") && f.toLowerCase() !== "readme.md",
+        );
+
+        for (const filename of subFiles) {
+          try {
+            const filepath = join(subdirPath, filename);
+            const stat = statSync(filepath);
+            if (!stat.isFile()) continue;
+
+            // Only process files created during or after this message
+            if (stat.birthtimeMs < messageCreatedAt) continue;
+
+            if (upsertSingleJournalFile(filepath, filename, messageCreatedAt, scopeId, messageId, db)) {
+              upserted += 1;
+            }
+          } catch (err) {
+            log.warn(
+              { filename, err: err instanceof Error ? err.message : String(err) },
+              "Failed to process journal file for memory — skipping",
+            );
+          }
+        }
+      } catch {
+        // Skip unreadable subdirectories
+      }
+    }
+
+    return upserted;
+  } catch (err) {
+    log.warn(
+      { err: err instanceof Error ? err.message : String(err) },
+      "Failed to scan journal directory for memories",
+    );
+    return 0;
+  }
+}

package/src/memory/migrations/001-job-deferrals.ts

@@ -49,3 +49,22 @@ export function migrateJobDeferrals(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Reverse the deferral reconciliation by moving `deferrals` back into `attempts`
+ * for pending embed jobs. Best-effort: jobs that accumulated real deferral counts
+ * after the forward migration ran cannot be distinguished from migrated ones.
+ */
+export function downJobDeferrals(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `
+    UPDATE memory_jobs
+    SET attempts = deferrals,
+        deferrals = 0,
+        updated_at = ${Date.now()}
+    WHERE status = 'pending'
+      AND deferrals > 0
+      AND attempts = 0
+      AND type IN ('embed_segment', 'embed_item', 'embed_summary')
+  `);
+}

package/src/memory/migrations/004-entity-relation-dedup.ts

@@ -91,3 +91,13 @@ export function migrateMemoryEntityRelationDedup(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * No-op down: deduplication is a lossy operation — deleted duplicate rows
+ * cannot be restored. The forward migration merged rows by keeping the most
+ * recent evidence per (source, target, relation) triple; the discarded rows
+ * are permanently lost.
+ */
+export function downMemoryEntityRelationDedup(_database: DrizzleDb): void {
+  // Intentionally empty — irreversible lossy migration.
+}

package/src/memory/migrations/005-fingerprint-scope-unique.ts

@@ -93,3 +93,79 @@ export function migrateMemoryItemsFingerprintScopeUnique(
     raw.exec("PRAGMA foreign_keys = ON");
   }
 }
+
+/**
+ * Reverse the compound (fingerprint, scope_id) unique index change by rebuilding
+ * memory_items with a column-level UNIQUE on fingerprint.
+ *
+ * WARNING: This is dangerous if data now relies on the compound constraint
+ * (i.e., the same fingerprint exists in multiple scopes). In that case, the
+ * rebuild will fail with a UNIQUE constraint violation. This is intentional —
+ * it prevents silent data loss on rollback.
+ */
+export function downMemoryItemsFingerprintScopeUnique(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  // Check if the column-level UNIQUE already exists — if so, nothing to do.
+  const tableDdl = raw
+    .query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_items'`,
+    )
+    .get() as { sql: string } | null;
+  if (
+    !tableDdl ||
+    tableDdl.sql.match(/fingerprint\s+TEXT\s+NOT\s+NULL\s+UNIQUE/i)
+  ) {
+    return;
+  }
+
+  raw.exec("PRAGMA foreign_keys = OFF");
+  try {
+    raw.exec("BEGIN");
+
+    raw.exec(/*sql*/ `
+      CREATE TABLE memory_items_new (
+        id TEXT PRIMARY KEY,
+        kind TEXT NOT NULL,
+        subject TEXT NOT NULL,
+        statement TEXT NOT NULL,
+        status TEXT NOT NULL,
+        confidence REAL NOT NULL,
+        fingerprint TEXT NOT NULL UNIQUE,
+        first_seen_at INTEGER NOT NULL,
+        last_seen_at INTEGER NOT NULL,
+        last_used_at INTEGER,
+        importance REAL,
+        access_count INTEGER NOT NULL DEFAULT 0,
+        valid_from INTEGER,
+        invalid_at INTEGER,
+        verification_state TEXT NOT NULL DEFAULT 'assistant_inferred',
+        scope_id TEXT NOT NULL DEFAULT 'default'
+      )
+    `);
+
+    raw.exec(/*sql*/ `
+      INSERT INTO memory_items_new
+      SELECT id, kind, subject, statement, status, confidence, fingerprint,
+             first_seen_at, last_seen_at, last_used_at, importance, access_count,
+             valid_from, invalid_at, verification_state, scope_id
+      FROM memory_items
+    `);
+
+    raw.exec(/*sql*/ `DROP TABLE memory_items`);
+    raw.exec(/*sql*/ `ALTER TABLE memory_items_new RENAME TO memory_items`);
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  } finally {
+    raw.exec("PRAGMA foreign_keys = ON");
+  }
+}

package/src/memory/migrations/006-scope-salted-fingerprints.ts

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import { computeMemoryFingerprint } from "../fingerprint.js";
 
@@ -75,3 +77,51 @@ export function migrateMemoryItemsScopeSaltedFingerprints(
     throw e;
   }
 }
+
+/**
+ * Reverse the scope-salted fingerprint migration by recomputing fingerprints
+ * WITHOUT the scope_id prefix.
+ *
+ * Old format: sha256(`${kind}|${subject.toLowerCase()}|${statement.toLowerCase()}`)
+ */
+export function downMemoryItemsScopeSaltedFingerprints(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  interface ItemRow {
+    id: string;
+    kind: string;
+    subject: string;
+    statement: string;
+  }
+
+  const items = raw
+    .query(`SELECT id, kind, subject, statement FROM memory_items`)
+    .all() as ItemRow[];
+
+  if (items.length === 0) return;
+
+  try {
+    raw.exec("BEGIN");
+
+    const updateStmt = raw.prepare(
+      `UPDATE memory_items SET fingerprint = ? WHERE id = ?`,
+    );
+
+    for (const item of items) {
+      const normalized = `${item.kind}|${item.subject.toLowerCase()}|${item.statement.toLowerCase()}`;
+      const fingerprint = createHash("sha256").update(normalized).digest("hex");
+      updateStmt.run(fingerprint, item.id);
+    }
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  }
+}

package/src/memory/migrations/007-assistant-id-to-self.ts

@@ -265,3 +265,13 @@ export function migrateAssistantIdToSelf(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * No-op down: the original assistant_id values are not recoverable. The forward
+ * migration normalized all assistant_id values to "self" and merged/deduplicated
+ * rows where the same logical entity existed under both the real assistantId and
+ * "self". The original per-assistant IDs are permanently lost.
+ */
+export function downAssistantIdToSelf(_database: DrizzleDb): void {
+  // Intentionally empty — original assistant_id values cannot be restored.
+}