@vellumai/assistant 0.8.0 → 0.8.2

package/src/documents/document-store.ts

@@ -5,11 +5,26 @@
  * background jobs (e.g. proactive artifact generation) can persist documents
  * without going through the HTTP layer.
  */
-import { rawGet, rawRun } from "../memory/raw-query.js";
+import { rawAll, rawGet, rawRun } from "../memory/raw-query.js";
 import { getLogger } from "../util/logger.js";
 
 const log = getLogger("document-store");
 
+// ---------------------------------------------------------------------------
+// Shared types
+// ---------------------------------------------------------------------------
+
+/** A document record with camelCase field names, mapped from the SQLite row. */
+export interface DocumentRecord {
+  surfaceId: string;
+  conversationId: string;
+  title: string;
+  content: string;
+  wordCount: number;
+  createdAt: number;
+  updatedAt: number;
+}
+
 // ---------------------------------------------------------------------------
 // Junction table helper
 // ---------------------------------------------------------------------------
@@ -27,6 +42,204 @@ export function addDocumentConversation(
   );
 }
 
+// ---------------------------------------------------------------------------
+// Shared query helpers
+// ---------------------------------------------------------------------------
+
+interface DocumentRow {
+  surface_id: string;
+  conversation_id: string;
+  title: string;
+  content: string;
+  word_count: number;
+  created_at: number;
+  updated_at: number;
+}
+
+type DocumentListRow = Omit<DocumentRow, "content">;
+
+function escapeSqlLikePattern(value: string): string {
+  return value.replace(/[\\%_]/g, "\\$&");
+}
+
+function mapRowToRecord(row: DocumentRow): DocumentRecord {
+  return {
+    surfaceId: row.surface_id,
+    conversationId: row.conversation_id,
+    title: row.title,
+    content: row.content,
+    wordCount: row.word_count,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+  };
+}
+
+/** Look up a single document by surface ID. Returns `null` when not found. */
+export function getDocumentById(surfaceId: string): DocumentRecord | null {
+  try {
+    const row = rawGet<DocumentRow>(
+      /*sql*/ `SELECT surface_id, conversation_id, title, content, word_count, created_at, updated_at
+       FROM documents
+       WHERE surface_id = ?`,
+      surfaceId,
+    );
+
+    if (!row) {
+      log.info({ surfaceId }, "Document not found");
+      return null;
+    }
+
+    log.info({ surfaceId }, "Loaded document");
+    return mapRowToRecord(row);
+  } catch (error) {
+    log.error({ err: error, surfaceId }, "Load error");
+    return null;
+  }
+}
+
+/** Return true when a document is associated with a conversation. */
+export function isDocumentAssociatedWithConversation(
+  surfaceId: string,
+  conversationId: string,
+): boolean {
+  try {
+    const row = rawGet<{ found: number }>(
+      /*sql*/ `
+      SELECT 1 AS found
+      FROM document_conversations
+      WHERE surface_id = ? AND conversation_id = ?
+      LIMIT 1
+      `,
+      surfaceId,
+      conversationId,
+    );
+    return row != null;
+  } catch (error) {
+    log.error(
+      { err: error, surfaceId, conversationId },
+      "Document association check error",
+    );
+    return false;
+  }
+}
+
+/**
+ * List documents for a given conversation (via the junction table).
+ * Returns an empty array when the conversation has no documents or on error.
+ */
+export function getDocumentsForConversation(
+  conversationId: string,
+): Omit<DocumentRecord, "content">[] {
+  try {
+    const rows = rawAll<DocumentListRow>(
+      /*sql*/ `
+      SELECT d.surface_id, dc.conversation_id AS conversation_id,
+             d.title, d.word_count, d.created_at, d.updated_at
+      FROM documents d
+      INNER JOIN document_conversations dc ON d.surface_id = dc.surface_id
+      WHERE dc.conversation_id = ?
+      ORDER BY d.updated_at DESC
+      `,
+      conversationId,
+    );
+
+    log.info(
+      { conversationId, count: rows.length },
+      "Listed documents for conversation",
+    );
+    return rows.map((row) => ({
+      surfaceId: row.surface_id,
+      conversationId: row.conversation_id,
+      title: row.title,
+      wordCount: row.word_count,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+    }));
+  } catch (error) {
+    log.error({ err: error, conversationId }, "List error");
+    return [];
+  }
+}
+
+/**
+ * Search documents by title substring (case-insensitive).
+ * When `conversationId` is supplied, only documents associated with that
+ * conversation are returned.
+ * Returns documents ordered by most recently updated.
+ */
+export function searchDocumentsByTitle(
+  query: string,
+  options: { conversationId?: string } = {},
+): Omit<DocumentRecord, "content">[] {
+  try {
+    const pattern = `%${escapeSqlLikePattern(query)}%`;
+    const rows = options.conversationId
+      ? rawAll<DocumentListRow>(
+          /*sql*/ `
+          SELECT d.surface_id, dc.conversation_id AS conversation_id,
+                 d.title, d.word_count, d.created_at, d.updated_at
+          FROM documents d
+          INNER JOIN document_conversations dc ON d.surface_id = dc.surface_id
+          WHERE dc.conversation_id = ?
+            AND d.title COLLATE NOCASE LIKE ? ESCAPE '\\'
+          ORDER BY d.updated_at DESC
+          LIMIT 20
+          `,
+          options.conversationId,
+          pattern,
+        )
+      : rawAll<DocumentListRow>(
+          /*sql*/ `
+          SELECT surface_id, conversation_id, title, word_count, created_at, updated_at
+          FROM documents
+          WHERE title COLLATE NOCASE LIKE ? ESCAPE '\\'
+          ORDER BY updated_at DESC
+          LIMIT 20
+          `,
+          pattern,
+        );
+
+    log.info(
+      { query, conversationId: options.conversationId, count: rows.length },
+      "Searched documents by title",
+    );
+    return rows.map((row) => ({
+      surfaceId: row.surface_id,
+      conversationId: row.conversation_id,
+      title: row.title,
+      wordCount: row.word_count,
+      createdAt: row.created_at,
+      updatedAt: row.updated_at,
+    }));
+  } catch (error) {
+    log.error({ err: error, query }, "Search error");
+    return [];
+  }
+}
+
+/**
+ * Delete a document and its conversation associations.
+ * Returns `true` if the document existed and was deleted, `false` otherwise.
+ */
+export function deleteDocument(surfaceId: string): boolean {
+  try {
+    const changes = rawRun(
+      /*sql*/ `DELETE FROM documents WHERE surface_id = ?`,
+      surfaceId,
+    );
+    rawRun(
+      /*sql*/ `DELETE FROM document_conversations WHERE surface_id = ?`,
+      surfaceId,
+    );
+    const existed = changes > 0;
+    log.info({ surfaceId, existed }, "Deleted document");
+    return existed;
+  } catch (error) {
+    log.error({ err: error, surfaceId }, "Delete error");
+    return false;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Document persistence
 // ---------------------------------------------------------------------------
@@ -89,7 +302,7 @@ export function updateDocumentContent(
   surfaceId: string,
   markdown: string,
   mode: string,
-): void {
+): { success: true } | { success: false; error: string } {
   try {
     const existing = rawGet<{ content: string }>(
       /*sql*/ `SELECT content FROM documents WHERE surface_id = ?`,
@@ -97,7 +310,7 @@ export function updateDocumentContent(
     );
     if (!existing) {
       log.info({ surfaceId }, "No persisted document to update");
-      return;
+      return { success: false, error: "Document not found" };
     }
     const sep = mode === "append" && existing.content.length > 0 ? "\n\n" : "";
     const newContent =
@@ -113,7 +326,12 @@ export function updateDocumentContent(
       surfaceId,
     );
     log.info({ surfaceId, mode }, "Updated document content");
+    return { success: true };
   } catch (error) {
     log.error({ err: error, surfaceId }, "Document content update error");
+    return {
+      success: false,
+      error: error instanceof Error ? error.message : "Unknown error",
+    };
   }
 }

package/src/embedded/plugin-api.ts

@@ -0,0 +1,40 @@
+/**
+ * Embedded handle for `@vellumai/plugin-api`.
+ *
+ * Standard `import * as pluginApi from "../plugin-api/index.ts"` lets Bun's
+ * normal bundler walk the plugin-api module graph at build time — relative
+ * imports inside plugin-api (`./types.js`, future runtime siblings) are
+ * inlined naturally, with no separate build step. In `bun --compile`, this
+ * means plugin-api ships as part of the assistant binary's regular code
+ * graph; in JIT/Docker, it loads from source.
+ *
+ * The loaded namespace is then installed on `globalThis` under a versioned
+ * symbol. The boot-time shim writer (`ensurePluginApiShim`) enumerates
+ * {@link PLUGIN_API_EXPORTS} and generates a tiny ESM module at
+ * `<workspaceDir>/node_modules/@vellumai/plugin-api/index.js` that
+ * re-binds each runtime export from `globalThis`. User plugins that
+ * `import { ... } from "@vellumai/plugin-api"` walk up to that shim and
+ * pick up the bindings.
+ *
+ * Type-only exports erase before this module loads, so `Object.keys`
+ * sees only runtime exports. That's correct — types are a dev-time
+ * concern, resolved against plugin-api source via tsconfig path-mapping
+ * (or, post-PR-5, against a generated `.d.ts` next to the runtime
+ * shim).
+ */
+
+import * as pluginApi from "../plugin-api/index.js";
+
+/** Symbol key under which the plugin-api namespace is published on globalThis. */
+export const PLUGIN_API_REGISTRY_KEY = Symbol.for("vellum.plugin-api.v1");
+
+// Install on globalThis once at module-load time. The shim writer reads
+// `PLUGIN_API_EXPORTS` to know which bindings to re-export; the shim's
+// generated body then reads `globalThis[PLUGIN_API_REGISTRY_KEY]` to grab
+// the live module namespace.
+(globalThis as Record<symbol, unknown>)[PLUGIN_API_REGISTRY_KEY] = pluginApi;
+
+/** Names of the runtime exports the workspace shim should re-bind. */
+export const PLUGIN_API_EXPORTS: readonly string[] = Object.freeze(
+  Object.keys(pluginApi),
+);

package/src/export/transcript-formatter.ts

@@ -97,6 +97,61 @@ function parseContent(raw: string): ContentBlock[] {
   }
 }
 
+type TranscriptMessage = ReturnType<typeof getMessages>[number];
+
+/**
+ * Format a slice of messages as a transcript body (no top-of-conversation
+ * header). Used by background jobs that process incremental slices — the
+ * memory-retrospective job re-renders only the messages added since its
+ * last successful run rather than the whole conversation. The format
+ * matches `buildAnalysisTranscript` per message so downstream agents see a
+ * consistent shape regardless of whether the input is a full transcript or
+ * a slice.
+ */
+export function formatMessageSliceForTranscript(
+  messages: TranscriptMessage[],
+): string {
+  const lines: string[] = [];
+  for (const msg of messages) {
+    appendMessageBlock(lines, msg);
+  }
+  return lines.join("\n");
+}
+
+function appendMessageBlock(lines: string[], msg: TranscriptMessage): void {
+  const role = formatRole(msg.role);
+  const time = formatTimestamp(msg.createdAt);
+  const content = parseContent(msg.content);
+  const text = extractAnalysisText(content);
+
+  lines.push(`## ${role} (${time})`);
+  lines.push(text);
+  lines.push("");
+
+  if (msg.metadata) {
+    try {
+      const parsed = messageMetadataSchema.safeParse(JSON.parse(msg.metadata));
+      if (parsed.success && parsed.data.subagentNotification) {
+        const notif = parsed.data.subagentNotification;
+        if (
+          (notif.status === "completed" ||
+            notif.status === "failed" ||
+            notif.status === "aborted") &&
+          notif.conversationId
+        ) {
+          const subMessages = getMessages(notif.conversationId);
+          lines.push(`### Subagent: ${notif.label} (${notif.status})`);
+          lines.push("");
+          lines.push(formatSubagentMessages(subMessages));
+          lines.push("");
+        }
+      }
+    } catch {
+      // Skip unparseable metadata
+    }
+  }
+}
+
 export function buildAnalysisTranscript(conversationId: string): string {
   const conversation = getConversation(conversationId);
   if (!conversation) {
@@ -124,11 +179,15 @@ export function buildAnalysisTranscript(conversationId: string): string {
     // Check for subagent notifications in metadata
     if (msg.metadata) {
       try {
-        const parsed = messageMetadataSchema.safeParse(JSON.parse(msg.metadata));
+        const parsed = messageMetadataSchema.safeParse(
+          JSON.parse(msg.metadata),
+        );
         if (parsed.success && parsed.data.subagentNotification) {
           const notif = parsed.data.subagentNotification;
           if (
-            (notif.status === "completed" || notif.status === "failed" || notif.status === "aborted") &&
+            (notif.status === "completed" ||
+              notif.status === "failed" ||
+              notif.status === "aborted") &&
             notif.conversationId
           ) {
             const subMessages = getMessages(notif.conversationId);

package/src/filing/filing-service.ts

@@ -8,8 +8,7 @@ import {
   diskPressureBackgroundSkipLogFields,
   shouldLogDiskPressureBackgroundSkip,
 } from "../daemon/disk-pressure-background-gate.js";
-import { processMessage } from "../daemon/process-message.js";
-import { bootstrapConversation } from "../memory/conversation-bootstrap.js";
+import { runBackgroundJob } from "../runtime/background-job-runner.js";
 import { getLogger } from "../util/logger.js";
 import { getWorkspaceDir } from "../util/platform.js";
 import { stripCommentLines } from "../util/strip-comment-lines.js";
@@ -18,6 +17,11 @@ const log = getLogger("filing-service");
 
 const FILING_TIMEOUT_MS = 15 * 60 * 1000; // 15 minutes
 
+// When compaction skips because a filing run holds the serialization lock,
+// retry on this near-term cadence so phase-aligned 24h timers don't starve
+// compaction across consecutive ticks.
+const COMPACTION_CONTENDED_RETRY_MS = 10 * 60 * 1000; // 10 minutes
+
 const FILING_PROMPT_TEMPLATE = `You are running a periodic knowledge base filing job. This is a background maintenance task focused on the buffer.
 
 Read \`pkb/buffer.md\`. For each item in the buffer:
@@ -66,11 +70,8 @@ If the disk shape changed (files split, files moved, files created, files remove
 This is your knowledge base — keep it sharp.`;
 
 export interface FilingDeps {
-  onConversationCreated?: (info: {
-    conversationId: string;
-    title: string;
-  }) => void;
   getCurrentHour?: () => number;
+  compactionContendedRetryMs?: number;
 }
 
 export class FilingService {
@@ -84,8 +85,10 @@ export class FilingService {
   private readonly deps: FilingDeps;
   private timer: ReturnType<typeof setInterval> | null = null;
   private compactionTimer: ReturnType<typeof setInterval> | null = null;
+  private compactionRetryTimer: ReturnType<typeof setTimeout> | null = null;
   private activeRun: Promise<void> | null = null;
   private activeCompactionRun: Promise<void> | null = null;
+  private stopped = false;
   private _lastRunAt: number | null = null;
   private _nextRunAt: number | null = null;
   private _lastCompactionAt: number | null = null;
@@ -113,6 +116,7 @@ export class FilingService {
   }
 
   start(): void {
+    this.stopped = false;
     const fullConfig = getConfig();
     if (fullConfig.memory.v2.enabled) {
       log.info("Filing service disabled — memory v2 is active");
@@ -162,12 +166,14 @@ export class FilingService {
       clearInterval(this.compactionTimer);
       this.compactionTimer = null;
     }
+    this.clearCompactionRetry();
     this._nextRunAt = null;
     this._nextCompactionAt = null;
     this.start();
   }
 
   async stop(): Promise<void> {
+    this.stopped = true;
     if (this.timer) {
       clearInterval(this.timer);
       this.timer = null;
@@ -176,6 +182,7 @@ export class FilingService {
       clearInterval(this.compactionTimer);
       this.compactionTimer = null;
     }
+    this.clearCompactionRetry();
     this._nextRunAt = null;
     this._nextCompactionAt = null;
     const inflight: Promise<void>[] = [];
@@ -217,6 +224,13 @@ export class FilingService {
       return false;
     }
 
+    if (this.activeCompactionRun) {
+      log.debug(
+        "Compaction run in progress, skipping filing to avoid concurrent PKB writes",
+      );
+      return false;
+    }
+
     // Skip if buffer is empty — no work to do
     if (!force && !this.hasBufferContent()) {
       log.debug("Buffer is empty, skipping filing");
@@ -227,15 +241,7 @@ export class FilingService {
     const run = this.executeRun();
     this.activeRun = run;
     try {
-      await Promise.race([
-        run,
-        new Promise<never>((_, reject) =>
-          setTimeout(
-            () => reject(new Error("Filing execution timed out")),
-            FILING_TIMEOUT_MS,
-          ),
-        ),
-      ]);
+      await run;
     } finally {
       this.activeRun = null;
       this._lastRunAt = Date.now();
@@ -271,18 +277,21 @@ export class FilingService {
       return false;
     }
 
+    if (this.activeRun) {
+      log.debug(
+        "Filing run in progress, skipping compaction to avoid concurrent PKB writes",
+      );
+      this.scheduleCompactionRetry(
+        this.deps.compactionContendedRetryMs ?? COMPACTION_CONTENDED_RETRY_MS,
+      );
+      return false;
+    }
+
+    this.clearCompactionRetry();
     const run = this.executeCompactionRun();
     this.activeCompactionRun = run;
     try {
-      await Promise.race([
-        run,
-        new Promise<never>((_, reject) =>
-          setTimeout(
-            () => reject(new Error("Compaction execution timed out")),
-            FILING_TIMEOUT_MS,
-          ),
-        ),
-      ]);
+      await run;
     } finally {
       this.activeCompactionRun = null;
       this._lastCompactionAt = Date.now();
@@ -308,6 +317,29 @@ export class FilingService {
     this._nextCompactionAt = Date.now() + intervalMs;
   }
 
+  private scheduleCompactionRetry(delayMs: number): void {
+    this.clearCompactionRetry();
+    if (this.stopped) return;
+    this.compactionRetryTimer = setTimeout(() => {
+      this.compactionRetryTimer = null;
+      if (this.stopped) return;
+      this.runCompactionOnce().catch((err) => {
+        log.error({ err }, "Compaction retry failed");
+      });
+    }, delayMs);
+    // unref so the pending retry doesn't keep the daemon process alive on
+    // shutdown paths that don't call stop().
+    this.compactionRetryTimer.unref?.();
+    this._nextCompactionAt = Date.now() + delayMs;
+  }
+
+  private clearCompactionRetry(): void {
+    if (this.compactionRetryTimer) {
+      clearTimeout(this.compactionRetryTimer);
+      this.compactionRetryTimer = null;
+    }
+  }
+
   private shouldSkipForDiskPressure(source: "filing" | "compaction"): boolean {
     const diskPressureGate = checkDiskPressureBackgroundGate("background-work");
     if (diskPressureGate.action === "allow") return false;
@@ -338,7 +370,8 @@ export class FilingService {
 
   private executeRun(): Promise<void> {
     return this.executeBackgroundJob({
-      title: "Knowledge base filing",
+      jobName: "filing",
+      systemHint: "Knowledge base filing",
       prompt: FILING_PROMPT_TEMPLATE,
       callSite: "filingAgent",
     });
@@ -346,47 +379,40 @@ export class FilingService {
 
   private executeCompactionRun(): Promise<void> {
     return this.executeBackgroundJob({
-      title: "Knowledge base compaction",
+      jobName: "compaction",
+      systemHint: "Knowledge base compaction",
       prompt: COMPACTION_PROMPT_TEMPLATE,
       callSite: "compactionAgent",
     });
   }
 
   private async executeBackgroundJob(opts: {
-    title: string;
+    jobName: string;
+    systemHint: string;
     prompt: string;
     callSite: LLMCallSite;
   }): Promise<void> {
-    log.info({ title: opts.title }, "Running background job");
-
-    try {
-      const conversation = bootstrapConversation({
-        conversationType: "background",
-        source: "filing",
-        groupId: "system:background",
-        origin: "filing",
-        systemHint: opts.title,
-      });
-
-      this.deps.onConversationCreated?.({
-        conversationId: conversation.id,
-        title: opts.title,
-      });
-
-      await processMessage(conversation.id, opts.prompt, undefined, {
-        trustContext: {
-          sourceChannel: "vellum",
-          trustClass: "guardian",
-        },
-        callSite: opts.callSite,
-      });
+    log.info({ jobName: opts.jobName }, "Running background job");
+
+    const result = await runBackgroundJob({
+      jobName: opts.jobName,
+      source: "filing",
+      systemHint: opts.systemHint,
+      prompt: opts.prompt,
+      trustContext: {
+        sourceChannel: "vellum",
+        trustClass: "guardian",
+      },
+      callSite: opts.callSite,
+      timeoutMs: FILING_TIMEOUT_MS,
+      origin: "filing",
+    });
 
+    if (result.ok) {
       log.info(
-        { conversationId: conversation.id, title: opts.title },
+        { conversationId: result.conversationId, jobName: opts.jobName },
         "Background job completed",
       );
-    } catch (err) {
-      log.error({ err, title: opts.title }, "Background job failed");
     }
   }
 }