@vellumai/assistant 0.10.7 → 0.10.8-dev.202607102228.5945895

package/src/persistence/attachments-store.ts

@@ -340,6 +340,38 @@ function scopeAttachmentToConversation(
   return row.id;
 }
 
+/**
+ * Scope a pre-uploaded attachment into a conversation WITHOUT linking it to a
+ * message, returning the scoped row's metadata. Mirrors {@link createInlineAttachment}
+ * for the already-uploaded case: it gives the persist path the final attachment
+ * id and stored MIME/size before it serializes the message content into a
+ * workspace reference. The message link is written separately once the message
+ * id exists. Returns null when the attachment row cannot be read after scoping.
+ */
+export function scopeAttachmentToMessageConversation(
+  conversationId: string,
+  conversationCreatedAt: number,
+  attachmentId: string,
+): (StoredAttachment & { filePath: string | null }) | null {
+  const scopedId = scopeAttachmentToConversation(
+    attachmentId,
+    conversationId,
+    conversationCreatedAt,
+  );
+  const row = getAttachmentRow(scopedId);
+  if (!row) return null;
+  return {
+    id: row.id,
+    originalFilename: row.originalFilename,
+    mimeType: row.mimeType,
+    sizeBytes: row.sizeBytes,
+    kind: row.kind,
+    thumbnailBase64: null,
+    createdAt: row.createdAt,
+    filePath: row.filePath ?? null,
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Size and encoding limits
 // ---------------------------------------------------------------------------
@@ -687,10 +719,13 @@ export function getFilePathBySourcePath(
 }
 
 /**
- * Return the raw binary content for an attachment by reading from its
- * on-disk file path.
+ * Return the raw binary content for a stored attachment by reading from its
+ * on-disk file path (or its inline base64, for legacy rows).
  *
- * Returns null if the attachment does not exist or the file is missing.
+ * Returns null if the attachment does not exist or the file is missing. To
+ * resolve an image/file block's `source` (base64 or workspace reference) to
+ * bytes, use `mediaSourceBytes` / `resolveMediaSourceData` in
+ * `providers/media-resolve.ts`, which delegates here for the attachment route.
  */
 export function getAttachmentContent(attachmentId: string): Buffer | null {
   const row = getAttachmentRow(attachmentId);
@@ -810,23 +845,33 @@ export function uploadAttachment(
   };
 }
 
-export function attachInlineAttachmentToMessage(
-  messageId: string,
-  position: number,
+export interface CreateInlineAttachmentOptions {
+  sourcePath?: string;
+  skipSizeLimit?: boolean;
+  /**
+   * Store HEIF/HEIC content as a JPEG master (with the filename extension
+   * rewritten) so every client surface can render it. User-sourced ingress
+   * opts in; assistant-produced attachments are stored verbatim — if the
+   * assistant deliberately emits a HEIC, rewriting it would be wrong.
+   */
+  normalizeImage?: boolean;
+}
+
+/**
+ * Create an attachment row from inline base64, materialized into the
+ * conversation's attachments/ directory, WITHOUT linking it to a message. The
+ * caller links it separately (`insertMessageAttachmentLink`) once the message
+ * id exists — which lets the persist path create the row (and learn its id,
+ * normalized MIME type, and byte size) before it serializes the message
+ * content into a workspace reference.
+ */
+export function createInlineAttachment(
+  conversationId: string,
+  conversationCreatedAt: number,
   filename: string,
   mimeType: string,
   dataBase64: string,
-  options?: {
-    sourcePath?: string;
-    skipSizeLimit?: boolean;
-    /**
-     * Store HEIF/HEIC content as a JPEG master (with the filename extension
-     * rewritten) so every client surface can render it. User-sourced ingress
-     * opts in; assistant-produced attachments are stored verbatim — if the
-     * assistant deliberately emits a HEIC, rewriting it would be wrong.
-     */
-    normalizeImage?: boolean;
-  },
+  options?: CreateInlineAttachmentOptions,
 ): StoredAttachment & { filePath: string } {
   if (options?.normalizeImage) {
     ({ filename, mimeType, dataBase64 } = normalizeUploadedImageBase64(
@@ -839,14 +884,10 @@ export function attachInlineAttachmentToMessage(
   const sizeBytes = validateAttachmentPayload(dataBase64, {
     skipSizeLimit: options?.skipSizeLimit,
   });
-  const ctx = getMessageConversationContext(messageId);
-  if (!ctx) {
-    throw new Error(`Message not found: ${messageId}`);
-  }
 
   const attachDir = getConversationAttachmentsDirPath(
-    ctx.conversationId,
-    ctx.conversationCreatedAt,
+    conversationId,
+    conversationCreatedAt,
   );
   mkdirSync(attachDir, { recursive: true });
   const resolvedName = resolveUniqueFilename(attachDir, filename);
@@ -874,15 +915,13 @@ export function attachInlineAttachmentToMessage(
 
   if (options?.sourcePath) {
     rawRun(
-      "attachments:attachInline:sourcePath",
+      "attachments:createInline:sourcePath",
       `UPDATE attachments SET source_path = ? WHERE id = ?`,
       options.sourcePath,
       id,
     );
   }
 
-  insertMessageAttachmentLink(messageId, id, position);
-
   return {
     id,
     originalFilename: filename,
@@ -895,6 +934,31 @@ export function attachInlineAttachmentToMessage(
   };
 }
 
+export function attachInlineAttachmentToMessage(
+  messageId: string,
+  position: number,
+  filename: string,
+  mimeType: string,
+  dataBase64: string,
+  options?: CreateInlineAttachmentOptions,
+): StoredAttachment & { filePath: string } {
+  const ctx = getMessageConversationContext(messageId);
+  if (!ctx) {
+    throw new Error(`Message not found: ${messageId}`);
+  }
+
+  const stored = createInlineAttachment(
+    ctx.conversationId,
+    ctx.conversationCreatedAt,
+    filename,
+    mimeType,
+    dataBase64,
+    options,
+  );
+  insertMessageAttachmentLink(messageId, stored.id, position);
+  return stored;
+}
+
 export function attachFileBackedAttachmentToMessage(
   messageId: string,
   position: number,

package/src/persistence/auto-analysis-constants.ts

@@ -1,17 +1,8 @@
-/**
- * Constants extracted from auto-analysis-guard.ts to break the
- * conversation-crud ↔ auto-analysis-guard cycle.
- */
-
 /**
  * Sentinel value for the `source` column of auto-analysis conversations.
- * Used both when creating them and when querying "all except auto-analysis."
+ * The auto-analysis feature is retired, but rows with this source persist
+ * on existing installs — consumers use it to keep those legacy rows
+ * rendered correctly (feed display labels) and excluded from memory
+ * extraction and context search.
  */
 export const AUTO_ANALYSIS_SOURCE = "auto-analysis";
-
-/**
- * Dedicated `group_id` value for auto-analysis rolling conversations.
- * Placed in the `system:background` group alongside heartbeat and filing
- * conversations, rendered as a "Reflections" sub-group in the sidebar.
- */
-export const AUTO_ANALYSIS_GROUP_ID = "system:background";

package/src/persistence/bulk-write-gate.ts

@@ -0,0 +1,82 @@
+// ---------------------------------------------------------------------------
+// Bulk-write gate — process-wide FIFO serialization of bulk main-DB writers.
+// ---------------------------------------------------------------------------
+//
+// The batched bulk writers (fork message-copy, conversation-row batch delete)
+// each hold the main database's write lock only briefly per batch — but SQLite
+// has a single writer, so two bulk streams running concurrently gain no
+// throughput. They only convoy: every batch of each stream contends with the
+// other's (waiting up to `busy_timeout`), and the inter-batch yield each
+// stream inserts for foreground fairness is consumed by the sibling stream
+// instead of the live user turn it was meant for. Concurrent streams are
+// reachable because the memory jobs worker groups jobs by
+// `(type, conversationId)` — retrospectives for two different conversations
+// run in parallel.
+//
+// This gate serializes those streams within the process: one bulk writer runs
+// at a time, the rest queue FIFO, and the write lock (plus the yields) go back
+// to foreground writers. Waiters are bounded by the jobs worker's lane
+// concurrency, and the durable jobs queue upstream coalesces to one pending
+// row per conversation, so the gate cannot accumulate an unbounded backlog.
+//
+// Scope and discipline:
+//   - Per-process. The daemon and each worker process have their own gate;
+//     bulk writers live almost entirely in the memory worker, so cross-process
+//     overlap is rare.
+//   - Main DB only. Dedicated files (e.g. the logs DB) have their own write
+//     locks and must not queue behind main-DB streams.
+//   - Callers wrap only their batch loop — never an LLM call, a wake, or an
+//     open transaction — so the worst-case hold is one bulk pass, itself
+//     bounded by {@link BULK_BATCH_TIMEOUT_MS} per batch.
+
+import { getLogger } from "../util/logger.js";
+import { Mutex } from "../util/mutex.js";
+
+const log = getLogger("bulk-write-gate");
+
+/**
+ * Per-batch subprocess timeout for gated bulk writers. `runAsyncSqlite`'s
+ * default cap (1 h) is sized for legitimately long single statements like
+ * `VACUUM`; a gated bulk batch is a sub-second write, so a batch anywhere near
+ * this bound is wedged (stale lock, dead disk) and must fail rather than hold
+ * the gate — and every queued bulk writer behind it — for the full default.
+ */
+export const BULK_BATCH_TIMEOUT_MS = 5 * 60 * 1000;
+
+/** Gate waits longer than this are logged — the convoy-formation signal. */
+const CONTENTION_LOG_THRESHOLD_MS = 1000;
+
+const gate = new Mutex();
+let heldBy: string | null = null;
+
+/** Label of the bulk writer currently holding the gate, or null when free. */
+export function bulkWriteGateHolder(): string | null {
+  return heldBy;
+}
+
+/**
+ * Run `fn` as the sole bulk main-DB writer in this process. Callers queue
+ * FIFO; the gate is released when `fn` settles (resolve or reject), so a
+ * throwing writer never wedges the queue.
+ */
+export async function withBulkWriteGate<T>(
+  label: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const waitStartMs = Date.now();
+  return await gate.withLock(async () => {
+    const waitedMs = Date.now() - waitStartMs;
+    if (waitedMs >= CONTENTION_LOG_THRESHOLD_MS) {
+      log.info(
+        { label, waitedMs },
+        "bulk-write gate: waited for prior bulk writers",
+      );
+    }
+    heldBy = label;
+    try {
+      return await fn();
+    } finally {
+      heldBy = null;
+    }
+  });
+}

package/src/persistence/cleanup-schedule-state.ts

@@ -1,37 +1,67 @@
 /**
- * Shared state for the cleanup-scheduler throttle.
+ * Per-job throttle state for the cleanup scheduler.
  *
- * `maybeEnqueueScheduledCleanupJobs` in jobs-worker.ts gates cleanup-job
- * enqueueing behind a 6-hour window (configurable via
- * memory.cleanup.enqueueIntervalMs). This module owns the "last enqueue"
- * timestamp so that code paths outside jobs-worker — notably
- * ConfigWatcher.refreshConfigFromSources — can reset the throttle without
- * pulling in jobs-worker's large transitive import graph.
+ * Each scheduled cleanup job is enqueued on a cadence equal to its own
+ * retention window: a job that keeps data for N is re-enqueued at most once
+ * per N. So conversation pruning, LLM-request-log pruning, and audit-log
+ * (`tool_invocations`) pruning each run on their own independent schedule
+ * derived from `conversationRetentionDays`, `llmRequestLogRetentionMs`, and
+ * `auditLog.retentionDays` respectively.
+ *
+ * `maybeEnqueueScheduledCleanupJobs` in jobs-worker.ts owns that decision;
+ * this module owns the in-memory per-job "last enqueue" timestamps so that code
+ * paths outside jobs-worker — notably ConfigWatcher.refreshConfigFromSources —
+ * can reset the throttle without pulling in jobs-worker's large transitive
+ * import graph.
+ *
+ * These timestamps are the hot-path source of truth but are not themselves
+ * durable: jobs-worker persists each enqueue to a checkpoint and seeds this map
+ * from those checkpoints at startup, so an unchanged job's cadence survives a
+ * restart instead of re-firing on every boot.
  *
  * The ConfigWatcher uses resetCleanupScheduleThrottle() to ensure that
  * retention changes made via the UI (which flow through config.json →
- * invalidateConfigCache → refreshConfigFromSources) take effect on the
- * very next scheduler tick instead of waiting out the remaining window.
+ * invalidateConfigCache → refreshConfigFromSources) take effect on the very
+ * next scheduler tick instead of waiting out the remaining window.
  */
 
-let lastScheduledCleanupEnqueueMs = 0;
+export type CleanupJobKind =
+  | "conversations"
+  | "llm_request_logs"
+  | "tool_invocations";
+
+const lastScheduledCleanupEnqueueMs: Record<CleanupJobKind, number> = {
+  conversations: 0,
+  llm_request_logs: 0,
+  tool_invocations: 0,
+};
 
-/** Read the timestamp of the most recent enqueue (0 if never/reset). */
-export function getLastScheduledCleanupEnqueueMs(): number {
-  return lastScheduledCleanupEnqueueMs;
+/**
+ * Read the timestamp of the most recent enqueue for `kind` (0 if never/reset).
+ */
+export function getLastScheduledCleanupEnqueueMs(kind: CleanupJobKind): number {
+  return lastScheduledCleanupEnqueueMs[kind];
 }
 
-/** Record that an enqueue just happened at `nowMs`. */
-export function markScheduledCleanupEnqueued(nowMs: number): void {
-  lastScheduledCleanupEnqueueMs = nowMs;
+/** Record that an enqueue for `kind` just happened at `nowMs`. */
+export function markScheduledCleanupEnqueued(
+  kind: CleanupJobKind,
+  nowMs: number,
+): void {
+  lastScheduledCleanupEnqueueMs[kind] = nowMs;
 }
 
 /**
- * Clear the throttle so the next `maybeEnqueueScheduledCleanupJobs` call
- * bypasses the `enqueueIntervalMs` window. Used by ConfigWatcher when
- * retention settings change, and by tests that need deterministic
- * scheduling.
+ * Clear every job's in-memory throttle so the next
+ * `maybeEnqueueScheduledCleanupJobs` call re-enqueues immediately regardless of
+ * each job's retention window. Used by ConfigWatcher when retention settings
+ * change, and by tests that need deterministic scheduling.
+ *
+ * This clears only the in-memory timestamps; the persisted checkpoints are
+ * rewritten by the next enqueue, so they self-heal on the following tick.
  */
 export function resetCleanupScheduleThrottle(): void {
-  lastScheduledCleanupEnqueueMs = 0;
+  lastScheduledCleanupEnqueueMs.conversations = 0;
+  lastScheduledCleanupEnqueueMs.llm_request_logs = 0;
+  lastScheduledCleanupEnqueueMs.tool_invocations = 0;
 }

package/src/persistence/conversation-bootstrap.ts

@@ -16,9 +16,9 @@ export interface BootstrapConversationOptions {
   /**
    * When set, the new conversation is linked to its parent via the
    * `fork_parent_conversation_id` column. Used by background jobs that
-   * spawn analysis conversations off a source conversation (auto-analyze,
-   * memory-retrospective) so the parent → child relationship is queryable
-   * later (e.g. "find the most recent retrospective for this source").
+   * spawn conversations off a source conversation (memory-retrospective)
+   * so the parent → child relationship is queryable later (e.g. "find the
+   * most recent retrospective for this source").
    */
   forkParentConversationId?: string;
 }