@vellumai/assistant 0.6.3 → 0.6.4

package/src/memory/__tests__/conversation-analyze-job.test.ts

@@ -0,0 +1,232 @@
+/**
+ * Unit tests for the `conversation_analyze` job handler.
+ *
+ * The handler bridges the jobs worker to `analyzeConversation()` via the
+ * singleton deps bundle. Tests stub both the singleton and the service so we
+ * exercise dispatch logic without pulling in HTTP-layer wiring.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+// Mock analyze-deps singleton — each test overrides via mockGetAnalysisDeps.
+type DepsStub = Record<string, unknown>;
+const mockGetAnalysisDeps = mock((): DepsStub | null => null);
+
+mock.module("../../runtime/services/analyze-deps-singleton.js", () => ({
+  getAnalysisDeps: mockGetAnalysisDeps,
+}));
+
+// Mock analyze-conversation service — default resolves with a success result.
+type AnalyzeArgs = {
+  conversationId: string;
+  deps: DepsStub;
+  opts: { trigger: "manual" | "auto" };
+};
+const analyzeCalls: AnalyzeArgs[] = [];
+type AnalyzeResultStub =
+  | { analysisConversationId: string; skipped?: true }
+  | { error: { kind: string; status: number; message: string } };
+const mockAnalyzeConversation = mock(
+  async (
+    conversationId: string,
+    deps: DepsStub,
+    opts: { trigger: "manual" | "auto" },
+  ): Promise<AnalyzeResultStub> => {
+    analyzeCalls.push({ conversationId, deps, opts });
+    return { analysisConversationId: "analysis-1" };
+  },
+);
+
+mock.module("../../runtime/services/analyze-conversation.js", () => ({
+  analyzeConversation: mockAnalyzeConversation,
+}));
+
+// Mock auto-analysis-enqueue — track calls so we can verify requeue behavior.
+type EnqueueArgs = {
+  conversationId: string;
+  trigger: string;
+};
+const enqueueCalls: EnqueueArgs[] = [];
+const mockEnqueueAutoAnalysisIfEnabled = mock((args: EnqueueArgs) => {
+  enqueueCalls.push(args);
+});
+
+mock.module("../auto-analysis-enqueue.js", () => ({
+  enqueueAutoAnalysisIfEnabled: mockEnqueueAutoAnalysisIfEnabled,
+}));
+
+import { DEFAULT_CONFIG } from "../../config/defaults.js";
+import type { AssistantConfig } from "../../config/types.js";
+import { conversationAnalyzeJob } from "../conversation-analyze-job.js";
+import type { MemoryJob } from "../jobs-store.js";
+
+const TEST_CONFIG: AssistantConfig = DEFAULT_CONFIG;
+
+function makeJob(payload: Record<string, unknown>): MemoryJob<{
+  conversationId?: string;
+}> {
+  return {
+    id: "job-1",
+    type: "conversation_analyze",
+    payload: payload as { conversationId?: string },
+    status: "running",
+    attempts: 0,
+    deferrals: 0,
+    runAfter: 0,
+    lastError: null,
+    startedAt: Date.now(),
+    createdAt: Date.now(),
+    updatedAt: Date.now(),
+  };
+}
+
+describe("conversationAnalyzeJob", () => {
+  beforeEach(() => {
+    analyzeCalls.length = 0;
+    enqueueCalls.length = 0;
+    mockGetAnalysisDeps.mockReset();
+    mockGetAnalysisDeps.mockImplementation(() => null);
+    mockAnalyzeConversation.mockReset();
+    mockAnalyzeConversation.mockImplementation(
+      async (
+        conversationId: string,
+        deps: DepsStub,
+        opts: { trigger: "manual" | "auto" },
+      ) => {
+        analyzeCalls.push({ conversationId, deps, opts });
+        return { analysisConversationId: "analysis-1" };
+      },
+    );
+  });
+
+  test("returns without calling the service when conversationId is missing", async () => {
+    await conversationAnalyzeJob(makeJob({}), TEST_CONFIG);
+    expect(analyzeCalls).toHaveLength(0);
+    expect(mockGetAnalysisDeps).not.toHaveBeenCalled();
+  });
+
+  test("returns without calling the service when conversationId is empty string", async () => {
+    await conversationAnalyzeJob(
+      makeJob({ conversationId: "" }),
+      TEST_CONFIG,
+    );
+    expect(analyzeCalls).toHaveLength(0);
+    expect(mockGetAnalysisDeps).not.toHaveBeenCalled();
+  });
+
+  test("returns without calling the service when deps singleton is not yet initialized", async () => {
+    // Deps singleton is unset during slow daemon startup. The handler must
+    // NOT throw — a plain Error here is classified as fatal by
+    // classifyError() and the worker would mark the job permanently failed
+    // (failMemoryJob with maxAttempts: 1). Returning lets the next batch /
+    // idle / lifecycle trigger from enqueueAutoAnalysisIfEnabled() produce
+    // a fresh job once the daemon has fully started.
+    mockGetAnalysisDeps.mockImplementation(() => null);
+    await expect(
+      conversationAnalyzeJob(
+        makeJob({ conversationId: "conv-1" }),
+        TEST_CONFIG,
+      ),
+    ).resolves.toBeUndefined();
+    expect(analyzeCalls).toHaveLength(0);
+    expect(mockGetAnalysisDeps).toHaveBeenCalled();
+  });
+
+  test("invokes analyzeConversation with trigger=auto and the conversationId", async () => {
+    const depsStub: DepsStub = { _tag: "deps-stub" };
+    mockGetAnalysisDeps.mockImplementation(() => depsStub);
+
+    await conversationAnalyzeJob(
+      makeJob({ conversationId: "conv-42" }),
+      TEST_CONFIG,
+    );
+
+    expect(analyzeCalls).toHaveLength(1);
+    expect(analyzeCalls[0]!.conversationId).toBe("conv-42");
+    expect(analyzeCalls[0]!.opts).toEqual({ trigger: "auto" });
+    expect(analyzeCalls[0]!.deps).toBe(depsStub);
+  });
+
+  test("requeues a follow-up idle trigger when the service returns skipped=true", async () => {
+    // If the rolling analysis conversation is already processing, the
+    // service returns { skipped: true } without running. The job handler
+    // completes successfully (no retry), so we must enqueue a follow-up
+    // ourselves — otherwise, if no later batch/idle/lifecycle trigger
+    // arrives, new source messages would never be analyzed.
+    mockGetAnalysisDeps.mockImplementation(() => ({ _tag: "deps" }));
+    mockAnalyzeConversation.mockImplementation(async () => ({
+      analysisConversationId: "analysis-1",
+      skipped: true as const,
+    }));
+
+    await conversationAnalyzeJob(
+      makeJob({ conversationId: "conv-busy" }),
+      TEST_CONFIG,
+    );
+
+    expect(enqueueCalls).toHaveLength(1);
+    expect(enqueueCalls[0]).toEqual({
+      conversationId: "conv-busy",
+      trigger: "idle",
+    });
+  });
+
+  test("does not requeue on a normal (non-skipped) successful run", async () => {
+    mockGetAnalysisDeps.mockImplementation(() => ({ _tag: "deps" }));
+    mockAnalyzeConversation.mockImplementation(async () => ({
+      analysisConversationId: "analysis-1",
+    }));
+
+    await conversationAnalyzeJob(
+      makeJob({ conversationId: "conv-ok" }),
+      TEST_CONFIG,
+    );
+
+    expect(enqueueCalls).toHaveLength(0);
+  });
+
+  test("does not requeue when the service returns an error result", async () => {
+    mockGetAnalysisDeps.mockImplementation(() => ({ _tag: "deps" }));
+    mockAnalyzeConversation.mockImplementation(async () => ({
+      error: {
+        kind: "BAD_REQUEST",
+        status: 400,
+        message: "Cannot auto-analyze an auto-analysis conversation",
+      },
+    }));
+
+    await conversationAnalyzeJob(
+      makeJob({ conversationId: "conv-err" }),
+      TEST_CONFIG,
+    );
+
+    expect(enqueueCalls).toHaveLength(0);
+  });
+
+  test("swallows (does not throw) when the service returns an error result", async () => {
+    mockGetAnalysisDeps.mockImplementation(() => ({ _tag: "deps" }));
+    mockAnalyzeConversation.mockImplementation(async () => ({
+      error: {
+        kind: "BAD_REQUEST",
+        status: 400,
+        message: "Cannot auto-analyze an auto-analysis conversation",
+      },
+    }));
+
+    // Must not throw — the worker would otherwise retry forever on a
+    // deterministic rejection (e.g. the recursion guard).
+    await expect(
+      conversationAnalyzeJob(
+        makeJob({ conversationId: "conv-2" }),
+        TEST_CONFIG,
+      ),
+    ).resolves.toBeUndefined();
+  });
+});

package/src/memory/__tests__/find-analysis-conversation.test.ts

@@ -0,0 +1,196 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { eq } from "drizzle-orm";
+
+mock.module("../../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+import {
+  createConversation,
+  findAnalysisConversationFor,
+  getConversationSource,
+} from "../conversation-crud.js";
+import { getDb, initializeDb } from "../db.js";
+import { conversations } from "../schema.js";
+
+initializeDb();
+
+function resetTables(): void {
+  const db = getDb();
+  db.run(`DELETE FROM messages`);
+  db.run(`DELETE FROM conversations`);
+}
+
+function setForkParent(conversationId: string, parentId: string): void {
+  const db = getDb();
+  db.update(conversations)
+    .set({ forkParentConversationId: parentId })
+    .where(eq(conversations.id, conversationId))
+    .run();
+}
+
+function setUpdatedAt(conversationId: string, updatedAt: number): void {
+  const db = getDb();
+  db.update(conversations)
+    .set({ updatedAt })
+    .where(eq(conversations.id, conversationId))
+    .run();
+}
+
+describe("findAnalysisConversationFor", () => {
+  beforeEach(() => {
+    resetTables();
+  });
+
+  test("returns null when no analysis conversation exists for the parent", () => {
+    const parent = createConversation("parent");
+    expect(findAnalysisConversationFor(parent.id)).toBeNull();
+  });
+
+  test("returns null when an unrelated analysis conversation exists", () => {
+    const parent = createConversation("parent");
+    const other = createConversation("other");
+    const analysisForOther = createConversation({
+      title: "analysis of other",
+      source: "auto-analysis",
+    });
+    setForkParent(analysisForOther.id, other.id);
+
+    expect(findAnalysisConversationFor(parent.id)).toBeNull();
+  });
+
+  test("returns the only matching analysis conversation when one exists", () => {
+    const parent = createConversation("parent");
+    const analysis = createConversation({
+      title: "rolling analysis",
+      source: "auto-analysis",
+    });
+    setForkParent(analysis.id, parent.id);
+
+    expect(findAnalysisConversationFor(parent.id)).toEqual({ id: analysis.id });
+  });
+
+  test("when multiple match, returns the most recently updated one", () => {
+    const parent = createConversation("parent");
+
+    const older = createConversation({
+      title: "older analysis",
+      source: "auto-analysis",
+    });
+    setForkParent(older.id, parent.id);
+    setUpdatedAt(older.id, 1_000);
+
+    const newer = createConversation({
+      title: "newer analysis",
+      source: "auto-analysis",
+    });
+    setForkParent(newer.id, parent.id);
+    setUpdatedAt(newer.id, 2_000);
+
+    const middle = createConversation({
+      title: "middle analysis",
+      source: "auto-analysis",
+    });
+    setForkParent(middle.id, parent.id);
+    setUpdatedAt(middle.id, 1_500);
+
+    expect(findAnalysisConversationFor(parent.id)).toEqual({ id: newer.id });
+  });
+
+  test("does not return regular user conversations whose forkParentConversationId matches", () => {
+    const parent = createConversation("parent");
+
+    // A regular user-forked conversation (source defaults to "user").
+    const userFork = createConversation("user fork");
+    setForkParent(userFork.id, parent.id);
+
+    expect(findAnalysisConversationFor(parent.id)).toBeNull();
+  });
+
+  test("ignores user forks even when an analysis conversation also exists", () => {
+    const parent = createConversation("parent");
+
+    const userFork = createConversation("user fork");
+    setForkParent(userFork.id, parent.id);
+    // Force the user fork to have the most recent updatedAt — it should
+    // still be ignored because its source is "user".
+    setUpdatedAt(userFork.id, 9_999);
+
+    const analysis = createConversation({
+      title: "analysis",
+      source: "auto-analysis",
+    });
+    setForkParent(analysis.id, parent.id);
+    setUpdatedAt(analysis.id, 1_000);
+
+    expect(findAnalysisConversationFor(parent.id)).toEqual({ id: analysis.id });
+  });
+
+  test("createConversation persists forkParentConversationId when supplied", () => {
+    const parent = createConversation("parent");
+
+    // Auto-analyze path creates the rolling analysis conversation with
+    // source + forkParentConversationId in the same call.
+    const analysis = createConversation({
+      title: "rolling analysis",
+      source: "auto-analysis",
+      forkParentConversationId: parent.id,
+    });
+
+    expect(findAnalysisConversationFor(parent.id)).toEqual({ id: analysis.id });
+  });
+
+  test("finds rolling analysis conversation regardless of group_id (backward-compat across the dedicated-group migration)", () => {
+    const parent = createConversation("parent");
+
+    const legacyAnalysis = createConversation({
+      title: "legacy rolling analysis",
+      source: "auto-analysis",
+      forkParentConversationId: parent.id,
+    });
+    setUpdatedAt(legacyAnalysis.id, 1_000);
+
+    expect(findAnalysisConversationFor(parent.id)).toEqual({
+      id: legacyAnalysis.id,
+    });
+
+    const newAnalysis = createConversation({
+      title: "new rolling analysis",
+      source: "auto-analysis",
+      groupId: "system:background",
+      forkParentConversationId: parent.id,
+    });
+    setUpdatedAt(newAnalysis.id, 2_000);
+
+    expect(findAnalysisConversationFor(parent.id)).toEqual({
+      id: newAnalysis.id,
+    });
+  });
+});
+
+describe("getConversationSource", () => {
+  beforeEach(() => {
+    resetTables();
+  });
+
+  test("returns the source string for an existing conversation", () => {
+    const conv = createConversation("user conv");
+    expect(getConversationSource(conv.id)).toBe("user");
+  });
+
+  test("returns the custom source for an analysis conversation", () => {
+    const conv = createConversation({
+      title: "analysis",
+      source: "auto-analysis",
+    });
+    expect(getConversationSource(conv.id)).toBe("auto-analysis");
+  });
+
+  test("returns null for a non-existent conversation ID", () => {
+    expect(getConversationSource("does-not-exist")).toBeNull();
+  });
+});

package/src/memory/app-store.ts

@@ -78,7 +78,7 @@ export function resolveEffectiveAppHtml(app: AppDefinition): string {
   if (existsSync(distIndex)) {
     return inlineDistAssets(appDir, readFileSync(distIndex, "utf-8"));
   }
-  return app.htmlDefinition;
+  return `<p>App compilation failed. Edit a source file to trigger a rebuild.</p>`;
 }
 
 /**

package/src/memory/attachments-store.ts

@@ -11,6 +11,7 @@ import {
   existsSync,
   mkdirSync,
   readFileSync,
+  realpathSync,
   unlinkSync,
   writeFileSync,
 } from "node:fs";
@@ -264,7 +265,40 @@ function materializeAttachmentIntoConversation(
     copyFileSync(readablePath, targetPath);
   }
 
+  // Remember the old file path before updating the DB row, so we can
+  // clean up the staging copy (e.g. in data/attachments/) after the
+  // canonical path moves to the conversation directory.
+  const previousFilePath = row.filePath;
+
   persistAttachmentFilePath(row.id, targetPath, sourcePath);
+
+  // Remove the old staging file now that the canonical copy lives in
+  // the conversation directory.  Only delete files that live in the
+  // staging area (workspace/data/attachments/).  When an attachment is
+  // cloned across conversations (e.g. during a fork), previousFilePath
+  // may point to another conversation's directory — deleting that would
+  // cause data loss for the source conversation.
+  const stagingDirRaw = join(getWorkspaceDir(), "data", "attachments");
+  let stagingDir: string;
+  try {
+    stagingDir = existsSync(stagingDirRaw)
+      ? realpathSync(stagingDirRaw)
+      : stagingDirRaw;
+  } catch {
+    stagingDir = stagingDirRaw;
+  }
+  if (
+    previousFilePath &&
+    previousFilePath !== targetPath &&
+    dirname(previousFilePath) === stagingDir &&
+    existsSync(previousFilePath)
+  ) {
+    try {
+      unlinkSync(previousFilePath);
+    } catch {
+      /* file may already be gone */
+    }
+  }
 }
 
 function scopeAttachmentToConversation(
@@ -458,6 +492,42 @@ export function validateAttachmentUpload(
   return { ok: true };
 }
 
+// ---------------------------------------------------------------------------
+// Binary upload helper (multipart / octet-stream)
+// ---------------------------------------------------------------------------
+
+/**
+ * Write raw bytes to the staging directory and register as a file-backed
+ * attachment. Used by the multipart/form-data and application/octet-stream
+ * upload paths.
+ *
+ * @param filename  Original filename from the client
+ * @param mimeType  MIME type of the file
+ * @param bytes     Raw file content
+ * @returns The stored attachment record
+ */
+export function uploadAttachmentFromBytes(
+  filename: string,
+  mimeType: string,
+  bytes: Uint8Array,
+): StoredAttachment {
+  const dir = join(getWorkspaceDir(), "data", "attachments");
+  mkdirSync(dir, { recursive: true });
+
+  const sanitized = filename.replace(/[^a-zA-Z0-9._-]/g, "_");
+  const stagingFilename = `${Date.now()}-${uuid().slice(0, 8)}-${sanitized}`;
+  const stagedPath = join(dir, stagingFilename);
+
+  writeFileSync(stagedPath, bytes);
+
+  return uploadFileBackedAttachment(
+    filename,
+    mimeType,
+    stagedPath,
+    bytes.length,
+  );
+}
+
 // ---------------------------------------------------------------------------
 // File-backed attachment storage (avoids reading large files into memory)
 // ---------------------------------------------------------------------------

package/src/memory/auto-analysis-enqueue.ts

@@ -0,0 +1,127 @@
+import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
+import { getConfig } from "../config/loader.js";
+import {
+  isUntrustedTrustClass,
+  type TrustClass,
+} from "../runtime/actor-trust-resolver.js";
+import { getLogger } from "../util/logger.js";
+import { isAutoAnalysisConversation } from "./auto-analysis-guard.js";
+import { getConversationType } from "./conversation-crud.js";
+import { upsertAutoAnalysisJob } from "./jobs-store.js";
+
+const log = getLogger("auto-analysis-enqueue");
+
+/**
+ * Trigger reason for an auto-analysis enqueue.
+ *   - `"batch"`: source conversation crossed the batch threshold — enqueue
+ *     immediately (`runAfter = now`) but still upsert so a pending job
+ *     coalesces rapid threshold crossings into one.
+ *   - `"idle"`: source conversation has been idle long enough to warrant a
+ *     debounced analysis pass.
+ *   - `"lifecycle"`: a conversation lifecycle transition (e.g. resume,
+ *     close) should trigger a debounced analysis pass.
+ *   - `"compaction"`: context was just compacted — some recent turns are
+ *     now hidden behind a summary, so crystallize anything worth
+ *     remembering before the window narrows further. Fires immediately
+ *     (`runAfter = now`) like `"batch"`.
+ */
+export type AutoAnalysisTrigger =
+  | "batch"
+  | "idle"
+  | "lifecycle"
+  | "compaction";
+
+/**
+ * Conditionally enqueue a `conversation_analyze` job for the given
+ * conversation. Skips silently when:
+ *   - the `auto-analyze` feature flag is disabled, OR
+ *   - the source conversation is itself an auto-analysis conversation
+ *     (recursion guard — we never analyze our own analysis output), OR
+ *   - the source conversation is private (`analyzeConversation` rejects
+ *     private conversations, so enqueueing would guarantee a failed job).
+ *
+ * Immediate triggers (`"batch"`, `"compaction"`) and debounced triggers
+ * (`"idle"`, `"lifecycle"`) are written to separate rows keyed by a
+ * `triggerGroup` discriminator. This prevents an idle enqueue from
+ * pushing an already-scheduled batch row's `runAfter` into the future
+ * (and vice versa). Within each group, rapid enqueues still coalesce to
+ * a single pending row via `upsertAutoAnalysisJob`.
+ */
+export function enqueueAutoAnalysisIfEnabled(args: {
+  conversationId: string;
+  trigger: AutoAnalysisTrigger;
+}): void {
+  const { conversationId, trigger } = args;
+
+  let config;
+  try {
+    config = getConfig();
+  } catch (err) {
+    log.warn(
+      { err, conversationId },
+      "Skipping auto-analysis enqueue: failed to load config",
+    );
+    return;
+  }
+
+  if (!isAssistantFeatureFlagEnabled("auto-analyze", config)) {
+    return;
+  }
+
+  if (isAutoAnalysisConversation(conversationId)) {
+    log.debug(
+      { conversationId, trigger },
+      "Skipping auto-analysis enqueue: source is an auto-analysis conversation",
+    );
+    return;
+  }
+
+  if (getConversationType(conversationId) === "private") {
+    log.debug(
+      { conversationId, trigger },
+      "Skipping auto-analysis enqueue: source is a private conversation",
+    );
+    return;
+  }
+
+  const idleTimeoutMs = config.analysis?.idleTimeoutMs ?? 600_000;
+  const runImmediately = trigger === "batch" || trigger === "compaction";
+  const triggerGroup: "immediate" | "debounced" = runImmediately
+    ? "immediate"
+    : "debounced";
+  const runAfter = runImmediately ? Date.now() : Date.now() + idleTimeoutMs;
+
+  try {
+    upsertAutoAnalysisJob({ conversationId, triggerGroup }, runAfter);
+  } catch (err) {
+    log.warn(
+      { err, conversationId, trigger },
+      "Failed to enqueue auto-analysis job",
+    );
+  }
+}
+
+/**
+ * Fire an auto-analysis enqueue from a compaction site. Wraps
+ * `enqueueAutoAnalysisIfEnabled` with the trust-class gate and
+ * best-effort error handling used at every compaction call site, so
+ * the six compaction paths (forceCompact, preflight, overflow reducer,
+ * mid-loop, and two emergency paths) stay in sync.
+ *
+ * Trust gate mirrors the memory-extraction trust boundary applied in
+ * `disposeConversation` — we don't trigger analysis (which runs with
+ * guardian trust + full tools) for conversations with an untrusted actor.
+ */
+export function enqueueAutoAnalysisOnCompaction(
+  conversationId: string,
+  trustClass: TrustClass | undefined,
+): void {
+  if (isUntrustedTrustClass(trustClass)) {
+    return;
+  }
+  try {
+    enqueueAutoAnalysisIfEnabled({ conversationId, trigger: "compaction" });
+  } catch {
+    // Best-effort — never block compaction on enqueue failures.
+  }
+}

package/src/memory/auto-analysis-guard.ts

@@ -0,0 +1,27 @@
+import { getConversationSource } from "./conversation-crud.js";
+
+/**
+ * The `source` value used for conversations created by the auto-analysis
+ * loop. Single source of truth — downstream code (enqueue helper,
+ * service auto-branch) imports this constant rather than hardcoding the
+ * string.
+ */
+export const AUTO_ANALYSIS_SOURCE = "auto-analysis";
+
+/**
+ * Dedicated `group_id` value for auto-analysis rolling conversations.
+ * Placed in the `system:background` group alongside heartbeat and filing
+ * conversations, rendered as a "Reflections" sub-group in the sidebar.
+ */
+export const AUTO_ANALYSIS_GROUP_ID = "system:background";
+
+/**
+ * Returns true if the conversation's `source` column is `"auto-analysis"`,
+ * meaning it was produced by the auto-analysis loop. Callers use this to
+ * skip both `graph_extract` and `conversation_analyze` enqueues so we
+ * never (a) analyze our own analysis output or (b) extract memory from
+ * reflective musings (the analysis agent writes memory directly via tools).
+ */
+export function isAutoAnalysisConversation(conversationId: string): boolean {
+  return getConversationSource(conversationId) === AUTO_ANALYSIS_SOURCE;
+}