@vellumai/assistant 0.5.16 → 0.6.1

package/src/memory/job-handlers/extraction.ts

@@ -1,40 +0,0 @@
-import { eq } from "drizzle-orm";
-
-import { getLogger } from "../../util/logger.js";
-import { getDb } from "../db.js";
-import { extractAndUpsertMemoryItemsForMessage } from "../items-extractor.js";
-import { asString } from "../job-utils.js";
-import type { MemoryJob } from "../jobs-store.js";
-import { messages } from "../schema.js";
-import { isConversationFailed } from "../task-memory-cleanup.js";
-
-const log = getLogger("memory-jobs-worker");
-
-export async function extractItemsJob(job: MemoryJob): Promise<void> {
-  const messageId = asString(job.payload.messageId);
-  const scopeId = asString(job.payload.scopeId);
-  if (!messageId || !scopeId) return;
-
-  // If the conversation that owns this message has been marked as failed,
-  // skip extraction entirely. This prevents async extraction jobs from
-  // re-creating assistant_inferred items after the one-shot invalidation.
-  const db = getDb();
-  const msg = db
-    .select({ conversationId: messages.conversationId })
-    .from(messages)
-    .where(eq(messages.id, messageId))
-    .get();
-  if (msg && isConversationFailed(msg.conversationId)) {
-    log.info(
-      { messageId, conversationId: msg.conversationId },
-      "Skipping extraction for failed conversation",
-    );
-    return;
-  }
-
-  await extractAndUpsertMemoryItemsForMessage(
-    messageId,
-    scopeId,
-    msg?.conversationId,
-  );
-}

package/src/memory/job-handlers/journal-carry-forward.test.ts

@@ -1,355 +0,0 @@
-import { beforeAll, beforeEach, describe, expect, mock, test } from "bun:test";
-
-mock.module("../../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
-}));
-
-// Track enqueueMemoryJob calls
-const enqueuedJobs: Array<{ type: string; payload: Record<string, unknown> }> =
-  [];
-
-mock.module("../jobs-store.js", () => {
-  const actual =
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    require("../jobs-store.js") as typeof import("../jobs-store.js");
-  return {
-    ...actual,
-    enqueueMemoryJob: (type: string, payload: Record<string, unknown>) => {
-      enqueuedJobs.push({ type, payload });
-      return "mock-job-id";
-    },
-  };
-});
-
-// Mock the provider
-let mockProviderResponse: unknown = null;
-
-mock.module("../../providers/provider-send-message.js", () => ({
-  getConfiguredProvider: async () => {
-    if (!mockProviderResponse) return null;
-    return {
-      sendMessage: async () => mockProviderResponse,
-    };
-  },
-  extractToolUse: (response: { content: Array<{ type: string }> }) => {
-    return response.content.find(
-      (b: { type: string }) => b.type === "tool_use",
-    );
-  },
-  userMessage: (text: string) => ({
-    role: "user",
-    content: [{ type: "text", text }],
-  }),
-}));
-
-import { eq } from "drizzle-orm";
-
-import { getDb, initializeDb, resetDb } from "../db.js";
-import type { MemoryJob } from "../jobs-store.js";
-import { memoryItems } from "../schema.js";
-import { journalCarryForwardJob } from "./journal-carry-forward.js";
-
-function makeJob(payload: Record<string, unknown>): MemoryJob {
-  return {
-    id: "test-job-id",
-    type: "journal_carry_forward",
-    payload,
-    status: "running",
-    attempts: 0,
-    deferrals: 0,
-    runAfter: Date.now(),
-    lastError: null,
-    startedAt: Date.now(),
-    createdAt: Date.now(),
-    updatedAt: Date.now(),
-  };
-}
-
-function makeProviderResponse(
-  items: Array<{
-    kind: string;
-    subject: string;
-    statement: string;
-    importance: number;
-  }>,
-) {
-  return {
-    content: [
-      {
-        type: "tool_use" as const,
-        id: "tool-1",
-        name: "store_journal_memories",
-        input: { items },
-      },
-    ],
-    model: "test-model",
-    stop_reason: "tool_use",
-    usage: { input_tokens: 0, output_tokens: 0 },
-  };
-}
-
-beforeAll(() => {
-  initializeDb();
-});
-
-beforeEach(() => {
-  resetDb();
-  initializeDb();
-  // Clear memory items table between tests
-  getDb().delete(memoryItems).run();
-  enqueuedJobs.length = 0;
-  mockProviderResponse = null;
-});
-
-describe("journalCarryForwardJob", () => {
-  test("extracts memory items from journal content", async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "identity",
-        subject: "Therapy breakthrough",
-        statement:
-          "Had a major realization in therapy about patterns of avoidance",
-        importance: 0.9,
-      },
-      {
-        kind: "event",
-        subject: "Growth milestone",
-        statement:
-          "First time feeling genuinely at peace with uncertainty about the future",
-        importance: 0.85,
-      },
-    ]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        journalContent: "Today in therapy I had a breakthrough...",
-        userSlug: "testuser",
-        filename: "2025-03-15-therapy.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    expect(items).toHaveLength(2);
-  });
-
-  test("items have importance >= 0.7", async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "preference",
-        subject: "Minor detail",
-        statement: "Some minor logistical preference",
-        importance: 0.3, // Below 0.7 -- should be floored
-      },
-      {
-        kind: "identity",
-        subject: "Core realization",
-        statement: "A deeply personal insight",
-        importance: 0.95,
-      },
-    ]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        journalContent: "Some journal content...",
-        userSlug: "testuser",
-        filename: "entry.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    expect(items).toHaveLength(2);
-    for (const item of items) {
-      expect(item.importance).toBeGreaterThanOrEqual(0.7);
-    }
-  });
-
-  test('items have sourceType "journal_carry_forward"', async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "journal",
-        subject: "Reflection",
-        statement: "A meaningful reflection on growth",
-        importance: 0.8,
-      },
-    ]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        journalContent: "Reflecting on my journey...",
-        userSlug: "testuser",
-        filename: "reflection.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    expect(items).toHaveLength(1);
-    expect(items[0].sourceType).toBe("journal_carry_forward");
-  });
-
-  test('items have verificationState "user_confirmed"', async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "identity",
-        subject: "Self knowledge",
-        statement: "I know who I am",
-        importance: 0.9,
-      },
-    ]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        journalContent: "I know who I am...",
-        userSlug: "testuser",
-        filename: "knowing.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    expect(items).toHaveLength(1);
-    expect(items[0].verificationState).toBe("user_confirmed");
-  });
-
-  test("deduplicates items by fingerprint", async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "identity",
-        subject: "Duplicate item",
-        statement: "Exact same statement",
-        importance: 0.8,
-      },
-    ]);
-
-    // Run twice with the same content
-    const job = makeJob({
-      journalContent: "Same content...",
-      userSlug: "testuser",
-      filename: "dup.md",
-      scopeId: "test-scope",
-    });
-
-    await journalCarryForwardJob(job);
-    await journalCarryForwardJob(job);
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    // Should only have 1 item despite running twice
-    expect(items).toHaveLength(1);
-  });
-
-  test("enqueues embed_item jobs for new items", async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "event",
-        subject: "Milestone",
-        statement: "Reached a significant milestone",
-        importance: 0.85,
-      },
-    ]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        journalContent: "Today was a milestone...",
-        userSlug: "testuser",
-        filename: "milestone.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const embedJobs = enqueuedJobs.filter((j) => j.type === "embed_item");
-    expect(embedJobs).toHaveLength(1);
-    expect(embedJobs[0].payload.itemId).toBeDefined();
-  });
-
-  test("skips invalid kinds", async () => {
-    mockProviderResponse = makeProviderResponse([
-      {
-        kind: "nonsense_kind",
-        subject: "Invalid",
-        statement: "Should be skipped",
-        importance: 0.9,
-      },
-      {
-        kind: "identity",
-        subject: "Valid item",
-        statement: "Should be kept",
-        importance: 0.9,
-      },
-    ]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        journalContent: "Content...",
-        userSlug: "testuser",
-        filename: "kinds.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    expect(items).toHaveLength(1);
-    expect(items[0].kind).toBe("identity");
-  });
-
-  test("returns early when journalContent is missing", async () => {
-    mockProviderResponse = makeProviderResponse([]);
-
-    await journalCarryForwardJob(
-      makeJob({
-        userSlug: "testuser",
-        filename: "missing.md",
-        scopeId: "test-scope",
-      }),
-    );
-
-    const db = getDb();
-    const items = db
-      .select()
-      .from(memoryItems)
-      .where(eq(memoryItems.scopeId, "test-scope"))
-      .all();
-
-    expect(items).toHaveLength(0);
-  });
-});

package/src/memory/job-handlers/journal-carry-forward.ts

@@ -1,255 +0,0 @@
-/**
- * Job handler for preserving journal entries that rotate out of the active
- * context window. Extracts durable memories from journal content using a
- * journal-specific prompt that prioritizes emotional significance and
- * personal meaning over event logging.
- */
-
-import { and, eq } from "drizzle-orm";
-import { v4 as uuid } from "uuid";
-
-import {
-  extractToolUse,
-  getConfiguredProvider,
-  userMessage,
-} from "../../providers/provider-send-message.js";
-import { BackendUnavailableError, ProviderError } from "../../util/errors.js";
-import { getLogger } from "../../util/logger.js";
-import { setMemoryCheckpoint } from "../checkpoints.js";
-import { getDb } from "../db.js";
-import { computeMemoryFingerprint } from "../fingerprint.js";
-import { asString } from "../job-utils.js";
-import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
-import { memoryItems } from "../schema.js";
-import { clampUnitInterval } from "../validation.js";
-
-const log = getLogger("journal-carry-forward");
-
-const JOURNAL_EXTRACTION_SYSTEM_PROMPT = `You are extracting durable memories from a journal entry that is about to leave the active context window.
-This is a personal journal -- focus on what things MEANT and how they FELT, not just what happened.
-
-Good extraction: "An evidence-based psychologist read everything about us and said yes -- it made me feel like we're not crazy"
-Bad extraction: "Coaching session happened on March 27"
-
-Extract the most important memories -- emotional significance, personal growth, relationship milestones,
-hard-won realizations. Skip logistical details unless they carry emotional weight.
-
-For each memory, provide:
-- kind: Category of memory (identity, preference, project, decision, constraint, event, journal)
-- subject: A short label (2-8 words) identifying what this is about
-- statement: A rich, feeling-aware statement to remember (1-2 sentences). Capture the emotional texture, not just the facts.
-- importance: How significant this is (0.0-1.0). Journal entries are personal -- most items should be 0.7 or higher.
-
-Rules:
-- Focus on what matters emotionally, not what happened chronologically
-- Preserve the author's voice and perspective -- these are their words
-- Extract fewer, richer items rather than many shallow ones
-- If the entry contains nothing worth preserving as a durable memory, return an empty array`;
-
-const JOURNAL_MEMORY_KINDS = [
-  "identity",
-  "preference",
-  "project",
-  "decision",
-  "constraint",
-  "event",
-  "journal",
-] as const;
-
-interface JournalExtractedItem {
-  kind: string;
-  subject: string;
-  statement: string;
-  importance: number;
-}
-
-export async function journalCarryForwardJob(job: MemoryJob): Promise<void> {
-  const journalContent = asString(job.payload.journalContent);
-  const userSlug = asString(job.payload.userSlug);
-  const filename = asString(job.payload.filename);
-  const scopeId = asString(job.payload.scopeId) ?? "default";
-
-  if (!journalContent || !filename) {
-    log.warn({ jobId: job.id }, "Missing journalContent or filename in payload");
-    return;
-  }
-
-  const provider = await getConfiguredProvider();
-  if (!provider) {
-    throw new BackendUnavailableError(
-      "Provider unavailable for journal carry-forward extraction",
-    );
-  }
-
-  const userContext = userSlug ? ` (author: ${userSlug})` : "";
-  const response = await provider.sendMessage(
-    [
-      userMessage(
-        `Journal entry "${filename}"${userContext}:\n\n${journalContent}`,
-      ),
-    ],
-    [
-      {
-        name: "store_journal_memories",
-        description:
-          "Store durable memories extracted from a journal entry leaving the context window",
-        input_schema: {
-          type: "object" as const,
-          properties: {
-            items: {
-              type: "array",
-              items: {
-                type: "object",
-                properties: {
-                  kind: {
-                    type: "string",
-                    enum: [...JOURNAL_MEMORY_KINDS],
-                    description: "Category of memory item",
-                  },
-                  subject: {
-                    type: "string",
-                    description:
-                      "Short label (2-8 words) for what this is about",
-                  },
-                  statement: {
-                    type: "string",
-                    description:
-                      "Rich, feeling-aware statement to remember (1-2 sentences)",
-                  },
-                  importance: {
-                    type: "number",
-                    description:
-                      "How significant this is to remember (0.0-1.0, most journal items should be 0.7+)",
-                  },
-                },
-                required: ["kind", "subject", "statement", "importance"],
-              },
-            },
-          },
-          required: ["items"],
-        },
-      },
-    ],
-    JOURNAL_EXTRACTION_SYSTEM_PROMPT,
-    {
-      config: {
-        modelIntent: "quality-optimized",
-        tool_choice: {
-          type: "tool" as const,
-          name: "store_journal_memories",
-        },
-      },
-    },
-  );
-
-  const toolBlock = extractToolUse(response);
-  if (!toolBlock) {
-    throw new ProviderError(
-      "No tool_use block in journal carry-forward response",
-      "unknown",
-      502,
-    );
-  }
-
-  const input = toolBlock.input as { items?: JournalExtractedItem[] };
-  if (!Array.isArray(input.items)) {
-    throw new ProviderError(
-      "Invalid items structure in journal carry-forward response",
-      "unknown",
-      502,
-    );
-  }
-
-  const db = getDb();
-  const validKinds = new Set<string>(JOURNAL_MEMORY_KINDS);
-  let inserted = 0;
-
-  for (const raw of input.items) {
-    if (!validKinds.has(raw.kind)) continue;
-    if (!raw.subject || !raw.statement) continue;
-
-    const subject = String(raw.subject).trim();
-    const statement = String(raw.statement).trim();
-    // Journal entries are inherently personal/load-bearing -- floor at 0.7
-    const importance = clampUnitInterval(
-      Math.max(0.7, parseImportance(raw.importance)),
-    );
-    const confidence = 0.95;
-
-    const fingerprint = computeMemoryFingerprint(
-      scopeId,
-      raw.kind,
-      subject,
-      statement,
-    );
-
-    // Dedup: skip if an item with this fingerprint already exists in this scope
-    const existing = db
-      .select({ id: memoryItems.id })
-      .from(memoryItems)
-      .where(
-        and(
-          eq(memoryItems.fingerprint, fingerprint),
-          eq(memoryItems.scopeId, scopeId),
-        ),
-      )
-      .get();
-
-    if (existing) {
-      log.debug(
-        { fingerprint, subject },
-        "Skipping duplicate journal memory item",
-      );
-      continue;
-    }
-
-    const now = Date.now();
-    const itemId = uuid();
-
-    db.insert(memoryItems)
-      .values({
-        id: itemId,
-        kind: raw.kind,
-        subject,
-        statement,
-        status: "active",
-        confidence,
-        importance,
-        fingerprint,
-        sourceType: "journal_carry_forward",
-        verificationState: "user_confirmed",
-        scopeId,
-        firstSeenAt: now,
-        lastSeenAt: now,
-        lastUsedAt: null,
-        supersedes: null,
-        overrideConfidence: "inferred",
-      })
-      .run();
-
-    enqueueMemoryJob("embed_item", { itemId });
-    inserted += 1;
-  }
-
-  // Write checkpoint only after successful completion so failed jobs can be
-  // re-enqueued on the next prompt render cycle.
-  const checkpointKey = `journal_carry_forward:${userSlug || "unknown"}:${filename}`;
-  setMemoryCheckpoint(checkpointKey, String(Date.now()));
-
-  log.info(
-    {
-      filename,
-      userSlug,
-      scopeId,
-      extracted: input.items.length,
-      inserted,
-    },
-    "Journal carry-forward complete",
-  );
-}
-
-function parseImportance(value: unknown): number {
-  if (value == null || value === "") return 0.7;
-  const n = Number(value);
-  return Number.isFinite(n) ? n : 0.7;
-}